1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo 5 * Copyright (C) 2013-2016 Universita` di Pisa 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * $FreeBSD$ 32 * 33 * The header contains the definitions of constants and function 34 * prototypes used only in kernelspace. 35 */ 36 37 #ifndef _NET_NETMAP_KERN_H_ 38 #define _NET_NETMAP_KERN_H_ 39 40 #if defined(linux) 41 42 #if defined(CONFIG_NETMAP_EXTMEM) 43 #define WITH_EXTMEM 44 #endif 45 #if defined(CONFIG_NETMAP_VALE) 46 #define WITH_VALE 47 #endif 48 #if defined(CONFIG_NETMAP_PIPE) 49 #define WITH_PIPES 50 #endif 51 #if defined(CONFIG_NETMAP_MONITOR) 52 #define WITH_MONITOR 53 #endif 54 #if defined(CONFIG_NETMAP_GENERIC) 55 #define WITH_GENERIC 56 #endif 57 #if defined(CONFIG_NETMAP_PTNETMAP_GUEST) 58 #define WITH_PTNETMAP_GUEST 59 #endif 60 #if defined(CONFIG_NETMAP_PTNETMAP_HOST) 61 #define WITH_PTNETMAP_HOST 62 #endif 63 #if defined(CONFIG_NETMAP_SINK) 64 #define WITH_SINK 65 #endif 66 67 #elif defined (_WIN32) 68 #define WITH_VALE // comment out to disable VALE support 69 #define WITH_PIPES 70 #define WITH_MONITOR 71 #define WITH_GENERIC 72 73 #else /* neither linux nor windows */ 74 #define WITH_VALE // comment out to disable VALE support 75 #define WITH_PIPES 76 #define WITH_MONITOR 77 #define WITH_GENERIC 78 #define WITH_PTNETMAP_HOST /* ptnetmap host support */ 79 #define WITH_PTNETMAP_GUEST /* ptnetmap guest support */ 80 81 #endif 82 83 #if defined(__FreeBSD__) 84 #include <sys/selinfo.h> 85 86 #define likely(x) __builtin_expect((long)!!(x), 1L) 87 #define unlikely(x) __builtin_expect((long)!!(x), 0L) 88 #define __user 89 90 #define NM_LOCK_T struct mtx /* low level spinlock, used to protect queues */ 91 92 #define NM_MTX_T struct sx /* OS-specific mutex (sleepable) */ 93 #define NM_MTX_INIT(m) sx_init(&(m), #m) 94 #define NM_MTX_DESTROY(m) sx_destroy(&(m)) 95 #define NM_MTX_LOCK(m) sx_xlock(&(m)) 96 #define NM_MTX_SPINLOCK(m) while (!sx_try_xlock(&(m))) ; 97 #define NM_MTX_UNLOCK(m) sx_xunlock(&(m)) 98 #define NM_MTX_ASSERT(m) sx_assert(&(m), SA_XLOCKED) 99 100 #define NM_SELINFO_T struct nm_selinfo 101 #define NM_SELRECORD_T struct thread 102 #define MBUF_LEN(m) ((m)->m_pkthdr.len) 103 #define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) 104 #define MBUF_TRANSMIT(na, ifp, m) ((na)->if_transmit(ifp, m)) 105 #define GEN_TX_MBUF_IFP(m) ((m)->m_pkthdr.rcvif) 106 107 #define NM_ATOMIC_T volatile int /* required by atomic/bitops.h */ 108 /* atomic operations */ 109 #include <machine/atomic.h> 110 #define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) 111 #define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) 112 113 #if __FreeBSD_version >= 1100030 114 #define WNA(_ifp) (_ifp)->if_netmap 115 #else /* older FreeBSD */ 116 #define WNA(_ifp) (_ifp)->if_pspare[0] 117 #endif /* older FreeBSD */ 118 119 #if __FreeBSD_version >= 1100005 120 struct netmap_adapter *netmap_getna(if_t ifp); 121 #endif 122 123 #if __FreeBSD_version >= 1100027 124 #define MBUF_REFCNT(m) ((m)->m_ext.ext_count) 125 #define SET_MBUF_REFCNT(m, x) (m)->m_ext.ext_count = x 126 #else 127 #define MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *((m)->m_ext.ref_cnt) : -1) 128 #define SET_MBUF_REFCNT(m, x) *((m)->m_ext.ref_cnt) = x 129 #endif 130 131 #define MBUF_QUEUED(m) 1 132 133 struct nm_selinfo { 134 struct selinfo si; 135 struct mtx m; 136 }; 137 138 139 struct hrtimer { 140 /* Not used in FreeBSD. */ 141 }; 142 143 #define NM_BNS_GET(b) 144 #define NM_BNS_PUT(b) 145 146 #elif defined (linux) 147 148 #define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 149 #define NM_SELINFO_T wait_queue_head_t 150 #define MBUF_LEN(m) ((m)->len) 151 #define MBUF_TRANSMIT(na, ifp, m) \ 152 ({ \ 153 /* Avoid infinite recursion with generic. */ \ 154 m->priority = NM_MAGIC_PRIORITY_TX; \ 155 (((struct net_device_ops *)(na)->if_transmit)->ndo_start_xmit(m, ifp)); \ 156 0; \ 157 }) 158 159 /* See explanation in nm_os_generic_xmit_frame. */ 160 #define GEN_TX_MBUF_IFP(m) ((struct ifnet *)skb_shinfo(m)->destructor_arg) 161 162 #define NM_ATOMIC_T volatile long unsigned int 163 164 #define NM_MTX_T struct mutex /* OS-specific sleepable lock */ 165 #define NM_MTX_INIT(m) mutex_init(&(m)) 166 #define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 167 #define NM_MTX_LOCK(m) mutex_lock(&(m)) 168 #define NM_MTX_UNLOCK(m) mutex_unlock(&(m)) 169 #define NM_MTX_ASSERT(m) mutex_is_locked(&(m)) 170 171 #ifndef DEV_NETMAP 172 #define DEV_NETMAP 173 #endif /* DEV_NETMAP */ 174 175 #elif defined (__APPLE__) 176 177 #warning apple support is incomplete. 178 #define likely(x) __builtin_expect(!!(x), 1) 179 #define unlikely(x) __builtin_expect(!!(x), 0) 180 #define NM_LOCK_T IOLock * 181 #define NM_SELINFO_T struct selinfo 182 #define MBUF_LEN(m) ((m)->m_pkthdr.len) 183 184 #elif defined (_WIN32) 185 #include "../../../WINDOWS/win_glue.h" 186 187 #define NM_SELRECORD_T IO_STACK_LOCATION 188 #define NM_SELINFO_T win_SELINFO // see win_glue.h 189 #define NM_LOCK_T win_spinlock_t // see win_glue.h 190 #define NM_MTX_T KGUARDED_MUTEX /* OS-specific mutex (sleepable) */ 191 192 #define NM_MTX_INIT(m) KeInitializeGuardedMutex(&m); 193 #define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 194 #define NM_MTX_LOCK(m) KeAcquireGuardedMutex(&(m)) 195 #define NM_MTX_UNLOCK(m) KeReleaseGuardedMutex(&(m)) 196 #define NM_MTX_ASSERT(m) assert(&m.Count>0) 197 198 //These linknames are for the NDIS driver 199 #define NETMAP_NDIS_LINKNAME_STRING L"\\DosDevices\\NMAPNDIS" 200 #define NETMAP_NDIS_NTDEVICE_STRING L"\\Device\\NMAPNDIS" 201 202 //Definition of internal driver-to-driver ioctl codes 203 #define NETMAP_KERNEL_XCHANGE_POINTERS _IO('i', 180) 204 #define NETMAP_KERNEL_SEND_SHUTDOWN_SIGNAL _IO_direct('i', 195) 205 206 typedef struct hrtimer{ 207 KTIMER timer; 208 BOOLEAN active; 209 KDPC deferred_proc; 210 }; 211 212 /* MSVC does not have likely/unlikely support */ 213 #ifdef _MSC_VER 214 #define likely(x) (x) 215 #define unlikely(x) (x) 216 #else 217 #define likely(x) __builtin_expect((long)!!(x), 1L) 218 #define unlikely(x) __builtin_expect((long)!!(x), 0L) 219 #endif //_MSC_VER 220 221 #else 222 223 #error unsupported platform 224 225 #endif /* end - platform-specific code */ 226 227 #ifndef _WIN32 /* support for emulated sysctl */ 228 #define SYSBEGIN(x) 229 #define SYSEND 230 #endif /* _WIN32 */ 231 232 #define NM_ACCESS_ONCE(x) (*(volatile __typeof__(x) *)&(x)) 233 234 #define NMG_LOCK_T NM_MTX_T 235 #define NMG_LOCK_INIT() NM_MTX_INIT(netmap_global_lock) 236 #define NMG_LOCK_DESTROY() NM_MTX_DESTROY(netmap_global_lock) 237 #define NMG_LOCK() NM_MTX_LOCK(netmap_global_lock) 238 #define NMG_UNLOCK() NM_MTX_UNLOCK(netmap_global_lock) 239 #define NMG_LOCK_ASSERT() NM_MTX_ASSERT(netmap_global_lock) 240 241 #if defined(__FreeBSD__) 242 #define nm_prerr printf 243 #define nm_prinf printf 244 #elif defined (_WIN32) 245 #define nm_prerr DbgPrint 246 #define nm_prinf DbgPrint 247 #elif defined(linux) 248 #define nm_prerr(fmt, arg...) printk(KERN_ERR fmt, ##arg) 249 #define nm_prinf(fmt, arg...) printk(KERN_INFO fmt, ##arg) 250 #endif 251 252 #define ND(format, ...) 253 #define D(format, ...) \ 254 do { \ 255 struct timeval __xxts; \ 256 microtime(&__xxts); \ 257 nm_prerr("%03d.%06d [%4d] %-25s " format "\n", \ 258 (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 259 __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 260 } while (0) 261 262 /* rate limited, lps indicates how many per second */ 263 #define RD(lps, format, ...) \ 264 do { \ 265 static int t0, __cnt; \ 266 if (t0 != time_second) { \ 267 t0 = time_second; \ 268 __cnt = 0; \ 269 } \ 270 if (__cnt++ < lps) \ 271 D(format, ##__VA_ARGS__); \ 272 } while (0) 273 274 struct netmap_adapter; 275 struct nm_bdg_fwd; 276 struct nm_bridge; 277 struct netmap_priv_d; 278 279 /* os-specific NM_SELINFO_T initialzation/destruction functions */ 280 void nm_os_selinfo_init(NM_SELINFO_T *); 281 void nm_os_selinfo_uninit(NM_SELINFO_T *); 282 283 const char *nm_dump_buf(char *p, int len, int lim, char *dst); 284 285 void nm_os_selwakeup(NM_SELINFO_T *si); 286 void nm_os_selrecord(NM_SELRECORD_T *sr, NM_SELINFO_T *si); 287 288 int nm_os_ifnet_init(void); 289 void nm_os_ifnet_fini(void); 290 void nm_os_ifnet_lock(void); 291 void nm_os_ifnet_unlock(void); 292 293 unsigned nm_os_ifnet_mtu(struct ifnet *ifp); 294 295 void nm_os_get_module(void); 296 void nm_os_put_module(void); 297 298 void netmap_make_zombie(struct ifnet *); 299 void netmap_undo_zombie(struct ifnet *); 300 301 /* os independent alloc/realloc/free */ 302 void *nm_os_malloc(size_t); 303 void *nm_os_vmalloc(size_t); 304 void *nm_os_realloc(void *, size_t new_size, size_t old_size); 305 void nm_os_free(void *); 306 void nm_os_vfree(void *); 307 308 /* passes a packet up to the host stack. 309 * If the packet is sent (or dropped) immediately it returns NULL, 310 * otherwise it links the packet to prev and returns m. 311 * In this case, a final call with m=NULL and prev != NULL will send up 312 * the entire chain to the host stack. 313 */ 314 void *nm_os_send_up(struct ifnet *, struct mbuf *m, struct mbuf *prev); 315 316 int nm_os_mbuf_has_offld(struct mbuf *m); 317 318 #include "netmap_mbq.h" 319 320 extern NMG_LOCK_T netmap_global_lock; 321 322 enum txrx { NR_RX = 0, NR_TX = 1, NR_TXRX }; 323 324 static __inline const char* 325 nm_txrx2str(enum txrx t) 326 { 327 return (t== NR_RX ? "RX" : "TX"); 328 } 329 330 static __inline enum txrx 331 nm_txrx_swap(enum txrx t) 332 { 333 return (t== NR_RX ? NR_TX : NR_RX); 334 } 335 336 #define for_rx_tx(t) for ((t) = 0; (t) < NR_TXRX; (t)++) 337 338 #ifdef WITH_MONITOR 339 struct netmap_zmon_list { 340 struct netmap_kring *next; 341 struct netmap_kring *prev; 342 }; 343 #endif /* WITH_MONITOR */ 344 345 /* 346 * private, kernel view of a ring. Keeps track of the status of 347 * a ring across system calls. 348 * 349 * nr_hwcur index of the next buffer to refill. 350 * It corresponds to ring->head 351 * at the time the system call returns. 352 * 353 * nr_hwtail index of the first buffer owned by the kernel. 354 * On RX, hwcur->hwtail are receive buffers 355 * not yet released. hwcur is advanced following 356 * ring->head, hwtail is advanced on incoming packets, 357 * and a wakeup is generated when hwtail passes ring->cur 358 * On TX, hwcur->rcur have been filled by the sender 359 * but not sent yet to the NIC; rcur->hwtail are available 360 * for new transmissions, and hwtail->hwcur-1 are pending 361 * transmissions not yet acknowledged. 362 * 363 * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 364 * This is so that, on a reset, buffers owned by userspace are not 365 * modified by the kernel. In particular: 366 * RX rings: the next empty buffer (hwtail + hwofs) coincides with 367 * the next empty buffer as known by the hardware (next_to_check or so). 368 * TX rings: hwcur + hwofs coincides with next_to_send 369 * 370 * For received packets, slot->flags is set to nkr_slot_flags 371 * so we can provide a proper initial value. 372 * 373 * The following fields are used to implement lock-free copy of packets 374 * from input to output ports in VALE switch: 375 * nkr_hwlease buffer after the last one being copied. 376 * A writer in nm_bdg_flush reserves N buffers 377 * from nr_hwlease, advances it, then does the 378 * copy outside the lock. 379 * In RX rings (used for VALE ports), 380 * nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1 381 * In TX rings (used for NIC or host stack ports) 382 * nkr_hwcur <= nkr_hwlease < nkr_hwtail 383 * nkr_leases array of nkr_num_slots where writers can report 384 * completion of their block. NR_NOSLOT (~0) indicates 385 * that the writer has not finished yet 386 * nkr_lease_idx index of next free slot in nr_leases, to be assigned 387 * 388 * The kring is manipulated by txsync/rxsync and generic netmap function. 389 * 390 * Concurrent rxsync or txsync on the same ring are prevented through 391 * by nm_kr_(try)lock() which in turn uses nr_busy. This is all we need 392 * for NIC rings, and for TX rings attached to the host stack. 393 * 394 * RX rings attached to the host stack use an mbq (rx_queue) on both 395 * rxsync_from_host() and netmap_transmit(). The mbq is protected 396 * by its internal lock. 397 * 398 * RX rings attached to the VALE switch are accessed by both senders 399 * and receiver. They are protected through the q_lock on the RX ring. 400 */ 401 struct netmap_kring { 402 struct netmap_ring *ring; 403 404 uint32_t nr_hwcur; 405 uint32_t nr_hwtail; 406 407 /* 408 * Copies of values in user rings, so we do not need to look 409 * at the ring (which could be modified). These are set in the 410 * *sync_prologue()/finalize() routines. 411 */ 412 uint32_t rhead; 413 uint32_t rcur; 414 uint32_t rtail; 415 416 uint32_t nr_kflags; /* private driver flags */ 417 #define NKR_PENDINTR 0x1 // Pending interrupt. 418 #define NKR_EXCLUSIVE 0x2 /* exclusive binding */ 419 #define NKR_FORWARD 0x4 /* (host ring only) there are 420 packets to forward 421 */ 422 #define NKR_NEEDRING 0x8 /* ring needed even if users==0 423 * (used internally by pipes and 424 * by ptnetmap host ports) 425 */ 426 #define NKR_NOINTR 0x10 /* don't use interrupts on this ring */ 427 428 uint32_t nr_mode; 429 uint32_t nr_pending_mode; 430 #define NKR_NETMAP_OFF 0x0 431 #define NKR_NETMAP_ON 0x1 432 433 uint32_t nkr_num_slots; 434 435 /* 436 * On a NIC reset, the NIC ring indexes may be reset but the 437 * indexes in the netmap rings remain the same. nkr_hwofs 438 * keeps track of the offset between the two. 439 */ 440 int32_t nkr_hwofs; 441 442 /* last_reclaim is opaque marker to help reduce the frequency 443 * of operations such as reclaiming tx buffers. A possible use 444 * is set it to ticks and do the reclaim only once per tick. 445 */ 446 uint64_t last_reclaim; 447 448 449 NM_SELINFO_T si; /* poll/select wait queue */ 450 NM_LOCK_T q_lock; /* protects kring and ring. */ 451 NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ 452 453 struct netmap_adapter *na; 454 455 /* The following fields are for VALE switch support */ 456 struct nm_bdg_fwd *nkr_ft; 457 uint32_t *nkr_leases; 458 #define NR_NOSLOT ((uint32_t)~0) /* used in nkr_*lease* */ 459 uint32_t nkr_hwlease; 460 uint32_t nkr_lease_idx; 461 462 /* while nkr_stopped is set, no new [tr]xsync operations can 463 * be started on this kring. 464 * This is used by netmap_disable_all_rings() 465 * to find a synchronization point where critical data 466 * structures pointed to by the kring can be added or removed 467 */ 468 volatile int nkr_stopped; 469 470 /* Support for adapters without native netmap support. 471 * On tx rings we preallocate an array of tx buffers 472 * (same size as the netmap ring), on rx rings we 473 * store incoming mbufs in a queue that is drained by 474 * a rxsync. 475 */ 476 struct mbuf **tx_pool; 477 struct mbuf *tx_event; /* TX event used as a notification */ 478 NM_LOCK_T tx_event_lock; /* protects the tx_event mbuf */ 479 struct mbq rx_queue; /* intercepted rx mbufs. */ 480 481 uint32_t users; /* existing bindings for this ring */ 482 483 uint32_t ring_id; /* kring identifier */ 484 enum txrx tx; /* kind of ring (tx or rx) */ 485 char name[64]; /* diagnostic */ 486 487 /* [tx]sync callback for this kring. 488 * The default nm_kring_create callback (netmap_krings_create) 489 * sets the nm_sync callback of each hardware tx(rx) kring to 490 * the corresponding nm_txsync(nm_rxsync) taken from the 491 * netmap_adapter; moreover, it sets the sync callback 492 * of the host tx(rx) ring to netmap_txsync_to_host 493 * (netmap_rxsync_from_host). 494 * 495 * Overrides: the above configuration is not changed by 496 * any of the nm_krings_create callbacks. 497 */ 498 int (*nm_sync)(struct netmap_kring *kring, int flags); 499 int (*nm_notify)(struct netmap_kring *kring, int flags); 500 501 #ifdef WITH_PIPES 502 struct netmap_kring *pipe; /* if this is a pipe ring, 503 * pointer to the other end 504 */ 505 #endif /* WITH_PIPES */ 506 507 #ifdef WITH_VALE 508 int (*save_notify)(struct netmap_kring *kring, int flags); 509 #endif 510 511 #ifdef WITH_MONITOR 512 /* array of krings that are monitoring this kring */ 513 struct netmap_kring **monitors; 514 uint32_t max_monitors; /* current size of the monitors array */ 515 uint32_t n_monitors; /* next unused entry in the monitor array */ 516 uint32_t mon_pos[NR_TXRX]; /* index of this ring in the monitored ring array */ 517 uint32_t mon_tail; /* last seen slot on rx */ 518 519 /* circular list of zero-copy monitors */ 520 struct netmap_zmon_list zmon_list[NR_TXRX]; 521 522 /* 523 * Monitors work by intercepting the sync and notify callbacks of the 524 * monitored krings. This is implemented by replacing the pointers 525 * above and saving the previous ones in mon_* pointers below 526 */ 527 int (*mon_sync)(struct netmap_kring *kring, int flags); 528 int (*mon_notify)(struct netmap_kring *kring, int flags); 529 530 #endif 531 } 532 #ifdef _WIN32 533 __declspec(align(64)); 534 #else 535 __attribute__((__aligned__(64))); 536 #endif 537 538 /* return 1 iff the kring needs to be turned on */ 539 static inline int 540 nm_kring_pending_on(struct netmap_kring *kring) 541 { 542 return kring->nr_pending_mode == NKR_NETMAP_ON && 543 kring->nr_mode == NKR_NETMAP_OFF; 544 } 545 546 /* return 1 iff the kring needs to be turned off */ 547 static inline int 548 nm_kring_pending_off(struct netmap_kring *kring) 549 { 550 return kring->nr_pending_mode == NKR_NETMAP_OFF && 551 kring->nr_mode == NKR_NETMAP_ON; 552 } 553 554 /* return the next index, with wraparound */ 555 static inline uint32_t 556 nm_next(uint32_t i, uint32_t lim) 557 { 558 return unlikely (i == lim) ? 0 : i + 1; 559 } 560 561 562 /* return the previous index, with wraparound */ 563 static inline uint32_t 564 nm_prev(uint32_t i, uint32_t lim) 565 { 566 return unlikely (i == 0) ? lim : i - 1; 567 } 568 569 570 /* 571 * 572 * Here is the layout for the Rx and Tx rings. 573 574 RxRING TxRING 575 576 +-----------------+ +-----------------+ 577 | | | | 578 | free | | free | 579 +-----------------+ +-----------------+ 580 head->| owned by user |<-hwcur | not sent to nic |<-hwcur 581 | | | yet | 582 +-----------------+ | | 583 cur->| available to | | | 584 | user, not read | +-----------------+ 585 | yet | cur->| (being | 586 | | | prepared) | 587 | | | | 588 +-----------------+ + ------ + 589 tail->| |<-hwtail | |<-hwlease 590 | (being | ... | | ... 591 | prepared) | ... | | ... 592 +-----------------+ ... | | ... 593 | |<-hwlease +-----------------+ 594 | | tail->| |<-hwtail 595 | | | | 596 | | | | 597 | | | | 598 +-----------------+ +-----------------+ 599 600 * The cur/tail (user view) and hwcur/hwtail (kernel view) 601 * are used in the normal operation of the card. 602 * 603 * When a ring is the output of a switch port (Rx ring for 604 * a VALE port, Tx ring for the host stack or NIC), slots 605 * are reserved in blocks through 'hwlease' which points 606 * to the next unused slot. 607 * On an Rx ring, hwlease is always after hwtail, 608 * and completions cause hwtail to advance. 609 * On a Tx ring, hwlease is always between cur and hwtail, 610 * and completions cause cur to advance. 611 * 612 * nm_kr_space() returns the maximum number of slots that 613 * can be assigned. 614 * nm_kr_lease() reserves the required number of buffers, 615 * advances nkr_hwlease and also returns an entry in 616 * a circular array where completions should be reported. 617 */ 618 619 struct lut_entry; 620 #ifdef __FreeBSD__ 621 #define plut_entry lut_entry 622 #endif 623 624 struct netmap_lut { 625 struct lut_entry *lut; 626 struct plut_entry *plut; 627 uint32_t objtotal; /* max buffer index */ 628 uint32_t objsize; /* buffer size */ 629 }; 630 631 struct netmap_vp_adapter; // forward 632 633 /* 634 * The "struct netmap_adapter" extends the "struct adapter" 635 * (or equivalent) device descriptor. 636 * It contains all base fields needed to support netmap operation. 637 * There are in fact different types of netmap adapters 638 * (native, generic, VALE switch...) so a netmap_adapter is 639 * just the first field in the derived type. 640 */ 641 struct netmap_adapter { 642 /* 643 * On linux we do not have a good way to tell if an interface 644 * is netmap-capable. So we always use the following trick: 645 * NA(ifp) points here, and the first entry (which hopefully 646 * always exists and is at least 32 bits) contains a magic 647 * value which we can use to detect that the interface is good. 648 */ 649 uint32_t magic; 650 uint32_t na_flags; /* enabled, and other flags */ 651 #define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 652 * useful during initialization 653 */ 654 #define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 655 #define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when 656 * forwarding packets coming from this 657 * interface 658 */ 659 #define NAF_MEM_OWNER 8 /* the adapter uses its own memory area 660 * that cannot be changed 661 */ 662 #define NAF_NATIVE 16 /* the adapter is native. 663 * Virtual ports (non persistent vale ports, 664 * pipes, monitors...) should never use 665 * this flag. 666 */ 667 #define NAF_NETMAP_ON 32 /* netmap is active (either native or 668 * emulated). Where possible (e.g. FreeBSD) 669 * IFCAP_NETMAP also mirrors this flag. 670 */ 671 #define NAF_HOST_RINGS 64 /* the adapter supports the host rings */ 672 #define NAF_FORCE_NATIVE 128 /* the adapter is always NATIVE */ 673 #define NAF_PTNETMAP_HOST 256 /* the adapter supports ptnetmap in the host */ 674 #define NAF_MOREFRAG 512 /* the adapter supports NS_MOREFRAG */ 675 #define NAF_ZOMBIE (1U<<30) /* the nic driver has been unloaded */ 676 #define NAF_BUSY (1U<<31) /* the adapter is used internally and 677 * cannot be registered from userspace 678 */ 679 int active_fds; /* number of user-space descriptors using this 680 interface, which is equal to the number of 681 struct netmap_if objs in the mapped region. */ 682 683 u_int num_rx_rings; /* number of adapter receive rings */ 684 u_int num_tx_rings; /* number of adapter transmit rings */ 685 686 u_int num_tx_desc; /* number of descriptor in each queue */ 687 u_int num_rx_desc; 688 689 /* tx_rings and rx_rings are private but allocated 690 * as a contiguous chunk of memory. Each array has 691 * N+1 entries, for the adapter queues and for the host queue. 692 */ 693 struct netmap_kring *tx_rings; /* array of TX rings. */ 694 struct netmap_kring *rx_rings; /* array of RX rings. */ 695 696 void *tailroom; /* space below the rings array */ 697 /* (used for leases) */ 698 699 700 NM_SELINFO_T si[NR_TXRX]; /* global wait queues */ 701 702 /* count users of the global wait queues */ 703 int si_users[NR_TXRX]; 704 705 void *pdev; /* used to store pci device */ 706 707 /* copy of if_qflush and if_transmit pointers, to intercept 708 * packets from the network stack when netmap is active. 709 */ 710 int (*if_transmit)(struct ifnet *, struct mbuf *); 711 712 /* copy of if_input for netmap_send_up() */ 713 void (*if_input)(struct ifnet *, struct mbuf *); 714 715 /* Back reference to the parent ifnet struct. Used for 716 * hardware ports (emulated netmap included). */ 717 struct ifnet *ifp; /* adapter is ifp->if_softc */ 718 719 /*---- callbacks for this netmap adapter -----*/ 720 /* 721 * nm_dtor() is the cleanup routine called when destroying 722 * the adapter. 723 * Called with NMG_LOCK held. 724 * 725 * nm_register() is called on NIOCREGIF and close() to enter 726 * or exit netmap mode on the NIC 727 * Called with NNG_LOCK held. 728 * 729 * nm_txsync() pushes packets to the underlying hw/switch 730 * 731 * nm_rxsync() collects packets from the underlying hw/switch 732 * 733 * nm_config() returns configuration information from the OS 734 * Called with NMG_LOCK held. 735 * 736 * nm_krings_create() create and init the tx_rings and 737 * rx_rings arrays of kring structures. In particular, 738 * set the nm_sync callbacks for each ring. 739 * There is no need to also allocate the corresponding 740 * netmap_rings, since netmap_mem_rings_create() will always 741 * be called to provide the missing ones. 742 * Called with NNG_LOCK held. 743 * 744 * nm_krings_delete() cleanup and delete the tx_rings and rx_rings 745 * arrays 746 * Called with NMG_LOCK held. 747 * 748 * nm_notify() is used to act after data have become available 749 * (or the stopped state of the ring has changed) 750 * For hw devices this is typically a selwakeup(), 751 * but for NIC/host ports attached to a switch (or vice-versa) 752 * we also need to invoke the 'txsync' code downstream. 753 * This callback pointer is actually used only to initialize 754 * kring->nm_notify. 755 * Return values are the same as for netmap_rx_irq(). 756 */ 757 void (*nm_dtor)(struct netmap_adapter *); 758 759 int (*nm_register)(struct netmap_adapter *, int onoff); 760 void (*nm_intr)(struct netmap_adapter *, int onoff); 761 762 int (*nm_txsync)(struct netmap_kring *kring, int flags); 763 int (*nm_rxsync)(struct netmap_kring *kring, int flags); 764 int (*nm_notify)(struct netmap_kring *kring, int flags); 765 #define NAF_FORCE_READ 1 766 #define NAF_FORCE_RECLAIM 2 767 #define NAF_CAN_FORWARD_DOWN 4 768 /* return configuration information */ 769 int (*nm_config)(struct netmap_adapter *, 770 u_int *txr, u_int *txd, u_int *rxr, u_int *rxd); 771 int (*nm_krings_create)(struct netmap_adapter *); 772 void (*nm_krings_delete)(struct netmap_adapter *); 773 #ifdef WITH_VALE 774 /* 775 * nm_bdg_attach() initializes the na_vp field to point 776 * to an adapter that can be attached to a VALE switch. If the 777 * current adapter is already a VALE port, na_vp is simply a cast; 778 * otherwise, na_vp points to a netmap_bwrap_adapter. 779 * If applicable, this callback also initializes na_hostvp, 780 * that can be used to connect the adapter host rings to the 781 * switch. 782 * Called with NMG_LOCK held. 783 * 784 * nm_bdg_ctl() is called on the actual attach/detach to/from 785 * to/from the switch, to perform adapter-specific 786 * initializations 787 * Called with NMG_LOCK held. 788 */ 789 int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *); 790 int (*nm_bdg_ctl)(struct netmap_adapter *, struct nmreq *, int); 791 792 /* adapter used to attach this adapter to a VALE switch (if any) */ 793 struct netmap_vp_adapter *na_vp; 794 /* adapter used to attach the host rings of this adapter 795 * to a VALE switch (if any) */ 796 struct netmap_vp_adapter *na_hostvp; 797 #endif 798 799 /* standard refcount to control the lifetime of the adapter 800 * (it should be equal to the lifetime of the corresponding ifp) 801 */ 802 int na_refcount; 803 804 /* memory allocator (opaque) 805 * We also cache a pointer to the lut_entry for translating 806 * buffer addresses, the total number of buffers and the buffer size. 807 */ 808 struct netmap_mem_d *nm_mem; 809 struct netmap_mem_d *nm_mem_prev; 810 struct netmap_lut na_lut; 811 812 /* additional information attached to this adapter 813 * by other netmap subsystems. Currently used by 814 * bwrap, LINUX/v1000 and ptnetmap 815 */ 816 void *na_private; 817 818 /* array of pipes that have this adapter as a parent */ 819 struct netmap_pipe_adapter **na_pipes; 820 int na_next_pipe; /* next free slot in the array */ 821 int na_max_pipes; /* size of the array */ 822 823 /* Offset of ethernet header for each packet. */ 824 u_int virt_hdr_len; 825 826 char name[64]; 827 }; 828 829 static __inline u_int 830 nma_get_ndesc(struct netmap_adapter *na, enum txrx t) 831 { 832 return (t == NR_TX ? na->num_tx_desc : na->num_rx_desc); 833 } 834 835 static __inline void 836 nma_set_ndesc(struct netmap_adapter *na, enum txrx t, u_int v) 837 { 838 if (t == NR_TX) 839 na->num_tx_desc = v; 840 else 841 na->num_rx_desc = v; 842 } 843 844 static __inline u_int 845 nma_get_nrings(struct netmap_adapter *na, enum txrx t) 846 { 847 return (t == NR_TX ? na->num_tx_rings : na->num_rx_rings); 848 } 849 850 static __inline void 851 nma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v) 852 { 853 if (t == NR_TX) 854 na->num_tx_rings = v; 855 else 856 na->num_rx_rings = v; 857 } 858 859 static __inline struct netmap_kring* 860 NMR(struct netmap_adapter *na, enum txrx t) 861 { 862 return (t == NR_TX ? na->tx_rings : na->rx_rings); 863 } 864 865 int nma_intr_enable(struct netmap_adapter *na, int onoff); 866 867 /* 868 * If the NIC is owned by the kernel 869 * (i.e., bridge), neither another bridge nor user can use it; 870 * if the NIC is owned by a user, only users can share it. 871 * Evaluation must be done under NMG_LOCK(). 872 */ 873 #define NETMAP_OWNED_BY_KERN(na) ((na)->na_flags & NAF_BUSY) 874 #define NETMAP_OWNED_BY_ANY(na) \ 875 (NETMAP_OWNED_BY_KERN(na) || ((na)->active_fds > 0)) 876 877 /* 878 * derived netmap adapters for various types of ports 879 */ 880 struct netmap_vp_adapter { /* VALE software port */ 881 struct netmap_adapter up; 882 883 /* 884 * Bridge support: 885 * 886 * bdg_port is the port number used in the bridge; 887 * na_bdg points to the bridge this NA is attached to. 888 */ 889 int bdg_port; 890 struct nm_bridge *na_bdg; 891 int retry; 892 int autodelete; /* remove the ifp on last reference */ 893 894 /* Maximum Frame Size, used in bdg_mismatch_datapath() */ 895 u_int mfs; 896 /* Last source MAC on this port */ 897 uint64_t last_smac; 898 }; 899 900 901 struct netmap_hw_adapter { /* physical device */ 902 struct netmap_adapter up; 903 904 #ifdef linux 905 struct net_device_ops nm_ndo; 906 struct ethtool_ops nm_eto; 907 #endif 908 const struct ethtool_ops* save_ethtool; 909 910 int (*nm_hw_register)(struct netmap_adapter *, int onoff); 911 }; 912 913 #ifdef WITH_GENERIC 914 /* Mitigation support. */ 915 struct nm_generic_mit { 916 struct hrtimer mit_timer; 917 int mit_pending; 918 int mit_ring_idx; /* index of the ring being mitigated */ 919 struct netmap_adapter *mit_na; /* backpointer */ 920 }; 921 922 struct netmap_generic_adapter { /* emulated device */ 923 struct netmap_hw_adapter up; 924 925 /* Pointer to a previously used netmap adapter. */ 926 struct netmap_adapter *prev; 927 928 /* Emulated netmap adapters support: 929 * - save_if_input saves the if_input hook (FreeBSD); 930 * - mit implements rx interrupt mitigation; 931 */ 932 void (*save_if_input)(struct ifnet *, struct mbuf *); 933 934 struct nm_generic_mit *mit; 935 #ifdef linux 936 netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *); 937 #endif 938 /* Is the adapter able to use multiple RX slots to scatter 939 * each packet pushed up by the driver? */ 940 int rxsg; 941 942 /* Is the transmission path controlled by a netmap-aware 943 * device queue (i.e. qdisc on linux)? */ 944 int txqdisc; 945 }; 946 #endif /* WITH_GENERIC */ 947 948 static __inline int 949 netmap_real_rings(struct netmap_adapter *na, enum txrx t) 950 { 951 return nma_get_nrings(na, t) + !!(na->na_flags & NAF_HOST_RINGS); 952 } 953 954 #ifdef WITH_VALE 955 struct nm_bdg_polling_state; 956 /* 957 * Bridge wrapper for non VALE ports attached to a VALE switch. 958 * 959 * The real device must already have its own netmap adapter (hwna). 960 * The bridge wrapper and the hwna adapter share the same set of 961 * netmap rings and buffers, but they have two separate sets of 962 * krings descriptors, with tx/rx meanings swapped: 963 * 964 * netmap 965 * bwrap krings rings krings hwna 966 * +------+ +------+ +-----+ +------+ +------+ 967 * |tx_rings->| |\ /| |----| |<-tx_rings| 968 * | | +------+ \ / +-----+ +------+ | | 969 * | | X | | 970 * | | / \ | | 971 * | | +------+/ \+-----+ +------+ | | 972 * |rx_rings->| | | |----| |<-rx_rings| 973 * | | +------+ +-----+ +------+ | | 974 * +------+ +------+ 975 * 976 * - packets coming from the bridge go to the brwap rx rings, 977 * which are also the hwna tx rings. The bwrap notify callback 978 * will then complete the hwna tx (see netmap_bwrap_notify). 979 * 980 * - packets coming from the outside go to the hwna rx rings, 981 * which are also the bwrap tx rings. The (overwritten) hwna 982 * notify method will then complete the bridge tx 983 * (see netmap_bwrap_intr_notify). 984 * 985 * The bridge wrapper may optionally connect the hwna 'host' rings 986 * to the bridge. This is done by using a second port in the 987 * bridge and connecting it to the 'host' netmap_vp_adapter 988 * contained in the netmap_bwrap_adapter. The brwap host adapter 989 * cross-links the hwna host rings in the same way as shown above. 990 * 991 * - packets coming from the bridge and directed to the host stack 992 * are handled by the bwrap host notify callback 993 * (see netmap_bwrap_host_notify) 994 * 995 * - packets coming from the host stack are still handled by the 996 * overwritten hwna notify callback (netmap_bwrap_intr_notify), 997 * but are diverted to the host adapter depending on the ring number. 998 * 999 */ 1000 struct netmap_bwrap_adapter { 1001 struct netmap_vp_adapter up; 1002 struct netmap_vp_adapter host; /* for host rings */ 1003 struct netmap_adapter *hwna; /* the underlying device */ 1004 1005 /* 1006 * When we attach a physical interface to the bridge, we 1007 * allow the controlling process to terminate, so we need 1008 * a place to store the n_detmap_priv_d data structure. 1009 * This is only done when physical interfaces 1010 * are attached to a bridge. 1011 */ 1012 struct netmap_priv_d *na_kpriv; 1013 struct nm_bdg_polling_state *na_polling_state; 1014 }; 1015 int netmap_bwrap_attach(const char *name, struct netmap_adapter *); 1016 int netmap_vi_create(struct nmreq *, int); 1017 1018 #else /* !WITH_VALE */ 1019 #define netmap_vi_create(nmr, a) (EOPNOTSUPP) 1020 #endif /* WITH_VALE */ 1021 1022 #ifdef WITH_PIPES 1023 1024 #define NM_MAXPIPES 64 /* max number of pipes per adapter */ 1025 1026 struct netmap_pipe_adapter { 1027 struct netmap_adapter up; 1028 1029 u_int id; /* pipe identifier */ 1030 int role; /* either NR_REG_PIPE_MASTER or NR_REG_PIPE_SLAVE */ 1031 1032 struct netmap_adapter *parent; /* adapter that owns the memory */ 1033 struct netmap_pipe_adapter *peer; /* the other end of the pipe */ 1034 int peer_ref; /* 1 iff we are holding a ref to the peer */ 1035 struct ifnet *parent_ifp; /* maybe null */ 1036 1037 u_int parent_slot; /* index in the parent pipe array */ 1038 }; 1039 1040 #endif /* WITH_PIPES */ 1041 1042 1043 /* return slots reserved to rx clients; used in drivers */ 1044 static inline uint32_t 1045 nm_kr_rxspace(struct netmap_kring *k) 1046 { 1047 int space = k->nr_hwtail - k->nr_hwcur; 1048 if (space < 0) 1049 space += k->nkr_num_slots; 1050 ND("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail); 1051 1052 return space; 1053 } 1054 1055 /* return slots reserved to tx clients */ 1056 #define nm_kr_txspace(_k) nm_kr_rxspace(_k) 1057 1058 1059 /* True if no space in the tx ring, only valid after txsync_prologue */ 1060 static inline int 1061 nm_kr_txempty(struct netmap_kring *kring) 1062 { 1063 return kring->rcur == kring->nr_hwtail; 1064 } 1065 1066 /* True if no more completed slots in the rx ring, only valid after 1067 * rxsync_prologue */ 1068 #define nm_kr_rxempty(_k) nm_kr_txempty(_k) 1069 1070 /* 1071 * protect against multiple threads using the same ring. 1072 * also check that the ring has not been stopped or locked 1073 */ 1074 #define NM_KR_BUSY 1 /* some other thread is syncing the ring */ 1075 #define NM_KR_STOPPED 2 /* unbounded stop (ifconfig down or driver unload) */ 1076 #define NM_KR_LOCKED 3 /* bounded, brief stop for mutual exclusion */ 1077 1078 1079 /* release the previously acquired right to use the *sync() methods of the ring */ 1080 static __inline void nm_kr_put(struct netmap_kring *kr) 1081 { 1082 NM_ATOMIC_CLEAR(&kr->nr_busy); 1083 } 1084 1085 1086 /* true if the ifp that backed the adapter has disappeared (e.g., the 1087 * driver has been unloaded) 1088 */ 1089 static inline int nm_iszombie(struct netmap_adapter *na); 1090 1091 /* try to obtain exclusive right to issue the *sync() operations on the ring. 1092 * The right is obtained and must be later relinquished via nm_kr_put() if and 1093 * only if nm_kr_tryget() returns 0. 1094 * If can_sleep is 1 there are only two other possible outcomes: 1095 * - the function returns NM_KR_BUSY 1096 * - the function returns NM_KR_STOPPED and sets the POLLERR bit in *perr 1097 * (if non-null) 1098 * In both cases the caller will typically skip the ring, possibly collecting 1099 * errors along the way. 1100 * If the calling context does not allow sleeping, the caller must pass 0 in can_sleep. 1101 * In the latter case, the function may also return NM_KR_LOCKED and leave *perr 1102 * untouched: ideally, the caller should try again at a later time. 1103 */ 1104 static __inline int nm_kr_tryget(struct netmap_kring *kr, int can_sleep, int *perr) 1105 { 1106 int busy = 1, stopped; 1107 /* check a first time without taking the lock 1108 * to avoid starvation for nm_kr_get() 1109 */ 1110 retry: 1111 stopped = kr->nkr_stopped; 1112 if (unlikely(stopped)) { 1113 goto stop; 1114 } 1115 busy = NM_ATOMIC_TEST_AND_SET(&kr->nr_busy); 1116 /* we should not return NM_KR_BUSY if the ring was 1117 * actually stopped, so check another time after 1118 * the barrier provided by the atomic operation 1119 */ 1120 stopped = kr->nkr_stopped; 1121 if (unlikely(stopped)) { 1122 goto stop; 1123 } 1124 1125 if (unlikely(nm_iszombie(kr->na))) { 1126 stopped = NM_KR_STOPPED; 1127 goto stop; 1128 } 1129 1130 return unlikely(busy) ? NM_KR_BUSY : 0; 1131 1132 stop: 1133 if (!busy) 1134 nm_kr_put(kr); 1135 if (stopped == NM_KR_STOPPED) { 1136 /* if POLLERR is defined we want to use it to simplify netmap_poll(). 1137 * Otherwise, any non-zero value will do. 1138 */ 1139 #ifdef POLLERR 1140 #define NM_POLLERR POLLERR 1141 #else 1142 #define NM_POLLERR 1 1143 #endif /* POLLERR */ 1144 if (perr) 1145 *perr |= NM_POLLERR; 1146 #undef NM_POLLERR 1147 } else if (can_sleep) { 1148 tsleep(kr, 0, "NM_KR_TRYGET", 4); 1149 goto retry; 1150 } 1151 return stopped; 1152 } 1153 1154 /* put the ring in the 'stopped' state and wait for the current user (if any) to 1155 * notice. stopped must be either NM_KR_STOPPED or NM_KR_LOCKED 1156 */ 1157 static __inline void nm_kr_stop(struct netmap_kring *kr, int stopped) 1158 { 1159 kr->nkr_stopped = stopped; 1160 while (NM_ATOMIC_TEST_AND_SET(&kr->nr_busy)) 1161 tsleep(kr, 0, "NM_KR_GET", 4); 1162 } 1163 1164 /* restart a ring after a stop */ 1165 static __inline void nm_kr_start(struct netmap_kring *kr) 1166 { 1167 kr->nkr_stopped = 0; 1168 nm_kr_put(kr); 1169 } 1170 1171 1172 /* 1173 * The following functions are used by individual drivers to 1174 * support netmap operation. 1175 * 1176 * netmap_attach() initializes a struct netmap_adapter, allocating the 1177 * struct netmap_ring's and the struct selinfo. 1178 * 1179 * netmap_detach() frees the memory allocated by netmap_attach(). 1180 * 1181 * netmap_transmit() replaces the if_transmit routine of the interface, 1182 * and is used to intercept packets coming from the stack. 1183 * 1184 * netmap_load_map/netmap_reload_map are helper routines to set/reset 1185 * the dmamap for a packet buffer 1186 * 1187 * netmap_reset() is a helper routine to be called in the hw driver 1188 * when reinitializing a ring. It should not be called by 1189 * virtual ports (vale, pipes, monitor) 1190 */ 1191 int netmap_attach(struct netmap_adapter *); 1192 int netmap_attach_ext(struct netmap_adapter *, size_t size, int override_reg); 1193 void netmap_detach(struct ifnet *); 1194 int netmap_transmit(struct ifnet *, struct mbuf *); 1195 struct netmap_slot *netmap_reset(struct netmap_adapter *na, 1196 enum txrx tx, u_int n, u_int new_cur); 1197 int netmap_ring_reinit(struct netmap_kring *); 1198 1199 /* Return codes for netmap_*x_irq. */ 1200 enum { 1201 /* Driver should do normal interrupt processing, e.g. because 1202 * the interface is not in netmap mode. */ 1203 NM_IRQ_PASS = 0, 1204 /* Port is in netmap mode, and the interrupt work has been 1205 * completed. The driver does not have to notify netmap 1206 * again before the next interrupt. */ 1207 NM_IRQ_COMPLETED = -1, 1208 /* Port is in netmap mode, but the interrupt work has not been 1209 * completed. The driver has to make sure netmap will be 1210 * notified again soon, even if no more interrupts come (e.g. 1211 * on Linux the driver should not call napi_complete()). */ 1212 NM_IRQ_RESCHED = -2, 1213 }; 1214 1215 /* default functions to handle rx/tx interrupts */ 1216 int netmap_rx_irq(struct ifnet *, u_int, u_int *); 1217 #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 1218 int netmap_common_irq(struct netmap_adapter *, u_int, u_int *work_done); 1219 1220 1221 #ifdef WITH_VALE 1222 /* functions used by external modules to interface with VALE */ 1223 #define netmap_vp_to_ifp(_vp) ((_vp)->up.ifp) 1224 #define netmap_ifp_to_vp(_ifp) (NA(_ifp)->na_vp) 1225 #define netmap_ifp_to_host_vp(_ifp) (NA(_ifp)->na_hostvp) 1226 #define netmap_bdg_idx(_vp) ((_vp)->bdg_port) 1227 const char *netmap_bdg_name(struct netmap_vp_adapter *); 1228 #else /* !WITH_VALE */ 1229 #define netmap_vp_to_ifp(_vp) NULL 1230 #define netmap_ifp_to_vp(_ifp) NULL 1231 #define netmap_ifp_to_host_vp(_ifp) NULL 1232 #define netmap_bdg_idx(_vp) -1 1233 #define netmap_bdg_name(_vp) NULL 1234 #endif /* WITH_VALE */ 1235 1236 static inline int 1237 nm_netmap_on(struct netmap_adapter *na) 1238 { 1239 return na && na->na_flags & NAF_NETMAP_ON; 1240 } 1241 1242 static inline int 1243 nm_native_on(struct netmap_adapter *na) 1244 { 1245 return nm_netmap_on(na) && (na->na_flags & NAF_NATIVE); 1246 } 1247 1248 static inline int 1249 nm_iszombie(struct netmap_adapter *na) 1250 { 1251 return na == NULL || (na->na_flags & NAF_ZOMBIE); 1252 } 1253 1254 static inline void 1255 nm_update_hostrings_mode(struct netmap_adapter *na) 1256 { 1257 /* Process nr_mode and nr_pending_mode for host rings. */ 1258 na->tx_rings[na->num_tx_rings].nr_mode = 1259 na->tx_rings[na->num_tx_rings].nr_pending_mode; 1260 na->rx_rings[na->num_rx_rings].nr_mode = 1261 na->rx_rings[na->num_rx_rings].nr_pending_mode; 1262 } 1263 1264 /* set/clear native flags and if_transmit/netdev_ops */ 1265 static inline void 1266 nm_set_native_flags(struct netmap_adapter *na) 1267 { 1268 struct ifnet *ifp = na->ifp; 1269 1270 /* We do the setup for intercepting packets only if we are the 1271 * first user of this adapapter. */ 1272 if (na->active_fds > 0) { 1273 return; 1274 } 1275 1276 na->na_flags |= NAF_NETMAP_ON; 1277 #ifdef IFCAP_NETMAP /* or FreeBSD ? */ 1278 ifp->if_capenable |= IFCAP_NETMAP; 1279 #endif 1280 #if defined (__FreeBSD__) 1281 na->if_transmit = ifp->if_transmit; 1282 ifp->if_transmit = netmap_transmit; 1283 #elif defined (_WIN32) 1284 (void)ifp; /* prevent a warning */ 1285 #elif defined (linux) 1286 na->if_transmit = (void *)ifp->netdev_ops; 1287 ifp->netdev_ops = &((struct netmap_hw_adapter *)na)->nm_ndo; 1288 ((struct netmap_hw_adapter *)na)->save_ethtool = ifp->ethtool_ops; 1289 ifp->ethtool_ops = &((struct netmap_hw_adapter*)na)->nm_eto; 1290 #endif /* linux */ 1291 nm_update_hostrings_mode(na); 1292 } 1293 1294 static inline void 1295 nm_clear_native_flags(struct netmap_adapter *na) 1296 { 1297 struct ifnet *ifp = na->ifp; 1298 1299 /* We undo the setup for intercepting packets only if we are the 1300 * last user of this adapapter. */ 1301 if (na->active_fds > 0) { 1302 return; 1303 } 1304 1305 nm_update_hostrings_mode(na); 1306 1307 #if defined(__FreeBSD__) 1308 ifp->if_transmit = na->if_transmit; 1309 #elif defined(_WIN32) 1310 (void)ifp; /* prevent a warning */ 1311 #else 1312 ifp->netdev_ops = (void *)na->if_transmit; 1313 ifp->ethtool_ops = ((struct netmap_hw_adapter*)na)->save_ethtool; 1314 #endif 1315 na->na_flags &= ~NAF_NETMAP_ON; 1316 #ifdef IFCAP_NETMAP /* or FreeBSD ? */ 1317 ifp->if_capenable &= ~IFCAP_NETMAP; 1318 #endif 1319 } 1320 1321 /* 1322 * nm_*sync_prologue() functions are used in ioctl/poll and ptnetmap 1323 * kthreads. 1324 * We need netmap_ring* parameter, because in ptnetmap it is decoupled 1325 * from host kring. 1326 * The user-space ring pointers (head/cur/tail) are shared through 1327 * CSB between host and guest. 1328 */ 1329 1330 /* 1331 * validates parameters in the ring/kring, returns a value for head 1332 * If any error, returns ring_size to force a reinit. 1333 */ 1334 uint32_t nm_txsync_prologue(struct netmap_kring *, struct netmap_ring *); 1335 1336 1337 /* 1338 * validates parameters in the ring/kring, returns a value for head 1339 * If any error, returns ring_size lim to force a reinit. 1340 */ 1341 uint32_t nm_rxsync_prologue(struct netmap_kring *, struct netmap_ring *); 1342 1343 1344 /* check/fix address and len in tx rings */ 1345 #if 1 /* debug version */ 1346 #define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1347 if (_a == NETMAP_BUF_BASE(_na) || _l > NETMAP_BUF_SIZE(_na)) { \ 1348 RD(5, "bad addr/len ring %d slot %d idx %d len %d", \ 1349 kring->ring_id, nm_i, slot->buf_idx, len); \ 1350 if (_l > NETMAP_BUF_SIZE(_na)) \ 1351 _l = NETMAP_BUF_SIZE(_na); \ 1352 } } while (0) 1353 #else /* no debug version */ 1354 #define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1355 if (_l > NETMAP_BUF_SIZE(_na)) \ 1356 _l = NETMAP_BUF_SIZE(_na); \ 1357 } while (0) 1358 #endif 1359 1360 1361 /*---------------------------------------------------------------*/ 1362 /* 1363 * Support routines used by netmap subsystems 1364 * (native drivers, VALE, generic, pipes, monitors, ...) 1365 */ 1366 1367 1368 /* common routine for all functions that create a netmap adapter. It performs 1369 * two main tasks: 1370 * - if the na points to an ifp, mark the ifp as netmap capable 1371 * using na as its native adapter; 1372 * - provide defaults for the setup callbacks and the memory allocator 1373 */ 1374 int netmap_attach_common(struct netmap_adapter *); 1375 /* fill priv->np_[tr]xq{first,last} using the ringid and flags information 1376 * coming from a struct nmreq 1377 */ 1378 int netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags); 1379 /* update the ring parameters (number and size of tx and rx rings). 1380 * It calls the nm_config callback, if available. 1381 */ 1382 int netmap_update_config(struct netmap_adapter *na); 1383 /* create and initialize the common fields of the krings array. 1384 * using the information that must be already available in the na. 1385 * tailroom can be used to request the allocation of additional 1386 * tailroom bytes after the krings array. This is used by 1387 * netmap_vp_adapter's (i.e., VALE ports) to make room for 1388 * leasing-related data structures 1389 */ 1390 int netmap_krings_create(struct netmap_adapter *na, u_int tailroom); 1391 /* deletes the kring array of the adapter. The array must have 1392 * been created using netmap_krings_create 1393 */ 1394 void netmap_krings_delete(struct netmap_adapter *na); 1395 1396 int netmap_hw_krings_create(struct netmap_adapter *na); 1397 void netmap_hw_krings_delete(struct netmap_adapter *na); 1398 1399 /* set the stopped/enabled status of ring 1400 * When stopping, they also wait for all current activity on the ring to 1401 * terminate. The status change is then notified using the na nm_notify 1402 * callback. 1403 */ 1404 void netmap_set_ring(struct netmap_adapter *, u_int ring_id, enum txrx, int stopped); 1405 /* set the stopped/enabled status of all rings of the adapter. */ 1406 void netmap_set_all_rings(struct netmap_adapter *, int stopped); 1407 /* convenience wrappers for netmap_set_all_rings */ 1408 void netmap_disable_all_rings(struct ifnet *); 1409 void netmap_enable_all_rings(struct ifnet *); 1410 1411 int netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, 1412 uint16_t ringid, uint32_t flags); 1413 void netmap_do_unregif(struct netmap_priv_d *priv); 1414 1415 u_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); 1416 int netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, 1417 struct ifnet **ifp, struct netmap_mem_d *nmd, int create); 1418 void netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp); 1419 int netmap_get_hw_na(struct ifnet *ifp, 1420 struct netmap_mem_d *nmd, struct netmap_adapter **na); 1421 1422 1423 #ifdef WITH_VALE 1424 /* 1425 * The following bridge-related functions are used by other 1426 * kernel modules. 1427 * 1428 * VALE only supports unicast or broadcast. The lookup 1429 * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports, 1430 * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 to indicate 1431 * drop. 1432 */ 1433 typedef u_int (*bdg_lookup_fn_t)(struct nm_bdg_fwd *ft, uint8_t *ring_nr, 1434 struct netmap_vp_adapter *); 1435 typedef int (*bdg_config_fn_t)(struct nm_ifreq *); 1436 typedef void (*bdg_dtor_fn_t)(const struct netmap_vp_adapter *); 1437 struct netmap_bdg_ops { 1438 bdg_lookup_fn_t lookup; 1439 bdg_config_fn_t config; 1440 bdg_dtor_fn_t dtor; 1441 }; 1442 1443 u_int netmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, 1444 struct netmap_vp_adapter *); 1445 1446 #define NM_BRIDGES 8 /* number of bridges */ 1447 #define NM_BDG_MAXPORTS 254 /* up to 254 */ 1448 #define NM_BDG_BROADCAST NM_BDG_MAXPORTS 1449 #define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1) 1450 1451 /* these are redefined in case of no VALE support */ 1452 int netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, 1453 struct netmap_mem_d *nmd, int create); 1454 struct nm_bridge *netmap_init_bridges2(u_int); 1455 void netmap_uninit_bridges2(struct nm_bridge *, u_int); 1456 int netmap_init_bridges(void); 1457 void netmap_uninit_bridges(void); 1458 int netmap_bdg_ctl(struct nmreq *nmr, struct netmap_bdg_ops *bdg_ops); 1459 int netmap_bdg_config(struct nmreq *nmr); 1460 1461 #else /* !WITH_VALE */ 1462 #define netmap_get_bdg_na(_1, _2, _3, _4) 0 1463 #define netmap_init_bridges(_1) 0 1464 #define netmap_uninit_bridges() 1465 #define netmap_bdg_ctl(_1, _2) EINVAL 1466 #endif /* !WITH_VALE */ 1467 1468 #ifdef WITH_PIPES 1469 /* max number of pipes per device */ 1470 #define NM_MAXPIPES 64 /* XXX this should probably be a sysctl */ 1471 void netmap_pipe_dealloc(struct netmap_adapter *); 1472 int netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, 1473 struct netmap_mem_d *nmd, int create); 1474 #else /* !WITH_PIPES */ 1475 #define NM_MAXPIPES 0 1476 #define netmap_pipe_alloc(_1, _2) 0 1477 #define netmap_pipe_dealloc(_1) 1478 #define netmap_get_pipe_na(nmr, _2, _3, _4) \ 1479 ({ int role__ = (nmr)->nr_flags & NR_REG_MASK; \ 1480 (role__ == NR_REG_PIPE_MASTER || \ 1481 role__ == NR_REG_PIPE_SLAVE) ? EOPNOTSUPP : 0; }) 1482 #endif 1483 1484 #ifdef WITH_MONITOR 1485 int netmap_get_monitor_na(struct nmreq *nmr, struct netmap_adapter **na, 1486 struct netmap_mem_d *nmd, int create); 1487 void netmap_monitor_stop(struct netmap_adapter *na); 1488 #else 1489 #define netmap_get_monitor_na(nmr, _2, _3, _4) \ 1490 ((nmr)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0) 1491 #endif 1492 1493 #ifdef CONFIG_NET_NS 1494 struct net *netmap_bns_get(void); 1495 void netmap_bns_put(struct net *); 1496 void netmap_bns_getbridges(struct nm_bridge **, u_int *); 1497 #else 1498 #define netmap_bns_get() 1499 #define netmap_bns_put(_1) 1500 #define netmap_bns_getbridges(b, n) \ 1501 do { *b = nm_bridges; *n = NM_BRIDGES; } while (0) 1502 #endif 1503 1504 /* Various prototypes */ 1505 int netmap_poll(struct netmap_priv_d *, int events, NM_SELRECORD_T *td); 1506 int netmap_init(void); 1507 void netmap_fini(void); 1508 int netmap_get_memory(struct netmap_priv_d* p); 1509 void netmap_dtor(void *data); 1510 1511 int netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, struct thread *); 1512 1513 /* netmap_adapter creation/destruction */ 1514 1515 // #define NM_DEBUG_PUTGET 1 1516 1517 #ifdef NM_DEBUG_PUTGET 1518 1519 #define NM_DBG(f) __##f 1520 1521 void __netmap_adapter_get(struct netmap_adapter *na); 1522 1523 #define netmap_adapter_get(na) \ 1524 do { \ 1525 struct netmap_adapter *__na = na; \ 1526 D("getting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1527 __netmap_adapter_get(__na); \ 1528 } while (0) 1529 1530 int __netmap_adapter_put(struct netmap_adapter *na); 1531 1532 #define netmap_adapter_put(na) \ 1533 ({ \ 1534 struct netmap_adapter *__na = na; \ 1535 D("putting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1536 __netmap_adapter_put(__na); \ 1537 }) 1538 1539 #else /* !NM_DEBUG_PUTGET */ 1540 1541 #define NM_DBG(f) f 1542 void netmap_adapter_get(struct netmap_adapter *na); 1543 int netmap_adapter_put(struct netmap_adapter *na); 1544 1545 #endif /* !NM_DEBUG_PUTGET */ 1546 1547 1548 /* 1549 * module variables 1550 */ 1551 #define NETMAP_BUF_BASE(_na) ((_na)->na_lut.lut[0].vaddr) 1552 #define NETMAP_BUF_SIZE(_na) ((_na)->na_lut.objsize) 1553 extern int netmap_no_pendintr; 1554 extern int netmap_mitigate; 1555 extern int netmap_verbose; /* for debugging */ 1556 enum { /* verbose flags */ 1557 NM_VERB_ON = 1, /* generic verbose */ 1558 NM_VERB_HOST = 0x2, /* verbose host stack */ 1559 NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ 1560 NM_VERB_TXSYNC = 0x20, 1561 NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ 1562 NM_VERB_TXINTR = 0x200, 1563 NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ 1564 NM_VERB_NIC_TXSYNC = 0x2000, 1565 }; 1566 1567 extern int netmap_txsync_retry; 1568 extern int netmap_flags; 1569 extern int netmap_generic_mit; 1570 extern int netmap_generic_ringsize; 1571 extern int netmap_generic_rings; 1572 #ifdef linux 1573 extern int netmap_generic_txqdisc; 1574 #endif 1575 extern int ptnetmap_tx_workers; 1576 1577 /* 1578 * NA returns a pointer to the struct netmap adapter from the ifp, 1579 * WNA is used to write it. 1580 */ 1581 #define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 1582 1583 /* 1584 * On old versions of FreeBSD, NA(ifp) is a pspare. On linux we 1585 * overload another pointer in the netdev. 1586 * 1587 * We check if NA(ifp) is set and its first element has a related 1588 * magic value. The capenable is within the struct netmap_adapter. 1589 */ 1590 #define NETMAP_MAGIC 0x52697a7a 1591 1592 #define NM_NA_VALID(ifp) (NA(ifp) && \ 1593 ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 1594 1595 #define NM_ATTACH_NA(ifp, na) do { \ 1596 WNA(ifp) = na; \ 1597 if (NA(ifp)) \ 1598 NA(ifp)->magic = \ 1599 ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC; \ 1600 } while(0) 1601 1602 #define NM_IS_NATIVE(ifp) (NM_NA_VALID(ifp) && NA(ifp)->nm_dtor == netmap_hw_dtor) 1603 1604 #if defined(__FreeBSD__) 1605 1606 /* Assigns the device IOMMU domain to an allocator. 1607 * Returns -ENOMEM in case the domain is different */ 1608 #define nm_iommu_group_id(dev) (0) 1609 1610 /* Callback invoked by the dma machinery after a successful dmamap_load */ 1611 static void netmap_dmamap_cb(__unused void *arg, 1612 __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 1613 { 1614 } 1615 1616 /* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 1617 * XXX can we do it without a callback ? 1618 */ 1619 static inline int 1620 netmap_load_map(struct netmap_adapter *na, 1621 bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1622 { 1623 if (map) 1624 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1625 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1626 return 0; 1627 } 1628 1629 static inline void 1630 netmap_unload_map(struct netmap_adapter *na, 1631 bus_dma_tag_t tag, bus_dmamap_t map) 1632 { 1633 if (map) 1634 bus_dmamap_unload(tag, map); 1635 } 1636 1637 #define netmap_sync_map(na, tag, map, sz, t) 1638 1639 /* update the map when a buffer changes. */ 1640 static inline void 1641 netmap_reload_map(struct netmap_adapter *na, 1642 bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1643 { 1644 if (map) { 1645 bus_dmamap_unload(tag, map); 1646 bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1647 netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1648 } 1649 } 1650 1651 #elif defined(_WIN32) 1652 1653 #else /* linux */ 1654 1655 int nm_iommu_group_id(bus_dma_tag_t dev); 1656 #include <linux/dma-mapping.h> 1657 1658 /* 1659 * on linux we need 1660 * dma_map_single(&pdev->dev, virt_addr, len, direction) 1661 * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction) 1662 */ 1663 #if 0 1664 struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 1665 /* set time_stamp *before* dma to help avoid a possible race */ 1666 buffer_info->time_stamp = jiffies; 1667 buffer_info->mapped_as_page = false; 1668 buffer_info->length = len; 1669 //buffer_info->next_to_watch = l; 1670 /* reload dma map */ 1671 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1672 NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1673 buffer_info->dma = dma_map_single(&adapter->pdev->dev, 1674 addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1675 1676 if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 1677 D("dma mapping error"); 1678 /* goto dma_error; See e1000_put_txbuf() */ 1679 /* XXX reset */ 1680 } 1681 tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 1682 1683 #endif 1684 1685 static inline int 1686 netmap_load_map(struct netmap_adapter *na, 1687 bus_dma_tag_t tag, bus_dmamap_t map, void *buf, u_int size) 1688 { 1689 if (map) { 1690 *map = dma_map_single(na->pdev, buf, size, 1691 DMA_BIDIRECTIONAL); 1692 if (dma_mapping_error(na->pdev, *map)) { 1693 *map = 0; 1694 return ENOMEM; 1695 } 1696 } 1697 return 0; 1698 } 1699 1700 static inline void 1701 netmap_unload_map(struct netmap_adapter *na, 1702 bus_dma_tag_t tag, bus_dmamap_t map, u_int sz) 1703 { 1704 if (*map) { 1705 dma_unmap_single(na->pdev, *map, sz, 1706 DMA_BIDIRECTIONAL); 1707 } 1708 } 1709 1710 static inline void 1711 netmap_sync_map(struct netmap_adapter *na, 1712 bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t) 1713 { 1714 if (*map) { 1715 if (t == NR_RX) 1716 dma_sync_single_for_cpu(na->pdev, *map, sz, 1717 DMA_FROM_DEVICE); 1718 else 1719 dma_sync_single_for_device(na->pdev, *map, sz, 1720 DMA_TO_DEVICE); 1721 } 1722 } 1723 1724 static inline void 1725 netmap_reload_map(struct netmap_adapter *na, 1726 bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1727 { 1728 u_int sz = NETMAP_BUF_SIZE(na); 1729 1730 if (*map) { 1731 dma_unmap_single(na->pdev, *map, sz, 1732 DMA_BIDIRECTIONAL); 1733 } 1734 1735 *map = dma_map_single(na->pdev, buf, sz, 1736 DMA_BIDIRECTIONAL); 1737 } 1738 1739 #endif /* linux */ 1740 1741 1742 /* 1743 * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 1744 */ 1745 static inline int 1746 netmap_idx_n2k(struct netmap_kring *kr, int idx) 1747 { 1748 int n = kr->nkr_num_slots; 1749 idx += kr->nkr_hwofs; 1750 if (idx < 0) 1751 return idx + n; 1752 else if (idx < n) 1753 return idx; 1754 else 1755 return idx - n; 1756 } 1757 1758 1759 static inline int 1760 netmap_idx_k2n(struct netmap_kring *kr, int idx) 1761 { 1762 int n = kr->nkr_num_slots; 1763 idx -= kr->nkr_hwofs; 1764 if (idx < 0) 1765 return idx + n; 1766 else if (idx < n) 1767 return idx; 1768 else 1769 return idx - n; 1770 } 1771 1772 1773 /* Entries of the look-up table. */ 1774 #ifdef __FreeBSD__ 1775 struct lut_entry { 1776 void *vaddr; /* virtual address. */ 1777 vm_paddr_t paddr; /* physical address. */ 1778 }; 1779 #else /* linux & _WIN32 */ 1780 /* dma-mapping in linux can assign a buffer a different address 1781 * depending on the device, so we need to have a separate 1782 * physical-address look-up table for each na. 1783 * We can still share the vaddrs, though, therefore we split 1784 * the lut_entry structure. 1785 */ 1786 struct lut_entry { 1787 void *vaddr; /* virtual address. */ 1788 }; 1789 1790 struct plut_entry { 1791 vm_paddr_t paddr; /* physical address. */ 1792 }; 1793 #endif /* linux & _WIN32 */ 1794 1795 struct netmap_obj_pool; 1796 1797 /* 1798 * NMB return the virtual address of a buffer (buffer 0 on bad index) 1799 * PNMB also fills the physical address 1800 */ 1801 static inline void * 1802 NMB(struct netmap_adapter *na, struct netmap_slot *slot) 1803 { 1804 struct lut_entry *lut = na->na_lut.lut; 1805 uint32_t i = slot->buf_idx; 1806 return (unlikely(i >= na->na_lut.objtotal)) ? 1807 lut[0].vaddr : lut[i].vaddr; 1808 } 1809 1810 static inline void * 1811 PNMB(struct netmap_adapter *na, struct netmap_slot *slot, uint64_t *pp) 1812 { 1813 uint32_t i = slot->buf_idx; 1814 struct lut_entry *lut = na->na_lut.lut; 1815 struct plut_entry *plut = na->na_lut.plut; 1816 void *ret = (i >= na->na_lut.objtotal) ? lut[0].vaddr : lut[i].vaddr; 1817 1818 #ifdef _WIN32 1819 *pp = (i >= na->na_lut.objtotal) ? (uint64_t)plut[0].paddr.QuadPart : (uint64_t)plut[i].paddr.QuadPart; 1820 #else 1821 *pp = (i >= na->na_lut.objtotal) ? plut[0].paddr : plut[i].paddr; 1822 #endif 1823 return ret; 1824 } 1825 1826 1827 /* 1828 * Structure associated to each netmap file descriptor. 1829 * It is created on open and left unbound (np_nifp == NULL). 1830 * A successful NIOCREGIF will set np_nifp and the first few fields; 1831 * this is protected by a global lock (NMG_LOCK) due to low contention. 1832 * 1833 * np_refs counts the number of references to the structure: one for the fd, 1834 * plus (on FreeBSD) one for each active mmap which we track ourselves 1835 * (linux automatically tracks them, but FreeBSD does not). 1836 * np_refs is protected by NMG_LOCK. 1837 * 1838 * Read access to the structure is lock free, because ni_nifp once set 1839 * can only go to 0 when nobody is using the entry anymore. Readers 1840 * must check that np_nifp != NULL before using the other fields. 1841 */ 1842 struct netmap_priv_d { 1843 struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ 1844 1845 struct netmap_adapter *np_na; 1846 struct ifnet *np_ifp; 1847 uint32_t np_flags; /* from the ioctl */ 1848 u_int np_qfirst[NR_TXRX], 1849 np_qlast[NR_TXRX]; /* range of tx/rx rings to scan */ 1850 uint16_t np_txpoll; 1851 int np_sync_flags; /* to be passed to nm_sync */ 1852 1853 int np_refs; /* use with NMG_LOCK held */ 1854 1855 /* pointers to the selinfo to be used for selrecord. 1856 * Either the local or the global one depending on the 1857 * number of rings. 1858 */ 1859 NM_SELINFO_T *np_si[NR_TXRX]; 1860 struct thread *np_td; /* kqueue, just debugging */ 1861 }; 1862 1863 struct netmap_priv_d *netmap_priv_new(void); 1864 void netmap_priv_delete(struct netmap_priv_d *); 1865 1866 static inline int nm_kring_pending(struct netmap_priv_d *np) 1867 { 1868 struct netmap_adapter *na = np->np_na; 1869 enum txrx t; 1870 int i; 1871 1872 for_rx_tx(t) { 1873 for (i = np->np_qfirst[t]; i < np->np_qlast[t]; i++) { 1874 struct netmap_kring *kring = &NMR(na, t)[i]; 1875 if (kring->nr_mode != kring->nr_pending_mode) { 1876 return 1; 1877 } 1878 } 1879 } 1880 return 0; 1881 } 1882 1883 #ifdef WITH_PIPES 1884 int netmap_pipe_txsync(struct netmap_kring *txkring, int flags); 1885 int netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags); 1886 #endif /* WITH_PIPES */ 1887 1888 #ifdef WITH_MONITOR 1889 1890 struct netmap_monitor_adapter { 1891 struct netmap_adapter up; 1892 1893 struct netmap_priv_d priv; 1894 uint32_t flags; 1895 }; 1896 1897 #endif /* WITH_MONITOR */ 1898 1899 1900 #ifdef WITH_GENERIC 1901 /* 1902 * generic netmap emulation for devices that do not have 1903 * native netmap support. 1904 */ 1905 int generic_netmap_attach(struct ifnet *ifp); 1906 int generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; 1907 1908 int nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept); 1909 int nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept); 1910 1911 int na_is_generic(struct netmap_adapter *na); 1912 1913 /* 1914 * the generic transmit routine is passed a structure to optionally 1915 * build a queue of descriptors, in an OS-specific way. 1916 * The payload is at addr, if non-null, and the routine should send or queue 1917 * the packet, returning 0 if successful, 1 on failure. 1918 * 1919 * At the end, if head is non-null, there will be an additional call 1920 * to the function with addr = NULL; this should tell the OS-specific 1921 * routine to send the queue and free any resources. Failure is ignored. 1922 */ 1923 struct nm_os_gen_arg { 1924 struct ifnet *ifp; 1925 void *m; /* os-specific mbuf-like object */ 1926 void *head, *tail; /* tailq, if the OS-specific routine needs to build one */ 1927 void *addr; /* payload of current packet */ 1928 u_int len; /* packet length */ 1929 u_int ring_nr; /* packet length */ 1930 u_int qevent; /* in txqdisc mode, place an event on this mbuf */ 1931 }; 1932 1933 int nm_os_generic_xmit_frame(struct nm_os_gen_arg *); 1934 int nm_os_generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); 1935 void nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); 1936 void nm_os_generic_set_features(struct netmap_generic_adapter *gna); 1937 1938 static inline struct ifnet* 1939 netmap_generic_getifp(struct netmap_generic_adapter *gna) 1940 { 1941 if (gna->prev) 1942 return gna->prev->ifp; 1943 1944 return gna->up.up.ifp; 1945 } 1946 1947 void netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done); 1948 1949 //#define RATE_GENERIC /* Enables communication statistics for generic. */ 1950 #ifdef RATE_GENERIC 1951 void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi); 1952 #else 1953 #define generic_rate(txp, txs, txi, rxp, rxs, rxi) 1954 #endif 1955 1956 /* 1957 * netmap_mitigation API. This is used by the generic adapter 1958 * to reduce the number of interrupt requests/selwakeup 1959 * to clients on incoming packets. 1960 */ 1961 void nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, 1962 struct netmap_adapter *na); 1963 void nm_os_mitigation_start(struct nm_generic_mit *mit); 1964 void nm_os_mitigation_restart(struct nm_generic_mit *mit); 1965 int nm_os_mitigation_active(struct nm_generic_mit *mit); 1966 void nm_os_mitigation_cleanup(struct nm_generic_mit *mit); 1967 #else /* !WITH_GENERIC */ 1968 #define generic_netmap_attach(ifp) (EOPNOTSUPP) 1969 #define na_is_generic(na) (0) 1970 #endif /* WITH_GENERIC */ 1971 1972 /* Shared declarations for the VALE switch. */ 1973 1974 /* 1975 * Each transmit queue accumulates a batch of packets into 1976 * a structure before forwarding. Packets to the same 1977 * destination are put in a list using ft_next as a link field. 1978 * ft_frags and ft_next are valid only on the first fragment. 1979 */ 1980 struct nm_bdg_fwd { /* forwarding entry for a bridge */ 1981 void *ft_buf; /* netmap or indirect buffer */ 1982 uint8_t ft_frags; /* how many fragments (only on 1st frag) */ 1983 uint8_t _ft_port; /* dst port (unused) */ 1984 uint16_t ft_flags; /* flags, e.g. indirect */ 1985 uint16_t ft_len; /* src fragment len */ 1986 uint16_t ft_next; /* next packet to same destination */ 1987 }; 1988 1989 /* struct 'virtio_net_hdr' from linux. */ 1990 struct nm_vnet_hdr { 1991 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ 1992 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ 1993 uint8_t flags; 1994 #define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ 1995 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ 1996 #define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ 1997 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ 1998 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ 1999 uint8_t gso_type; 2000 uint16_t hdr_len; 2001 uint16_t gso_size; 2002 uint16_t csum_start; 2003 uint16_t csum_offset; 2004 }; 2005 2006 #define WORST_CASE_GSO_HEADER (14+40+60) /* IPv6 + TCP */ 2007 2008 /* Private definitions for IPv4, IPv6, UDP and TCP headers. */ 2009 2010 struct nm_iphdr { 2011 uint8_t version_ihl; 2012 uint8_t tos; 2013 uint16_t tot_len; 2014 uint16_t id; 2015 uint16_t frag_off; 2016 uint8_t ttl; 2017 uint8_t protocol; 2018 uint16_t check; 2019 uint32_t saddr; 2020 uint32_t daddr; 2021 /*The options start here. */ 2022 }; 2023 2024 struct nm_tcphdr { 2025 uint16_t source; 2026 uint16_t dest; 2027 uint32_t seq; 2028 uint32_t ack_seq; 2029 uint8_t doff; /* Data offset + Reserved */ 2030 uint8_t flags; 2031 uint16_t window; 2032 uint16_t check; 2033 uint16_t urg_ptr; 2034 }; 2035 2036 struct nm_udphdr { 2037 uint16_t source; 2038 uint16_t dest; 2039 uint16_t len; 2040 uint16_t check; 2041 }; 2042 2043 struct nm_ipv6hdr { 2044 uint8_t priority_version; 2045 uint8_t flow_lbl[3]; 2046 2047 uint16_t payload_len; 2048 uint8_t nexthdr; 2049 uint8_t hop_limit; 2050 2051 uint8_t saddr[16]; 2052 uint8_t daddr[16]; 2053 }; 2054 2055 /* Type used to store a checksum (in host byte order) that hasn't been 2056 * folded yet. 2057 */ 2058 #define rawsum_t uint32_t 2059 2060 rawsum_t nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum); 2061 uint16_t nm_os_csum_ipv4(struct nm_iphdr *iph); 2062 void nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 2063 size_t datalen, uint16_t *check); 2064 void nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 2065 size_t datalen, uint16_t *check); 2066 uint16_t nm_os_csum_fold(rawsum_t cur_sum); 2067 2068 void bdg_mismatch_datapath(struct netmap_vp_adapter *na, 2069 struct netmap_vp_adapter *dst_na, 2070 const struct nm_bdg_fwd *ft_p, 2071 struct netmap_ring *dst_ring, 2072 u_int *j, u_int lim, u_int *howmany); 2073 2074 /* persistent virtual port routines */ 2075 int nm_os_vi_persist(const char *, struct ifnet **); 2076 void nm_os_vi_detach(struct ifnet *); 2077 void nm_os_vi_init_index(void); 2078 2079 /* 2080 * kernel thread routines 2081 */ 2082 struct nm_kctx; /* OS-specific kernel context - opaque */ 2083 typedef void (*nm_kctx_worker_fn_t)(void *data, int is_kthread); 2084 typedef void (*nm_kctx_notify_fn_t)(void *data); 2085 2086 /* kthread configuration */ 2087 struct nm_kctx_cfg { 2088 long type; /* kthread type/identifier */ 2089 nm_kctx_worker_fn_t worker_fn; /* worker function */ 2090 void *worker_private;/* worker parameter */ 2091 nm_kctx_notify_fn_t notify_fn; /* notify function */ 2092 int attach_user; /* attach kthread to user process */ 2093 int use_kthread; /* use a kthread for the context */ 2094 }; 2095 /* kthread configuration */ 2096 struct nm_kctx *nm_os_kctx_create(struct nm_kctx_cfg *cfg, 2097 unsigned int cfgtype, 2098 void *opaque); 2099 int nm_os_kctx_worker_start(struct nm_kctx *); 2100 void nm_os_kctx_worker_stop(struct nm_kctx *); 2101 void nm_os_kctx_destroy(struct nm_kctx *); 2102 void nm_os_kctx_worker_wakeup(struct nm_kctx *nmk); 2103 void nm_os_kctx_send_irq(struct nm_kctx *); 2104 void nm_os_kctx_worker_setaff(struct nm_kctx *, int); 2105 u_int nm_os_ncpus(void); 2106 2107 #ifdef WITH_PTNETMAP_HOST 2108 /* 2109 * netmap adapter for host ptnetmap ports 2110 */ 2111 struct netmap_pt_host_adapter { 2112 struct netmap_adapter up; 2113 2114 /* the passed-through adapter */ 2115 struct netmap_adapter *parent; 2116 /* parent->na_flags, saved at NETMAP_PT_HOST_CREATE time, 2117 * and restored at NETMAP_PT_HOST_DELETE time */ 2118 uint32_t parent_na_flags; 2119 2120 int (*parent_nm_notify)(struct netmap_kring *kring, int flags); 2121 void *ptns; 2122 }; 2123 /* ptnetmap HOST routines */ 2124 int netmap_get_pt_host_na(struct nmreq *nmr, struct netmap_adapter **na, 2125 struct netmap_mem_d * nmd, int create); 2126 int ptnetmap_ctl(struct nmreq *nmr, struct netmap_adapter *na); 2127 static inline int 2128 nm_ptnetmap_host_on(struct netmap_adapter *na) 2129 { 2130 return na && na->na_flags & NAF_PTNETMAP_HOST; 2131 } 2132 #else /* !WITH_PTNETMAP_HOST */ 2133 #define netmap_get_pt_host_na(nmr, _2, _3, _4) \ 2134 ((nmr)->nr_flags & (NR_PTNETMAP_HOST) ? EOPNOTSUPP : 0) 2135 #define ptnetmap_ctl(_1, _2) EINVAL 2136 #define nm_ptnetmap_host_on(_1) EINVAL 2137 #endif /* !WITH_PTNETMAP_HOST */ 2138 2139 #ifdef WITH_PTNETMAP_GUEST 2140 /* ptnetmap GUEST routines */ 2141 2142 /* 2143 * netmap adapter for guest ptnetmap ports 2144 */ 2145 struct netmap_pt_guest_adapter { 2146 /* The netmap adapter to be used by netmap applications. 2147 * This field must be the first, to allow upcast. */ 2148 struct netmap_hw_adapter hwup; 2149 2150 /* The netmap adapter to be used by the driver. */ 2151 struct netmap_hw_adapter dr; 2152 2153 /* Reference counter to track users of backend netmap port: the 2154 * network stack and netmap clients. 2155 * Used to decide when we need (de)allocate krings/rings and 2156 * start (stop) ptnetmap kthreads. */ 2157 int backend_regifs; 2158 2159 }; 2160 2161 int netmap_pt_guest_attach(struct netmap_adapter *na, 2162 unsigned int nifp_offset, 2163 unsigned int memid); 2164 struct ptnet_csb_gh; 2165 struct ptnet_csb_hg; 2166 bool netmap_pt_guest_txsync(struct ptnet_csb_gh *ptgh, 2167 struct ptnet_csb_hg *pthg, 2168 struct netmap_kring *kring, 2169 int flags); 2170 bool netmap_pt_guest_rxsync(struct ptnet_csb_gh *ptgh, 2171 struct ptnet_csb_hg *pthg, 2172 struct netmap_kring *kring, int flags); 2173 int ptnet_nm_krings_create(struct netmap_adapter *na); 2174 void ptnet_nm_krings_delete(struct netmap_adapter *na); 2175 void ptnet_nm_dtor(struct netmap_adapter *na); 2176 #endif /* WITH_PTNETMAP_GUEST */ 2177 2178 #endif /* _NET_NETMAP_KERN_H_ */ 2179