1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* $FreeBSD$ */ 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 32 #include <sys/param.h> 33 #include <sys/module.h> 34 #include <sys/errno.h> 35 #include <sys/jail.h> 36 #include <sys/poll.h> /* POLLIN, POLLOUT */ 37 #include <sys/kernel.h> /* types used in module initialization */ 38 #include <sys/conf.h> /* DEV_MODULE_ORDERED */ 39 #include <sys/endian.h> 40 #include <sys/syscallsubr.h> /* kern_ioctl() */ 41 42 #include <sys/rwlock.h> 43 44 #include <vm/vm.h> /* vtophys */ 45 #include <vm/pmap.h> /* vtophys */ 46 #include <vm/vm_param.h> 47 #include <vm/vm_object.h> 48 #include <vm/vm_page.h> 49 #include <vm/vm_pager.h> 50 #include <vm/uma.h> 51 52 53 #include <sys/malloc.h> 54 #include <sys/socket.h> /* sockaddrs */ 55 #include <sys/selinfo.h> 56 #include <sys/kthread.h> /* kthread_add() */ 57 #include <sys/proc.h> /* PROC_LOCK() */ 58 #include <sys/unistd.h> /* RFNOWAIT */ 59 #include <sys/sched.h> /* sched_bind() */ 60 #include <sys/smp.h> /* mp_maxid */ 61 #include <net/if.h> 62 #include <net/if_var.h> 63 #include <net/if_types.h> /* IFT_ETHER */ 64 #include <net/ethernet.h> /* ether_ifdetach */ 65 #include <net/if_dl.h> /* LLADDR */ 66 #include <machine/bus.h> /* bus_dmamap_* */ 67 #include <netinet/in.h> /* in6_cksum_pseudo() */ 68 #include <machine/in_cksum.h> /* in_pseudo(), in_cksum_hdr() */ 69 70 #include <net/netmap.h> 71 #include <dev/netmap/netmap_kern.h> 72 #include <net/netmap_virt.h> 73 #include <dev/netmap/netmap_mem2.h> 74 75 76 /* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 77 78 void nm_os_selinfo_init(NM_SELINFO_T *si) { 79 struct mtx *m = &si->m; 80 mtx_init(m, "nm_kn_lock", NULL, MTX_DEF); 81 knlist_init_mtx(&si->si.si_note, m); 82 } 83 84 void 85 nm_os_selinfo_uninit(NM_SELINFO_T *si) 86 { 87 /* XXX kqueue(9) needed; these will mirror knlist_init. */ 88 knlist_delete(&si->si.si_note, curthread, /*islocked=*/0); 89 knlist_destroy(&si->si.si_note); 90 /* now we don't need the mutex anymore */ 91 mtx_destroy(&si->m); 92 } 93 94 void * 95 nm_os_malloc(size_t size) 96 { 97 return malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 98 } 99 100 void * 101 nm_os_realloc(void *addr, size_t new_size, size_t old_size __unused) 102 { 103 return realloc(addr, new_size, M_DEVBUF, M_NOWAIT | M_ZERO); 104 } 105 106 void 107 nm_os_free(void *addr) 108 { 109 free(addr, M_DEVBUF); 110 } 111 112 void 113 nm_os_ifnet_lock(void) 114 { 115 IFNET_RLOCK(); 116 } 117 118 void 119 nm_os_ifnet_unlock(void) 120 { 121 IFNET_RUNLOCK(); 122 } 123 124 static int netmap_use_count = 0; 125 126 void 127 nm_os_get_module(void) 128 { 129 netmap_use_count++; 130 } 131 132 void 133 nm_os_put_module(void) 134 { 135 netmap_use_count--; 136 } 137 138 static void 139 netmap_ifnet_arrival_handler(void *arg __unused, struct ifnet *ifp) 140 { 141 netmap_undo_zombie(ifp); 142 } 143 144 static void 145 netmap_ifnet_departure_handler(void *arg __unused, struct ifnet *ifp) 146 { 147 netmap_make_zombie(ifp); 148 } 149 150 static eventhandler_tag nm_ifnet_ah_tag; 151 static eventhandler_tag nm_ifnet_dh_tag; 152 153 int 154 nm_os_ifnet_init(void) 155 { 156 nm_ifnet_ah_tag = 157 EVENTHANDLER_REGISTER(ifnet_arrival_event, 158 netmap_ifnet_arrival_handler, 159 NULL, EVENTHANDLER_PRI_ANY); 160 nm_ifnet_dh_tag = 161 EVENTHANDLER_REGISTER(ifnet_departure_event, 162 netmap_ifnet_departure_handler, 163 NULL, EVENTHANDLER_PRI_ANY); 164 return 0; 165 } 166 167 void 168 nm_os_ifnet_fini(void) 169 { 170 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 171 nm_ifnet_ah_tag); 172 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 173 nm_ifnet_dh_tag); 174 } 175 176 unsigned 177 nm_os_ifnet_mtu(struct ifnet *ifp) 178 { 179 #if __FreeBSD_version < 1100030 180 return ifp->if_data.ifi_mtu; 181 #else /* __FreeBSD_version >= 1100030 */ 182 return ifp->if_mtu; 183 #endif 184 } 185 186 rawsum_t 187 nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum) 188 { 189 /* TODO XXX please use the FreeBSD implementation for this. */ 190 uint16_t *words = (uint16_t *)data; 191 int nw = len / 2; 192 int i; 193 194 for (i = 0; i < nw; i++) 195 cur_sum += be16toh(words[i]); 196 197 if (len & 1) 198 cur_sum += (data[len-1] << 8); 199 200 return cur_sum; 201 } 202 203 /* Fold a raw checksum: 'cur_sum' is in host byte order, while the 204 * return value is in network byte order. 205 */ 206 uint16_t 207 nm_os_csum_fold(rawsum_t cur_sum) 208 { 209 /* TODO XXX please use the FreeBSD implementation for this. */ 210 while (cur_sum >> 16) 211 cur_sum = (cur_sum & 0xFFFF) + (cur_sum >> 16); 212 213 return htobe16((~cur_sum) & 0xFFFF); 214 } 215 216 uint16_t nm_os_csum_ipv4(struct nm_iphdr *iph) 217 { 218 #if 0 219 return in_cksum_hdr((void *)iph); 220 #else 221 return nm_os_csum_fold(nm_os_csum_raw((uint8_t*)iph, sizeof(struct nm_iphdr), 0)); 222 #endif 223 } 224 225 void 226 nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 227 size_t datalen, uint16_t *check) 228 { 229 #ifdef INET 230 uint16_t pseudolen = datalen + iph->protocol; 231 232 /* Compute and insert the pseudo-header cheksum. */ 233 *check = in_pseudo(iph->saddr, iph->daddr, 234 htobe16(pseudolen)); 235 /* Compute the checksum on TCP/UDP header + payload 236 * (includes the pseudo-header). 237 */ 238 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 239 #else 240 static int notsupported = 0; 241 if (!notsupported) { 242 notsupported = 1; 243 nm_prerr("inet4 segmentation not supported"); 244 } 245 #endif 246 } 247 248 void 249 nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 250 size_t datalen, uint16_t *check) 251 { 252 #ifdef INET6 253 *check = in6_cksum_pseudo((void*)ip6h, datalen, ip6h->nexthdr, 0); 254 *check = nm_os_csum_fold(nm_os_csum_raw(data, datalen, 0)); 255 #else 256 static int notsupported = 0; 257 if (!notsupported) { 258 notsupported = 1; 259 nm_prerr("inet6 segmentation not supported"); 260 } 261 #endif 262 } 263 264 /* on FreeBSD we send up one packet at a time */ 265 void * 266 nm_os_send_up(struct ifnet *ifp, struct mbuf *m, struct mbuf *prev) 267 { 268 NA(ifp)->if_input(ifp, m); 269 return NULL; 270 } 271 272 int 273 nm_os_mbuf_has_csum_offld(struct mbuf *m) 274 { 275 return m->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_SCTP | 276 CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | 277 CSUM_SCTP_IPV6); 278 } 279 280 int 281 nm_os_mbuf_has_seg_offld(struct mbuf *m) 282 { 283 return m->m_pkthdr.csum_flags & CSUM_TSO; 284 } 285 286 static void 287 freebsd_generic_rx_handler(struct ifnet *ifp, struct mbuf *m) 288 { 289 int stolen; 290 291 if (unlikely(!NM_NA_VALID(ifp))) { 292 nm_prlim(1, "Warning: RX packet intercepted, but no" 293 " emulated adapter"); 294 return; 295 } 296 297 stolen = generic_rx_handler(ifp, m); 298 if (!stolen) { 299 struct netmap_generic_adapter *gna = 300 (struct netmap_generic_adapter *)NA(ifp); 301 gna->save_if_input(ifp, m); 302 } 303 } 304 305 /* 306 * Intercept the rx routine in the standard device driver. 307 * Second argument is non-zero to intercept, 0 to restore 308 */ 309 int 310 nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept) 311 { 312 struct netmap_adapter *na = &gna->up.up; 313 struct ifnet *ifp = na->ifp; 314 int ret = 0; 315 316 nm_os_ifnet_lock(); 317 if (intercept) { 318 if (gna->save_if_input) { 319 nm_prerr("RX on %s already intercepted", na->name); 320 ret = EBUSY; /* already set */ 321 goto out; 322 } 323 gna->save_if_input = ifp->if_input; 324 ifp->if_input = freebsd_generic_rx_handler; 325 } else { 326 if (!gna->save_if_input) { 327 nm_prerr("Failed to undo RX intercept on %s", 328 na->name); 329 ret = EINVAL; /* not saved */ 330 goto out; 331 } 332 ifp->if_input = gna->save_if_input; 333 gna->save_if_input = NULL; 334 } 335 out: 336 nm_os_ifnet_unlock(); 337 338 return ret; 339 } 340 341 342 /* 343 * Intercept the packet steering routine in the tx path, 344 * so that we can decide which queue is used for an mbuf. 345 * Second argument is non-zero to intercept, 0 to restore. 346 * On freebsd we just intercept if_transmit. 347 */ 348 int 349 nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept) 350 { 351 struct netmap_adapter *na = &gna->up.up; 352 struct ifnet *ifp = netmap_generic_getifp(gna); 353 354 nm_os_ifnet_lock(); 355 if (intercept) { 356 na->if_transmit = ifp->if_transmit; 357 ifp->if_transmit = netmap_transmit; 358 } else { 359 ifp->if_transmit = na->if_transmit; 360 } 361 nm_os_ifnet_unlock(); 362 363 return 0; 364 } 365 366 367 /* 368 * Transmit routine used by generic_netmap_txsync(). Returns 0 on success 369 * and non-zero on error (which may be packet drops or other errors). 370 * addr and len identify the netmap buffer, m is the (preallocated) 371 * mbuf to use for transmissions. 372 * 373 * We should add a reference to the mbuf so the m_freem() at the end 374 * of the transmission does not consume resources. 375 * 376 * On FreeBSD, and on multiqueue cards, we can force the queue using 377 * if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 378 * i = m->m_pkthdr.flowid % adapter->num_queues; 379 * else 380 * i = curcpu % adapter->num_queues; 381 * 382 */ 383 int 384 nm_os_generic_xmit_frame(struct nm_os_gen_arg *a) 385 { 386 int ret; 387 u_int len = a->len; 388 struct ifnet *ifp = a->ifp; 389 struct mbuf *m = a->m; 390 391 #if __FreeBSD_version < 1100000 392 /* 393 * Old FreeBSD versions. The mbuf has a cluster attached, 394 * we need to copy from the cluster to the netmap buffer. 395 */ 396 if (MBUF_REFCNT(m) != 1) { 397 nm_prerr("invalid refcnt %d for %p", MBUF_REFCNT(m), m); 398 panic("in generic_xmit_frame"); 399 } 400 if (m->m_ext.ext_size < len) { 401 nm_prlim(2, "size %d < len %d", m->m_ext.ext_size, len); 402 len = m->m_ext.ext_size; 403 } 404 bcopy(a->addr, m->m_data, len); 405 #else /* __FreeBSD_version >= 1100000 */ 406 /* New FreeBSD versions. Link the external storage to 407 * the netmap buffer, so that no copy is necessary. */ 408 m->m_ext.ext_buf = m->m_data = a->addr; 409 m->m_ext.ext_size = len; 410 #endif /* __FreeBSD_version >= 1100000 */ 411 412 m->m_len = m->m_pkthdr.len = len; 413 414 /* mbuf refcnt is not contended, no need to use atomic 415 * (a memory barrier is enough). */ 416 SET_MBUF_REFCNT(m, 2); 417 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 418 m->m_pkthdr.flowid = a->ring_nr; 419 m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 420 ret = NA(ifp)->if_transmit(ifp, m); 421 return ret ? -1 : 0; 422 } 423 424 425 #if __FreeBSD_version >= 1100005 426 struct netmap_adapter * 427 netmap_getna(if_t ifp) 428 { 429 return (NA((struct ifnet *)ifp)); 430 } 431 #endif /* __FreeBSD_version >= 1100005 */ 432 433 /* 434 * The following two functions are empty until we have a generic 435 * way to extract the info from the ifp 436 */ 437 int 438 nm_os_generic_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) 439 { 440 return 0; 441 } 442 443 444 void 445 nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) 446 { 447 unsigned num_rings = netmap_generic_rings ? netmap_generic_rings : 1; 448 449 *txq = num_rings; 450 *rxq = num_rings; 451 } 452 453 void 454 nm_os_generic_set_features(struct netmap_generic_adapter *gna) 455 { 456 457 gna->rxsg = 1; /* Supported through m_copydata. */ 458 gna->txqdisc = 0; /* Not supported. */ 459 } 460 461 void 462 nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, struct netmap_adapter *na) 463 { 464 mit->mit_pending = 0; 465 mit->mit_ring_idx = idx; 466 mit->mit_na = na; 467 } 468 469 470 void 471 nm_os_mitigation_start(struct nm_generic_mit *mit) 472 { 473 } 474 475 476 void 477 nm_os_mitigation_restart(struct nm_generic_mit *mit) 478 { 479 } 480 481 482 int 483 nm_os_mitigation_active(struct nm_generic_mit *mit) 484 { 485 486 return 0; 487 } 488 489 490 void 491 nm_os_mitigation_cleanup(struct nm_generic_mit *mit) 492 { 493 } 494 495 static int 496 nm_vi_dummy(struct ifnet *ifp, u_long cmd, caddr_t addr) 497 { 498 499 return EINVAL; 500 } 501 502 static void 503 nm_vi_start(struct ifnet *ifp) 504 { 505 panic("nm_vi_start() must not be called"); 506 } 507 508 /* 509 * Index manager of persistent virtual interfaces. 510 * It is used to decide the lowest byte of the MAC address. 511 * We use the same algorithm with management of bridge port index. 512 */ 513 #define NM_VI_MAX 255 514 static struct { 515 uint8_t index[NM_VI_MAX]; /* XXX just for a reasonable number */ 516 uint8_t active; 517 struct mtx lock; 518 } nm_vi_indices; 519 520 void 521 nm_os_vi_init_index(void) 522 { 523 int i; 524 for (i = 0; i < NM_VI_MAX; i++) 525 nm_vi_indices.index[i] = i; 526 nm_vi_indices.active = 0; 527 mtx_init(&nm_vi_indices.lock, "nm_vi_indices_lock", NULL, MTX_DEF); 528 } 529 530 /* return -1 if no index available */ 531 static int 532 nm_vi_get_index(void) 533 { 534 int ret; 535 536 mtx_lock(&nm_vi_indices.lock); 537 ret = nm_vi_indices.active == NM_VI_MAX ? -1 : 538 nm_vi_indices.index[nm_vi_indices.active++]; 539 mtx_unlock(&nm_vi_indices.lock); 540 return ret; 541 } 542 543 static void 544 nm_vi_free_index(uint8_t val) 545 { 546 int i, lim; 547 548 mtx_lock(&nm_vi_indices.lock); 549 lim = nm_vi_indices.active; 550 for (i = 0; i < lim; i++) { 551 if (nm_vi_indices.index[i] == val) { 552 /* swap index[lim-1] and j */ 553 int tmp = nm_vi_indices.index[lim-1]; 554 nm_vi_indices.index[lim-1] = val; 555 nm_vi_indices.index[i] = tmp; 556 nm_vi_indices.active--; 557 break; 558 } 559 } 560 if (lim == nm_vi_indices.active) 561 nm_prerr("Index %u not found", val); 562 mtx_unlock(&nm_vi_indices.lock); 563 } 564 #undef NM_VI_MAX 565 566 /* 567 * Implementation of a netmap-capable virtual interface that 568 * registered to the system. 569 * It is based on if_tap.c and ip_fw_log.c in FreeBSD 9. 570 * 571 * Note: Linux sets refcount to 0 on allocation of net_device, 572 * then increments it on registration to the system. 573 * FreeBSD sets refcount to 1 on if_alloc(), and does not 574 * increment this refcount on if_attach(). 575 */ 576 int 577 nm_os_vi_persist(const char *name, struct ifnet **ret) 578 { 579 struct ifnet *ifp; 580 u_short macaddr_hi; 581 uint32_t macaddr_mid; 582 u_char eaddr[6]; 583 int unit = nm_vi_get_index(); /* just to decide MAC address */ 584 585 if (unit < 0) 586 return EBUSY; 587 /* 588 * We use the same MAC address generation method with tap 589 * except for the highest octet is 00:be instead of 00:bd 590 */ 591 macaddr_hi = htons(0x00be); /* XXX tap + 1 */ 592 macaddr_mid = (uint32_t) ticks; 593 bcopy(&macaddr_hi, eaddr, sizeof(short)); 594 bcopy(&macaddr_mid, &eaddr[2], sizeof(uint32_t)); 595 eaddr[5] = (uint8_t)unit; 596 597 ifp = if_alloc(IFT_ETHER); 598 if (ifp == NULL) { 599 nm_prerr("if_alloc failed"); 600 return ENOMEM; 601 } 602 if_initname(ifp, name, IF_DUNIT_NONE); 603 ifp->if_mtu = 65536; 604 ifp->if_flags = IFF_UP | IFF_SIMPLEX | IFF_MULTICAST; 605 ifp->if_init = (void *)nm_vi_dummy; 606 ifp->if_ioctl = nm_vi_dummy; 607 ifp->if_start = nm_vi_start; 608 ifp->if_mtu = ETHERMTU; 609 IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen); 610 ifp->if_capabilities |= IFCAP_LINKSTATE; 611 ifp->if_capenable |= IFCAP_LINKSTATE; 612 613 ether_ifattach(ifp, eaddr); 614 *ret = ifp; 615 return 0; 616 } 617 618 /* unregister from the system and drop the final refcount */ 619 void 620 nm_os_vi_detach(struct ifnet *ifp) 621 { 622 nm_vi_free_index(((char *)IF_LLADDR(ifp))[5]); 623 ether_ifdetach(ifp); 624 if_free(ifp); 625 } 626 627 #ifdef WITH_EXTMEM 628 #include <vm/vm_map.h> 629 #include <vm/vm_kern.h> 630 struct nm_os_extmem { 631 vm_object_t obj; 632 vm_offset_t kva; 633 vm_offset_t size; 634 uintptr_t scan; 635 }; 636 637 void 638 nm_os_extmem_delete(struct nm_os_extmem *e) 639 { 640 nm_prinf("freeing %zx bytes", (size_t)e->size); 641 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 642 nm_os_free(e); 643 } 644 645 char * 646 nm_os_extmem_nextpage(struct nm_os_extmem *e) 647 { 648 char *rv = NULL; 649 if (e->scan < e->kva + e->size) { 650 rv = (char *)e->scan; 651 e->scan += PAGE_SIZE; 652 } 653 return rv; 654 } 655 656 int 657 nm_os_extmem_isequal(struct nm_os_extmem *e1, struct nm_os_extmem *e2) 658 { 659 return (e1->obj == e2->obj); 660 } 661 662 int 663 nm_os_extmem_nr_pages(struct nm_os_extmem *e) 664 { 665 return e->size >> PAGE_SHIFT; 666 } 667 668 struct nm_os_extmem * 669 nm_os_extmem_create(unsigned long p, struct nmreq_pools_info *pi, int *perror) 670 { 671 vm_map_t map; 672 vm_map_entry_t entry; 673 vm_object_t obj; 674 vm_prot_t prot; 675 vm_pindex_t index; 676 boolean_t wired; 677 struct nm_os_extmem *e = NULL; 678 int rv, error = 0; 679 680 e = nm_os_malloc(sizeof(*e)); 681 if (e == NULL) { 682 error = ENOMEM; 683 goto out; 684 } 685 686 map = &curthread->td_proc->p_vmspace->vm_map; 687 rv = vm_map_lookup(&map, p, VM_PROT_RW, &entry, 688 &obj, &index, &prot, &wired); 689 if (rv != KERN_SUCCESS) { 690 nm_prerr("address %lx not found", p); 691 goto out_free; 692 } 693 /* check that we are given the whole vm_object ? */ 694 vm_map_lookup_done(map, entry); 695 696 // XXX can we really use obj after releasing the map lock? 697 e->obj = obj; 698 vm_object_reference(obj); 699 /* wire the memory and add the vm_object to the kernel map, 700 * to make sure that it is not fred even if the processes that 701 * are mmap()ing it all exit 702 */ 703 e->kva = vm_map_min(kernel_map); 704 e->size = obj->size << PAGE_SHIFT; 705 rv = vm_map_find(kernel_map, obj, 0, &e->kva, e->size, 0, 706 VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, 707 VM_PROT_READ | VM_PROT_WRITE, 0); 708 if (rv != KERN_SUCCESS) { 709 nm_prerr("vm_map_find(%zx) failed", (size_t)e->size); 710 goto out_rel; 711 } 712 rv = vm_map_wire(kernel_map, e->kva, e->kva + e->size, 713 VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); 714 if (rv != KERN_SUCCESS) { 715 nm_prerr("vm_map_wire failed"); 716 goto out_rem; 717 } 718 719 e->scan = e->kva; 720 721 return e; 722 723 out_rem: 724 vm_map_remove(kernel_map, e->kva, e->kva + e->size); 725 e->obj = NULL; 726 out_rel: 727 vm_object_deallocate(e->obj); 728 out_free: 729 nm_os_free(e); 730 out: 731 if (perror) 732 *perror = error; 733 return NULL; 734 } 735 #endif /* WITH_EXTMEM */ 736 737 /* ================== PTNETMAP GUEST SUPPORT ==================== */ 738 739 #ifdef WITH_PTNETMAP 740 #include <sys/bus.h> 741 #include <sys/rman.h> 742 #include <machine/bus.h> /* bus_dmamap_* */ 743 #include <machine/resource.h> 744 #include <dev/pci/pcivar.h> 745 #include <dev/pci/pcireg.h> 746 /* 747 * ptnetmap memory device (memdev) for freebsd guest, 748 * ssed to expose host netmap memory to the guest through a PCI BAR. 749 */ 750 751 /* 752 * ptnetmap memdev private data structure 753 */ 754 struct ptnetmap_memdev { 755 device_t dev; 756 struct resource *pci_io; 757 struct resource *pci_mem; 758 struct netmap_mem_d *nm_mem; 759 }; 760 761 static int ptn_memdev_probe(device_t); 762 static int ptn_memdev_attach(device_t); 763 static int ptn_memdev_detach(device_t); 764 static int ptn_memdev_shutdown(device_t); 765 766 static device_method_t ptn_memdev_methods[] = { 767 DEVMETHOD(device_probe, ptn_memdev_probe), 768 DEVMETHOD(device_attach, ptn_memdev_attach), 769 DEVMETHOD(device_detach, ptn_memdev_detach), 770 DEVMETHOD(device_shutdown, ptn_memdev_shutdown), 771 DEVMETHOD_END 772 }; 773 774 static driver_t ptn_memdev_driver = { 775 PTNETMAP_MEMDEV_NAME, 776 ptn_memdev_methods, 777 sizeof(struct ptnetmap_memdev), 778 }; 779 780 /* We use (SI_ORDER_MIDDLE+1) here, see DEV_MODULE_ORDERED() invocation 781 * below. */ 782 static devclass_t ptnetmap_devclass; 783 DRIVER_MODULE_ORDERED(ptn_memdev, pci, ptn_memdev_driver, ptnetmap_devclass, 784 NULL, NULL, SI_ORDER_MIDDLE + 1); 785 786 /* 787 * Map host netmap memory through PCI-BAR in the guest OS, 788 * returning physical (nm_paddr) and virtual (nm_addr) addresses 789 * of the netmap memory mapped in the guest. 790 */ 791 int 792 nm_os_pt_memdev_iomap(struct ptnetmap_memdev *ptn_dev, vm_paddr_t *nm_paddr, 793 void **nm_addr, uint64_t *mem_size) 794 { 795 int rid; 796 797 nm_prinf("ptn_memdev_driver iomap"); 798 799 rid = PCIR_BAR(PTNETMAP_MEM_PCI_BAR); 800 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_HI); 801 *mem_size = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMSIZE_LO) | 802 (*mem_size << 32); 803 804 /* map memory allocator */ 805 ptn_dev->pci_mem = bus_alloc_resource(ptn_dev->dev, SYS_RES_MEMORY, 806 &rid, 0, ~0, *mem_size, RF_ACTIVE); 807 if (ptn_dev->pci_mem == NULL) { 808 *nm_paddr = 0; 809 *nm_addr = NULL; 810 return ENOMEM; 811 } 812 813 *nm_paddr = rman_get_start(ptn_dev->pci_mem); 814 *nm_addr = rman_get_virtual(ptn_dev->pci_mem); 815 816 nm_prinf("=== BAR %d start %lx len %lx mem_size %lx ===", 817 PTNETMAP_MEM_PCI_BAR, 818 (unsigned long)(*nm_paddr), 819 (unsigned long)rman_get_size(ptn_dev->pci_mem), 820 (unsigned long)*mem_size); 821 return (0); 822 } 823 824 uint32_t 825 nm_os_pt_memdev_ioread(struct ptnetmap_memdev *ptn_dev, unsigned int reg) 826 { 827 return bus_read_4(ptn_dev->pci_io, reg); 828 } 829 830 /* Unmap host netmap memory. */ 831 void 832 nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *ptn_dev) 833 { 834 nm_prinf("ptn_memdev_driver iounmap"); 835 836 if (ptn_dev->pci_mem) { 837 bus_release_resource(ptn_dev->dev, SYS_RES_MEMORY, 838 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 839 ptn_dev->pci_mem = NULL; 840 } 841 } 842 843 /* Device identification routine, return BUS_PROBE_DEFAULT on success, 844 * positive on failure */ 845 static int 846 ptn_memdev_probe(device_t dev) 847 { 848 char desc[256]; 849 850 if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID) 851 return (ENXIO); 852 if (pci_get_device(dev) != PTNETMAP_PCI_DEVICE_ID) 853 return (ENXIO); 854 855 snprintf(desc, sizeof(desc), "%s PCI adapter", 856 PTNETMAP_MEMDEV_NAME); 857 device_set_desc_copy(dev, desc); 858 859 return (BUS_PROBE_DEFAULT); 860 } 861 862 /* Device initialization routine. */ 863 static int 864 ptn_memdev_attach(device_t dev) 865 { 866 struct ptnetmap_memdev *ptn_dev; 867 int rid; 868 uint16_t mem_id; 869 870 ptn_dev = device_get_softc(dev); 871 ptn_dev->dev = dev; 872 873 pci_enable_busmaster(dev); 874 875 rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR); 876 ptn_dev->pci_io = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 877 RF_ACTIVE); 878 if (ptn_dev->pci_io == NULL) { 879 device_printf(dev, "cannot map I/O space\n"); 880 return (ENXIO); 881 } 882 883 mem_id = bus_read_4(ptn_dev->pci_io, PTNET_MDEV_IO_MEMID); 884 885 /* create guest allocator */ 886 ptn_dev->nm_mem = netmap_mem_pt_guest_attach(ptn_dev, mem_id); 887 if (ptn_dev->nm_mem == NULL) { 888 ptn_memdev_detach(dev); 889 return (ENOMEM); 890 } 891 netmap_mem_get(ptn_dev->nm_mem); 892 893 nm_prinf("ptnetmap memdev attached, host memid: %u", mem_id); 894 895 return (0); 896 } 897 898 /* Device removal routine. */ 899 static int 900 ptn_memdev_detach(device_t dev) 901 { 902 struct ptnetmap_memdev *ptn_dev; 903 904 ptn_dev = device_get_softc(dev); 905 906 if (ptn_dev->nm_mem) { 907 nm_prinf("ptnetmap memdev detached, host memid %u", 908 netmap_mem_get_id(ptn_dev->nm_mem)); 909 netmap_mem_put(ptn_dev->nm_mem); 910 ptn_dev->nm_mem = NULL; 911 } 912 if (ptn_dev->pci_mem) { 913 bus_release_resource(dev, SYS_RES_MEMORY, 914 PCIR_BAR(PTNETMAP_MEM_PCI_BAR), ptn_dev->pci_mem); 915 ptn_dev->pci_mem = NULL; 916 } 917 if (ptn_dev->pci_io) { 918 bus_release_resource(dev, SYS_RES_IOPORT, 919 PCIR_BAR(PTNETMAP_IO_PCI_BAR), ptn_dev->pci_io); 920 ptn_dev->pci_io = NULL; 921 } 922 923 return (0); 924 } 925 926 static int 927 ptn_memdev_shutdown(device_t dev) 928 { 929 return bus_generic_shutdown(dev); 930 } 931 932 #endif /* WITH_PTNETMAP */ 933 934 /* 935 * In order to track whether pages are still mapped, we hook into 936 * the standard cdev_pager and intercept the constructor and 937 * destructor. 938 */ 939 940 struct netmap_vm_handle_t { 941 struct cdev *dev; 942 struct netmap_priv_d *priv; 943 }; 944 945 946 static int 947 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 948 vm_ooffset_t foff, struct ucred *cred, u_short *color) 949 { 950 struct netmap_vm_handle_t *vmh = handle; 951 952 if (netmap_verbose) 953 nm_prinf("handle %p size %jd prot %d foff %jd", 954 handle, (intmax_t)size, prot, (intmax_t)foff); 955 if (color) 956 *color = 0; 957 dev_ref(vmh->dev); 958 return 0; 959 } 960 961 962 static void 963 netmap_dev_pager_dtor(void *handle) 964 { 965 struct netmap_vm_handle_t *vmh = handle; 966 struct cdev *dev = vmh->dev; 967 struct netmap_priv_d *priv = vmh->priv; 968 969 if (netmap_verbose) 970 nm_prinf("handle %p", handle); 971 netmap_dtor(priv); 972 free(vmh, M_DEVBUF); 973 dev_rel(dev); 974 } 975 976 977 static int 978 netmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 979 int prot, vm_page_t *mres) 980 { 981 struct netmap_vm_handle_t *vmh = object->handle; 982 struct netmap_priv_d *priv = vmh->priv; 983 struct netmap_adapter *na = priv->np_na; 984 vm_paddr_t paddr; 985 vm_page_t page; 986 vm_memattr_t memattr; 987 vm_pindex_t pidx; 988 989 nm_prdis("object %p offset %jd prot %d mres %p", 990 object, (intmax_t)offset, prot, mres); 991 memattr = object->memattr; 992 pidx = OFF_TO_IDX(offset); 993 paddr = netmap_mem_ofstophys(na->nm_mem, offset); 994 if (paddr == 0) 995 return VM_PAGER_FAIL; 996 997 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 998 /* 999 * If the passed in result page is a fake page, update it with 1000 * the new physical address. 1001 */ 1002 page = *mres; 1003 vm_page_updatefake(page, paddr, memattr); 1004 } else { 1005 /* 1006 * Replace the passed in reqpage page with our own fake page and 1007 * free up the all of the original pages. 1008 */ 1009 #ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ 1010 #define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK 1011 #define VM_OBJECT_WLOCK VM_OBJECT_LOCK 1012 #endif /* VM_OBJECT_WUNLOCK */ 1013 1014 VM_OBJECT_WUNLOCK(object); 1015 page = vm_page_getfake(paddr, memattr); 1016 VM_OBJECT_WLOCK(object); 1017 vm_page_lock(*mres); 1018 vm_page_free(*mres); 1019 vm_page_unlock(*mres); 1020 *mres = page; 1021 vm_page_insert(page, object, pidx); 1022 } 1023 page->valid = VM_PAGE_BITS_ALL; 1024 return (VM_PAGER_OK); 1025 } 1026 1027 1028 static struct cdev_pager_ops netmap_cdev_pager_ops = { 1029 .cdev_pg_ctor = netmap_dev_pager_ctor, 1030 .cdev_pg_dtor = netmap_dev_pager_dtor, 1031 .cdev_pg_fault = netmap_dev_pager_fault, 1032 }; 1033 1034 1035 static int 1036 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 1037 vm_size_t objsize, vm_object_t *objp, int prot) 1038 { 1039 int error; 1040 struct netmap_vm_handle_t *vmh; 1041 struct netmap_priv_d *priv; 1042 vm_object_t obj; 1043 1044 if (netmap_verbose) 1045 nm_prinf("cdev %p foff %jd size %jd objp %p prot %d", cdev, 1046 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 1047 1048 vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 1049 M_NOWAIT | M_ZERO); 1050 if (vmh == NULL) 1051 return ENOMEM; 1052 vmh->dev = cdev; 1053 1054 NMG_LOCK(); 1055 error = devfs_get_cdevpriv((void**)&priv); 1056 if (error) 1057 goto err_unlock; 1058 if (priv->np_nifp == NULL) { 1059 error = EINVAL; 1060 goto err_unlock; 1061 } 1062 vmh->priv = priv; 1063 priv->np_refs++; 1064 NMG_UNLOCK(); 1065 1066 obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 1067 &netmap_cdev_pager_ops, objsize, prot, 1068 *foff, NULL); 1069 if (obj == NULL) { 1070 nm_prerr("cdev_pager_allocate failed"); 1071 error = EINVAL; 1072 goto err_deref; 1073 } 1074 1075 *objp = obj; 1076 return 0; 1077 1078 err_deref: 1079 NMG_LOCK(); 1080 priv->np_refs--; 1081 err_unlock: 1082 NMG_UNLOCK(); 1083 // err: 1084 free(vmh, M_DEVBUF); 1085 return error; 1086 } 1087 1088 /* 1089 * On FreeBSD the close routine is only called on the last close on 1090 * the device (/dev/netmap) so we cannot do anything useful. 1091 * To track close() on individual file descriptors we pass netmap_dtor() to 1092 * devfs_set_cdevpriv() on open(). The FreeBSD kernel will call the destructor 1093 * when the last fd pointing to the device is closed. 1094 * 1095 * Note that FreeBSD does not even munmap() on close() so we also have 1096 * to track mmap() ourselves, and postpone the call to 1097 * netmap_dtor() is called when the process has no open fds and no active 1098 * memory maps on /dev/netmap, as in linux. 1099 */ 1100 static int 1101 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1102 { 1103 if (netmap_verbose) 1104 nm_prinf("dev %p fflag 0x%x devtype %d td %p", 1105 dev, fflag, devtype, td); 1106 return 0; 1107 } 1108 1109 1110 static int 1111 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1112 { 1113 struct netmap_priv_d *priv; 1114 int error; 1115 1116 (void)dev; 1117 (void)oflags; 1118 (void)devtype; 1119 (void)td; 1120 1121 NMG_LOCK(); 1122 priv = netmap_priv_new(); 1123 if (priv == NULL) { 1124 error = ENOMEM; 1125 goto out; 1126 } 1127 error = devfs_set_cdevpriv(priv, netmap_dtor); 1128 if (error) { 1129 netmap_priv_delete(priv); 1130 } 1131 out: 1132 NMG_UNLOCK(); 1133 return error; 1134 } 1135 1136 /******************** kthread wrapper ****************/ 1137 #include <sys/sysproto.h> 1138 u_int 1139 nm_os_ncpus(void) 1140 { 1141 return mp_maxid + 1; 1142 } 1143 1144 struct nm_kctx_ctx { 1145 /* Userspace thread (kthread creator). */ 1146 struct thread *user_td; 1147 1148 /* worker function and parameter */ 1149 nm_kctx_worker_fn_t worker_fn; 1150 void *worker_private; 1151 1152 struct nm_kctx *nmk; 1153 1154 /* integer to manage multiple worker contexts (e.g., RX or TX on ptnetmap) */ 1155 long type; 1156 }; 1157 1158 struct nm_kctx { 1159 struct thread *worker; 1160 struct mtx worker_lock; 1161 struct nm_kctx_ctx worker_ctx; 1162 int run; /* used to stop kthread */ 1163 int attach_user; /* kthread attached to user_process */ 1164 int affinity; 1165 }; 1166 1167 static void 1168 nm_kctx_worker(void *data) 1169 { 1170 struct nm_kctx *nmk = data; 1171 struct nm_kctx_ctx *ctx = &nmk->worker_ctx; 1172 1173 if (nmk->affinity >= 0) { 1174 thread_lock(curthread); 1175 sched_bind(curthread, nmk->affinity); 1176 thread_unlock(curthread); 1177 } 1178 1179 while (nmk->run) { 1180 /* 1181 * check if the parent process dies 1182 * (when kthread is attached to user process) 1183 */ 1184 if (ctx->user_td) { 1185 PROC_LOCK(curproc); 1186 thread_suspend_check(0); 1187 PROC_UNLOCK(curproc); 1188 } else { 1189 kthread_suspend_check(); 1190 } 1191 1192 /* Continuously execute worker process. */ 1193 ctx->worker_fn(ctx->worker_private); /* worker body */ 1194 } 1195 1196 kthread_exit(); 1197 } 1198 1199 void 1200 nm_os_kctx_worker_setaff(struct nm_kctx *nmk, int affinity) 1201 { 1202 nmk->affinity = affinity; 1203 } 1204 1205 struct nm_kctx * 1206 nm_os_kctx_create(struct nm_kctx_cfg *cfg, void *opaque) 1207 { 1208 struct nm_kctx *nmk = NULL; 1209 1210 nmk = malloc(sizeof(*nmk), M_DEVBUF, M_NOWAIT | M_ZERO); 1211 if (!nmk) 1212 return NULL; 1213 1214 mtx_init(&nmk->worker_lock, "nm_kthread lock", NULL, MTX_DEF); 1215 nmk->worker_ctx.worker_fn = cfg->worker_fn; 1216 nmk->worker_ctx.worker_private = cfg->worker_private; 1217 nmk->worker_ctx.type = cfg->type; 1218 nmk->affinity = -1; 1219 1220 /* attach kthread to user process (ptnetmap) */ 1221 nmk->attach_user = cfg->attach_user; 1222 1223 return nmk; 1224 } 1225 1226 int 1227 nm_os_kctx_worker_start(struct nm_kctx *nmk) 1228 { 1229 struct proc *p = NULL; 1230 int error = 0; 1231 1232 /* Temporarily disable this function as it is currently broken 1233 * and causes kernel crashes. The failure can be triggered by 1234 * the "vale_polling_enable_disable" test in ctrl-api-test.c. */ 1235 return EOPNOTSUPP; 1236 1237 if (nmk->worker) 1238 return EBUSY; 1239 1240 /* check if we want to attach kthread to user process */ 1241 if (nmk->attach_user) { 1242 nmk->worker_ctx.user_td = curthread; 1243 p = curthread->td_proc; 1244 } 1245 1246 /* enable kthread main loop */ 1247 nmk->run = 1; 1248 /* create kthread */ 1249 if((error = kthread_add(nm_kctx_worker, nmk, p, 1250 &nmk->worker, RFNOWAIT /* to be checked */, 0, "nm-kthread-%ld", 1251 nmk->worker_ctx.type))) { 1252 goto err; 1253 } 1254 1255 nm_prinf("nm_kthread started td %p", nmk->worker); 1256 1257 return 0; 1258 err: 1259 nm_prerr("nm_kthread start failed err %d", error); 1260 nmk->worker = NULL; 1261 return error; 1262 } 1263 1264 void 1265 nm_os_kctx_worker_stop(struct nm_kctx *nmk) 1266 { 1267 if (!nmk->worker) 1268 return; 1269 1270 /* tell to kthread to exit from main loop */ 1271 nmk->run = 0; 1272 1273 /* wake up kthread if it sleeps */ 1274 kthread_resume(nmk->worker); 1275 1276 nmk->worker = NULL; 1277 } 1278 1279 void 1280 nm_os_kctx_destroy(struct nm_kctx *nmk) 1281 { 1282 if (!nmk) 1283 return; 1284 1285 if (nmk->worker) 1286 nm_os_kctx_worker_stop(nmk); 1287 1288 free(nmk, M_DEVBUF); 1289 } 1290 1291 /******************** kqueue support ****************/ 1292 1293 /* 1294 * In addition to calling selwakeuppri(), nm_os_selwakeup() also 1295 * needs to call KNOTE to wake up kqueue listeners. 1296 * We use a non-zero 'hint' argument to inform the netmap_knrw() 1297 * function that it is being called from 'nm_os_selwakeup'; this 1298 * is necessary because when netmap_knrw() is called by the kevent 1299 * subsystem (i.e. kevent_scan()) we also need to call netmap_poll(). 1300 * The knote uses a private mutex associated to the 'si' (see struct 1301 * selinfo, struct nm_selinfo, and nm_os_selinfo_init). 1302 * 1303 * The netmap_kqfilter() function registers one or another f_event 1304 * depending on read or write mode. A pointer to the struct 1305 * 'netmap_priv_d' is stored into kn->kn_hook, so that it can later 1306 * be passed to netmap_poll(). We pass NULL as a third argument to 1307 * netmap_poll(), so that the latter only runs the txsync/rxsync 1308 * (if necessary), and skips the nm_os_selrecord() calls. 1309 */ 1310 1311 1312 void 1313 nm_os_selwakeup(struct nm_selinfo *si) 1314 { 1315 if (netmap_verbose) 1316 nm_prinf("on knote %p", &si->si.si_note); 1317 selwakeuppri(&si->si, PI_NET); 1318 /* We use a non-zero hint to distinguish this notification call 1319 * from the call done in kqueue_scan(), which uses hint=0. 1320 */ 1321 KNOTE(&si->si.si_note, /*hint=*/0x100, 1322 mtx_owned(&si->m) ? KNF_LISTLOCKED : 0); 1323 } 1324 1325 void 1326 nm_os_selrecord(struct thread *td, struct nm_selinfo *si) 1327 { 1328 selrecord(td, &si->si); 1329 } 1330 1331 static void 1332 netmap_knrdetach(struct knote *kn) 1333 { 1334 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1335 struct selinfo *si = &priv->np_si[NR_RX]->si; 1336 1337 nm_prinf("remove selinfo %p", si); 1338 knlist_remove(&si->si_note, kn, /*islocked=*/0); 1339 } 1340 1341 static void 1342 netmap_knwdetach(struct knote *kn) 1343 { 1344 struct netmap_priv_d *priv = (struct netmap_priv_d *)kn->kn_hook; 1345 struct selinfo *si = &priv->np_si[NR_TX]->si; 1346 1347 nm_prinf("remove selinfo %p", si); 1348 knlist_remove(&si->si_note, kn, /*islocked=*/0); 1349 } 1350 1351 /* 1352 * Callback triggered by netmap notifications (see netmap_notify()), 1353 * and by the application calling kevent(). In the former case we 1354 * just return 1 (events ready), since we are not able to do better. 1355 * In the latter case we use netmap_poll() to see which events are 1356 * ready. 1357 */ 1358 static int 1359 netmap_knrw(struct knote *kn, long hint, int events) 1360 { 1361 struct netmap_priv_d *priv; 1362 int revents; 1363 1364 if (hint != 0) { 1365 /* Called from netmap_notify(), typically from a 1366 * thread different from the one issuing kevent(). 1367 * Assume we are ready. */ 1368 return 1; 1369 } 1370 1371 /* Called from kevent(). */ 1372 priv = kn->kn_hook; 1373 revents = netmap_poll(priv, events, /*thread=*/NULL); 1374 1375 return (events & revents) ? 1 : 0; 1376 } 1377 1378 static int 1379 netmap_knread(struct knote *kn, long hint) 1380 { 1381 return netmap_knrw(kn, hint, POLLIN); 1382 } 1383 1384 static int 1385 netmap_knwrite(struct knote *kn, long hint) 1386 { 1387 return netmap_knrw(kn, hint, POLLOUT); 1388 } 1389 1390 static struct filterops netmap_rfiltops = { 1391 .f_isfd = 1, 1392 .f_detach = netmap_knrdetach, 1393 .f_event = netmap_knread, 1394 }; 1395 1396 static struct filterops netmap_wfiltops = { 1397 .f_isfd = 1, 1398 .f_detach = netmap_knwdetach, 1399 .f_event = netmap_knwrite, 1400 }; 1401 1402 1403 /* 1404 * This is called when a thread invokes kevent() to record 1405 * a change in the configuration of the kqueue(). 1406 * The 'priv' is the one associated to the open netmap device. 1407 */ 1408 static int 1409 netmap_kqfilter(struct cdev *dev, struct knote *kn) 1410 { 1411 struct netmap_priv_d *priv; 1412 int error; 1413 struct netmap_adapter *na; 1414 struct nm_selinfo *si; 1415 int ev = kn->kn_filter; 1416 1417 if (ev != EVFILT_READ && ev != EVFILT_WRITE) { 1418 nm_prerr("bad filter request %d", ev); 1419 return 1; 1420 } 1421 error = devfs_get_cdevpriv((void**)&priv); 1422 if (error) { 1423 nm_prerr("device not yet setup"); 1424 return 1; 1425 } 1426 na = priv->np_na; 1427 if (na == NULL) { 1428 nm_prerr("no netmap adapter for this file descriptor"); 1429 return 1; 1430 } 1431 /* the si is indicated in the priv */ 1432 si = priv->np_si[(ev == EVFILT_WRITE) ? NR_TX : NR_RX]; 1433 kn->kn_fop = (ev == EVFILT_WRITE) ? 1434 &netmap_wfiltops : &netmap_rfiltops; 1435 kn->kn_hook = priv; 1436 knlist_add(&si->si.si_note, kn, /*islocked=*/0); 1437 1438 return 0; 1439 } 1440 1441 static int 1442 freebsd_netmap_poll(struct cdev *cdevi __unused, int events, struct thread *td) 1443 { 1444 struct netmap_priv_d *priv; 1445 if (devfs_get_cdevpriv((void **)&priv)) { 1446 return POLLERR; 1447 } 1448 return netmap_poll(priv, events, td); 1449 } 1450 1451 static int 1452 freebsd_netmap_ioctl(struct cdev *dev __unused, u_long cmd, caddr_t data, 1453 int ffla __unused, struct thread *td) 1454 { 1455 int error; 1456 struct netmap_priv_d *priv; 1457 1458 CURVNET_SET(TD_TO_VNET(td)); 1459 error = devfs_get_cdevpriv((void **)&priv); 1460 if (error) { 1461 /* XXX ENOENT should be impossible, since the priv 1462 * is now created in the open */ 1463 if (error == ENOENT) 1464 error = ENXIO; 1465 goto out; 1466 } 1467 error = netmap_ioctl(priv, cmd, data, td, /*nr_body_is_user=*/1); 1468 out: 1469 CURVNET_RESTORE(); 1470 1471 return error; 1472 } 1473 1474 void 1475 nm_os_onattach(struct ifnet *ifp) 1476 { 1477 ifp->if_capabilities |= IFCAP_NETMAP; 1478 } 1479 1480 void 1481 nm_os_onenter(struct ifnet *ifp) 1482 { 1483 struct netmap_adapter *na = NA(ifp); 1484 1485 na->if_transmit = ifp->if_transmit; 1486 ifp->if_transmit = netmap_transmit; 1487 ifp->if_capenable |= IFCAP_NETMAP; 1488 } 1489 1490 void 1491 nm_os_onexit(struct ifnet *ifp) 1492 { 1493 struct netmap_adapter *na = NA(ifp); 1494 1495 ifp->if_transmit = na->if_transmit; 1496 ifp->if_capenable &= ~IFCAP_NETMAP; 1497 } 1498 1499 extern struct cdevsw netmap_cdevsw; /* XXX used in netmap.c, should go elsewhere */ 1500 struct cdevsw netmap_cdevsw = { 1501 .d_version = D_VERSION, 1502 .d_name = "netmap", 1503 .d_open = netmap_open, 1504 .d_mmap_single = netmap_mmap_single, 1505 .d_ioctl = freebsd_netmap_ioctl, 1506 .d_poll = freebsd_netmap_poll, 1507 .d_kqfilter = netmap_kqfilter, 1508 .d_close = netmap_close, 1509 }; 1510 /*--- end of kqueue support ----*/ 1511 1512 /* 1513 * Kernel entry point. 1514 * 1515 * Initialize/finalize the module and return. 1516 * 1517 * Return 0 on success, errno on failure. 1518 */ 1519 static int 1520 netmap_loader(__unused struct module *module, int event, __unused void *arg) 1521 { 1522 int error = 0; 1523 1524 switch (event) { 1525 case MOD_LOAD: 1526 error = netmap_init(); 1527 break; 1528 1529 case MOD_UNLOAD: 1530 /* 1531 * if some one is still using netmap, 1532 * then the module can not be unloaded. 1533 */ 1534 if (netmap_use_count) { 1535 nm_prerr("netmap module can not be unloaded - netmap_use_count: %d", 1536 netmap_use_count); 1537 error = EBUSY; 1538 break; 1539 } 1540 netmap_fini(); 1541 break; 1542 1543 default: 1544 error = EOPNOTSUPP; 1545 break; 1546 } 1547 1548 return (error); 1549 } 1550 1551 #ifdef DEV_MODULE_ORDERED 1552 /* 1553 * The netmap module contains three drivers: (i) the netmap character device 1554 * driver; (ii) the ptnetmap memdev PCI device driver, (iii) the ptnet PCI 1555 * device driver. The attach() routines of both (ii) and (iii) need the 1556 * lock of the global allocator, and such lock is initialized in netmap_init(), 1557 * which is part of (i). 1558 * Therefore, we make sure that (i) is loaded before (ii) and (iii), using 1559 * the 'order' parameter of driver declaration macros. For (i), we specify 1560 * SI_ORDER_MIDDLE, while higher orders are used with the DRIVER_MODULE_ORDERED 1561 * macros for (ii) and (iii). 1562 */ 1563 DEV_MODULE_ORDERED(netmap, netmap_loader, NULL, SI_ORDER_MIDDLE); 1564 #else /* !DEV_MODULE_ORDERED */ 1565 DEV_MODULE(netmap, netmap_loader, NULL); 1566 #endif /* DEV_MODULE_ORDERED */ 1567 MODULE_DEPEND(netmap, pci, 1, 1, 1); 1568 MODULE_VERSION(netmap, 1); 1569 /* reduce conditional code */ 1570 // linux API, use for the knlist in FreeBSD 1571 /* use a private mutex for the knlist */ 1572