1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004-2006 Kip Macy 5 * Copyright (c) 2015 Wei Liu <[email protected]> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_inet.h" 32 #include "opt_inet6.h" 33 34 #include <sys/param.h> 35 #include <sys/sockio.h> 36 #include <sys/limits.h> 37 #include <sys/mbuf.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/kernel.h> 41 #include <sys/socket.h> 42 #include <sys/sysctl.h> 43 #include <sys/taskqueue.h> 44 45 #include <net/if.h> 46 #include <net/if_var.h> 47 #include <net/if_arp.h> 48 #include <net/ethernet.h> 49 #include <net/if_media.h> 50 #include <net/bpf.h> 51 #include <net/if_types.h> 52 53 #include <netinet/in.h> 54 #include <netinet/ip.h> 55 #include <netinet/if_ether.h> 56 #include <netinet/tcp.h> 57 #include <netinet/tcp_lro.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 62 #include <sys/bus.h> 63 64 #include <xen/xen-os.h> 65 #include <xen/hypervisor.h> 66 #include <xen/xen_intr.h> 67 #include <xen/gnttab.h> 68 #include <contrib/xen/memory.h> 69 #include <contrib/xen/io/netif.h> 70 #include <xen/xenbus/xenbusvar.h> 71 72 #include <machine/bus.h> 73 74 #include "xenbus_if.h" 75 76 /* Features supported by all backends. TSO and LRO can be negotiated */ 77 #define XN_CSUM_FEATURES (CSUM_TCP | CSUM_UDP) 78 79 #define NET_TX_RING_SIZE __CONST_RING_SIZE(netif_tx, PAGE_SIZE) 80 #define NET_RX_RING_SIZE __CONST_RING_SIZE(netif_rx, PAGE_SIZE) 81 82 #define NET_RX_SLOTS_MIN (XEN_NETIF_NR_SLOTS_MIN + 1) 83 84 /* 85 * Should the driver do LRO on the RX end 86 * this can be toggled on the fly, but the 87 * interface must be reset (down/up) for it 88 * to take effect. 89 */ 90 static int xn_enable_lro = 1; 91 TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro); 92 93 /* 94 * Number of pairs of queues. 95 */ 96 static unsigned long xn_num_queues = 4; 97 TUNABLE_ULONG("hw.xn.num_queues", &xn_num_queues); 98 99 /** 100 * \brief The maximum allowed data fragments in a single transmit 101 * request. 102 * 103 * This limit is imposed by the backend driver. We assume here that 104 * we are dealing with a Linux driver domain and have set our limit 105 * to mirror the Linux MAX_SKB_FRAGS constant. 106 */ 107 #define MAX_TX_REQ_FRAGS (65536 / PAGE_SIZE + 2) 108 109 #define RX_COPY_THRESHOLD 256 110 111 #define net_ratelimit() 0 112 113 struct netfront_rxq; 114 struct netfront_txq; 115 struct netfront_info; 116 struct netfront_rx_info; 117 118 static void xn_txeof(struct netfront_txq *); 119 static void xn_rxeof(struct netfront_rxq *); 120 static void xn_alloc_rx_buffers(struct netfront_rxq *); 121 static void xn_alloc_rx_buffers_callout(void *arg); 122 123 static void xn_release_rx_bufs(struct netfront_rxq *); 124 static void xn_release_tx_bufs(struct netfront_txq *); 125 126 static void xn_rxq_intr(struct netfront_rxq *); 127 static void xn_txq_intr(struct netfront_txq *); 128 static void xn_intr(void *); 129 static int xn_assemble_tx_request(struct netfront_txq *, struct mbuf *); 130 static int xn_ioctl(if_t, u_long, caddr_t); 131 static void xn_ifinit_locked(struct netfront_info *); 132 static void xn_ifinit(void *); 133 static void xn_stop(struct netfront_info *); 134 static void xn_query_features(struct netfront_info *np); 135 static int xn_configure_features(struct netfront_info *np); 136 static void netif_free(struct netfront_info *info); 137 static int netfront_detach(device_t dev); 138 139 static int xn_txq_mq_start_locked(struct netfront_txq *, struct mbuf *); 140 static int xn_txq_mq_start(if_t, struct mbuf *); 141 142 static int talk_to_backend(device_t dev, struct netfront_info *info); 143 static int create_netdev(device_t dev); 144 static void netif_disconnect_backend(struct netfront_info *info); 145 static int setup_device(device_t dev, struct netfront_info *info, 146 unsigned long); 147 static int xn_ifmedia_upd(if_t ifp); 148 static void xn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr); 149 150 static int xn_connect(struct netfront_info *); 151 static void xn_kick_rings(struct netfront_info *); 152 153 static int xn_get_responses(struct netfront_rxq *, 154 struct netfront_rx_info *, RING_IDX, RING_IDX *, 155 struct mbuf **); 156 157 #define virt_to_mfn(x) (vtophys(x) >> PAGE_SHIFT) 158 159 #define INVALID_P2M_ENTRY (~0UL) 160 #define XN_QUEUE_NAME_LEN 8 /* xn{t,r}x_%u, allow for two digits */ 161 struct netfront_rxq { 162 struct netfront_info *info; 163 u_int id; 164 char name[XN_QUEUE_NAME_LEN]; 165 struct mtx lock; 166 167 int ring_ref; 168 netif_rx_front_ring_t ring; 169 xen_intr_handle_t xen_intr_handle; 170 171 grant_ref_t gref_head; 172 grant_ref_t grant_ref[NET_RX_RING_SIZE + 1]; 173 174 struct mbuf *mbufs[NET_RX_RING_SIZE + 1]; 175 176 struct lro_ctrl lro; 177 178 struct callout rx_refill; 179 }; 180 181 struct netfront_txq { 182 struct netfront_info *info; 183 u_int id; 184 char name[XN_QUEUE_NAME_LEN]; 185 struct mtx lock; 186 187 int ring_ref; 188 netif_tx_front_ring_t ring; 189 xen_intr_handle_t xen_intr_handle; 190 191 grant_ref_t gref_head; 192 grant_ref_t grant_ref[NET_TX_RING_SIZE + 1]; 193 194 struct mbuf *mbufs[NET_TX_RING_SIZE + 1]; 195 int mbufs_cnt; 196 struct buf_ring *br; 197 198 struct taskqueue *tq; 199 struct task defrtask; 200 201 bus_dma_segment_t segs[MAX_TX_REQ_FRAGS]; 202 struct mbuf_xennet { 203 struct m_tag tag; 204 bus_dma_tag_t dma_tag; 205 bus_dmamap_t dma_map; 206 struct netfront_txq *txq; 207 SLIST_ENTRY(mbuf_xennet) next; 208 u_int count; 209 } xennet_tag[NET_TX_RING_SIZE + 1]; 210 SLIST_HEAD(, mbuf_xennet) tags; 211 212 bool full; 213 }; 214 215 struct netfront_info { 216 if_t xn_ifp; 217 218 struct mtx sc_lock; 219 220 u_int num_queues; 221 struct netfront_rxq *rxq; 222 struct netfront_txq *txq; 223 224 u_int carrier; 225 u_int maxfrags; 226 227 device_t xbdev; 228 uint8_t mac[ETHER_ADDR_LEN]; 229 230 int xn_if_flags; 231 232 struct ifmedia sc_media; 233 234 bus_dma_tag_t dma_tag; 235 236 bool xn_reset; 237 }; 238 239 struct netfront_rx_info { 240 struct netif_rx_response rx; 241 struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; 242 }; 243 244 #define XN_RX_LOCK(_q) mtx_lock(&(_q)->lock) 245 #define XN_RX_UNLOCK(_q) mtx_unlock(&(_q)->lock) 246 247 #define XN_TX_LOCK(_q) mtx_lock(&(_q)->lock) 248 #define XN_TX_TRYLOCK(_q) mtx_trylock(&(_q)->lock) 249 #define XN_TX_UNLOCK(_q) mtx_unlock(&(_q)->lock) 250 251 #define XN_LOCK(_sc) mtx_lock(&(_sc)->sc_lock); 252 #define XN_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_lock); 253 254 #define XN_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_lock, MA_OWNED); 255 #define XN_RX_LOCK_ASSERT(_q) mtx_assert(&(_q)->lock, MA_OWNED); 256 #define XN_TX_LOCK_ASSERT(_q) mtx_assert(&(_q)->lock, MA_OWNED); 257 258 #define netfront_carrier_on(netif) ((netif)->carrier = 1) 259 #define netfront_carrier_off(netif) ((netif)->carrier = 0) 260 #define netfront_carrier_ok(netif) ((netif)->carrier) 261 262 /* Access macros for acquiring freeing slots in xn_free_{tx,rx}_idxs[]. */ 263 264 static inline void 265 add_id_to_freelist(struct mbuf **list, uintptr_t id) 266 { 267 268 KASSERT(id != 0, 269 ("%s: the head item (0) must always be free.", __func__)); 270 list[id] = list[0]; 271 list[0] = (struct mbuf *)id; 272 } 273 274 static inline unsigned short 275 get_id_from_freelist(struct mbuf **list) 276 { 277 uintptr_t id; 278 279 id = (uintptr_t)list[0]; 280 KASSERT(id != 0, 281 ("%s: the head item (0) must always remain free.", __func__)); 282 list[0] = list[id]; 283 return (id); 284 } 285 286 static inline int 287 xn_rxidx(RING_IDX idx) 288 { 289 290 return idx & (NET_RX_RING_SIZE - 1); 291 } 292 293 static inline struct mbuf * 294 xn_get_rx_mbuf(struct netfront_rxq *rxq, RING_IDX ri) 295 { 296 int i; 297 struct mbuf *m; 298 299 i = xn_rxidx(ri); 300 m = rxq->mbufs[i]; 301 rxq->mbufs[i] = NULL; 302 return (m); 303 } 304 305 static inline grant_ref_t 306 xn_get_rx_ref(struct netfront_rxq *rxq, RING_IDX ri) 307 { 308 int i = xn_rxidx(ri); 309 grant_ref_t ref = rxq->grant_ref[i]; 310 311 KASSERT(ref != GRANT_REF_INVALID, ("Invalid grant reference!\n")); 312 rxq->grant_ref[i] = GRANT_REF_INVALID; 313 return (ref); 314 } 315 316 #define MTAG_COOKIE 1218492000 317 #define MTAG_XENNET 0 318 319 static void mbuf_grab(struct mbuf *m) 320 { 321 struct mbuf_xennet *ref; 322 323 ref = (struct mbuf_xennet *)m_tag_locate(m, MTAG_COOKIE, 324 MTAG_XENNET, NULL); 325 KASSERT(ref != NULL, ("Cannot find refcount")); 326 ref->count++; 327 } 328 329 static void mbuf_release(struct mbuf *m) 330 { 331 struct mbuf_xennet *ref; 332 333 ref = (struct mbuf_xennet *)m_tag_locate(m, MTAG_COOKIE, 334 MTAG_XENNET, NULL); 335 KASSERT(ref != NULL, ("Cannot find refcount")); 336 KASSERT(ref->count > 0, ("Invalid reference count")); 337 338 if (--ref->count == 0) { 339 /* 340 * Explicitly free the tag while we hold the tx queue lock. 341 * This ensures that the tag is deleted promptly in case 342 * something else is holding extra references to the mbuf chain, 343 * such as netmap. 344 */ 345 m_tag_delete(m, &ref->tag); 346 m_freem(m); 347 } 348 } 349 350 static void tag_free(struct m_tag *t) 351 { 352 struct mbuf_xennet *ref = (struct mbuf_xennet *)t; 353 354 KASSERT(ref->count == 0, ("Free mbuf tag with pending refcnt")); 355 bus_dmamap_sync(ref->dma_tag, ref->dma_map, BUS_DMASYNC_POSTWRITE); 356 bus_dmamap_destroy(ref->dma_tag, ref->dma_map); 357 SLIST_INSERT_HEAD(&ref->txq->tags, ref, next); 358 } 359 360 #define IPRINTK(fmt, args...) \ 361 printf("[XEN] " fmt, ##args) 362 #ifdef INVARIANTS 363 #define WPRINTK(fmt, args...) \ 364 printf("[XEN] " fmt, ##args) 365 #else 366 #define WPRINTK(fmt, args...) 367 #endif 368 #ifdef DEBUG 369 #define DPRINTK(fmt, args...) \ 370 printf("[XEN] %s: " fmt, __func__, ##args) 371 #else 372 #define DPRINTK(fmt, args...) 373 #endif 374 375 /** 376 * Read the 'mac' node at the given device's node in the store, and parse that 377 * as colon-separated octets, placing result the given mac array. mac must be 378 * a preallocated array of length ETH_ALEN (as declared in linux/if_ether.h). 379 * Return 0 on success, or errno on error. 380 */ 381 static int 382 xen_net_read_mac(device_t dev, uint8_t mac[]) 383 { 384 int error, i; 385 char *s, *e, *macstr; 386 const char *path; 387 388 path = xenbus_get_node(dev); 389 error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr); 390 if (error == ENOENT) { 391 /* 392 * Deal with missing mac XenStore nodes on devices with 393 * HVM emulation (the 'ioemu' configuration attribute) 394 * enabled. 395 * 396 * The HVM emulator may execute in a stub device model 397 * domain which lacks the permission, only given to Dom0, 398 * to update the guest's XenStore tree. For this reason, 399 * the HVM emulator doesn't even attempt to write the 400 * front-side mac node, even when operating in Dom0. 401 * However, there should always be a mac listed in the 402 * backend tree. Fallback to this version if our query 403 * of the front side XenStore location doesn't find 404 * anything. 405 */ 406 path = xenbus_get_otherend_path(dev); 407 error = xs_read(XST_NIL, path, "mac", NULL, (void **) &macstr); 408 } 409 if (error != 0) { 410 xenbus_dev_fatal(dev, error, "parsing %s/mac", path); 411 return (error); 412 } 413 414 s = macstr; 415 for (i = 0; i < ETHER_ADDR_LEN; i++) { 416 mac[i] = strtoul(s, &e, 16); 417 if (s == e || (e[0] != ':' && e[0] != 0)) { 418 free(macstr, M_XENBUS); 419 return (ENOENT); 420 } 421 s = &e[1]; 422 } 423 free(macstr, M_XENBUS); 424 return (0); 425 } 426 427 /** 428 * Entry point to this code when a new device is created. Allocate the basic 429 * structures and the ring buffers for communication with the backend, and 430 * inform the backend of the appropriate details for those. Switch to 431 * Connected state. 432 */ 433 static int 434 netfront_probe(device_t dev) 435 { 436 437 if (xen_pv_nics_disabled()) 438 return (ENXIO); 439 440 if (!strcmp(xenbus_get_type(dev), "vif")) { 441 device_set_desc(dev, "Virtual Network Interface"); 442 return (0); 443 } 444 445 return (ENXIO); 446 } 447 448 static int 449 netfront_attach(device_t dev) 450 { 451 int err; 452 453 err = create_netdev(dev); 454 if (err != 0) { 455 xenbus_dev_fatal(dev, err, "creating netdev"); 456 return (err); 457 } 458 459 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 460 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 461 OID_AUTO, "enable_lro", CTLFLAG_RW, 462 &xn_enable_lro, 0, "Large Receive Offload"); 463 464 SYSCTL_ADD_ULONG(device_get_sysctl_ctx(dev), 465 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 466 OID_AUTO, "num_queues", CTLFLAG_RD, 467 &xn_num_queues, "Number of pairs of queues"); 468 469 return (0); 470 } 471 472 static int 473 netfront_suspend(device_t dev) 474 { 475 struct netfront_info *np = device_get_softc(dev); 476 u_int i; 477 478 for (i = 0; i < np->num_queues; i++) { 479 XN_RX_LOCK(&np->rxq[i]); 480 XN_TX_LOCK(&np->txq[i]); 481 } 482 netfront_carrier_off(np); 483 for (i = 0; i < np->num_queues; i++) { 484 XN_RX_UNLOCK(&np->rxq[i]); 485 XN_TX_UNLOCK(&np->txq[i]); 486 } 487 return (0); 488 } 489 490 /** 491 * We are reconnecting to the backend, due to a suspend/resume, or a backend 492 * driver restart. We tear down our netif structure and recreate it, but 493 * leave the device-layer structures intact so that this is transparent to the 494 * rest of the kernel. 495 */ 496 static int 497 netfront_resume(device_t dev) 498 { 499 struct netfront_info *info = device_get_softc(dev); 500 u_int i; 501 502 if (xen_suspend_cancelled) { 503 for (i = 0; i < info->num_queues; i++) { 504 XN_RX_LOCK(&info->rxq[i]); 505 XN_TX_LOCK(&info->txq[i]); 506 } 507 netfront_carrier_on(info); 508 for (i = 0; i < info->num_queues; i++) { 509 XN_RX_UNLOCK(&info->rxq[i]); 510 XN_TX_UNLOCK(&info->txq[i]); 511 } 512 return (0); 513 } 514 515 netif_disconnect_backend(info); 516 return (0); 517 } 518 519 static int 520 write_queue_xenstore_keys(device_t dev, 521 struct netfront_rxq *rxq, 522 struct netfront_txq *txq, 523 struct xs_transaction *xst, bool hierarchy) 524 { 525 int err; 526 const char *message; 527 const char *node = xenbus_get_node(dev); 528 char *path; 529 size_t path_size; 530 531 KASSERT(rxq->id == txq->id, ("Mismatch between RX and TX queue ids")); 532 /* Split event channel support is not yet there. */ 533 KASSERT(rxq->xen_intr_handle == txq->xen_intr_handle, 534 ("Split event channels are not supported")); 535 536 if (hierarchy) { 537 path_size = strlen(node) + 10; 538 path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO); 539 snprintf(path, path_size, "%s/queue-%u", node, rxq->id); 540 } else { 541 path_size = strlen(node) + 1; 542 path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO); 543 snprintf(path, path_size, "%s", node); 544 } 545 546 err = xs_printf(*xst, path, "tx-ring-ref","%u", txq->ring_ref); 547 if (err != 0) { 548 message = "writing tx ring-ref"; 549 goto error; 550 } 551 err = xs_printf(*xst, path, "rx-ring-ref","%u", rxq->ring_ref); 552 if (err != 0) { 553 message = "writing rx ring-ref"; 554 goto error; 555 } 556 err = xs_printf(*xst, path, "event-channel", "%u", 557 xen_intr_port(rxq->xen_intr_handle)); 558 if (err != 0) { 559 message = "writing event-channel"; 560 goto error; 561 } 562 563 free(path, M_DEVBUF); 564 565 return (0); 566 567 error: 568 free(path, M_DEVBUF); 569 xenbus_dev_fatal(dev, err, "%s", message); 570 571 return (err); 572 } 573 574 /* Common code used when first setting up, and when resuming. */ 575 static int 576 talk_to_backend(device_t dev, struct netfront_info *info) 577 { 578 const char *message; 579 struct xs_transaction xst; 580 const char *node = xenbus_get_node(dev); 581 int err; 582 unsigned long num_queues, max_queues = 0; 583 unsigned int i; 584 585 err = xen_net_read_mac(dev, info->mac); 586 if (err != 0) { 587 xenbus_dev_fatal(dev, err, "parsing %s/mac", node); 588 goto out; 589 } 590 591 err = xs_scanf(XST_NIL, xenbus_get_otherend_path(info->xbdev), 592 "multi-queue-max-queues", NULL, "%lu", &max_queues); 593 if (err != 0) 594 max_queues = 1; 595 num_queues = xn_num_queues; 596 if (num_queues > max_queues) 597 num_queues = max_queues; 598 599 err = setup_device(dev, info, num_queues); 600 if (err != 0) 601 goto out; 602 603 again: 604 err = xs_transaction_start(&xst); 605 if (err != 0) { 606 xenbus_dev_fatal(dev, err, "starting transaction"); 607 goto free; 608 } 609 610 if (info->num_queues == 1) { 611 err = write_queue_xenstore_keys(dev, &info->rxq[0], 612 &info->txq[0], &xst, false); 613 if (err != 0) 614 goto abort_transaction_no_def_error; 615 } else { 616 err = xs_printf(xst, node, "multi-queue-num-queues", 617 "%u", info->num_queues); 618 if (err != 0) { 619 message = "writing multi-queue-num-queues"; 620 goto abort_transaction; 621 } 622 623 for (i = 0; i < info->num_queues; i++) { 624 err = write_queue_xenstore_keys(dev, &info->rxq[i], 625 &info->txq[i], &xst, true); 626 if (err != 0) 627 goto abort_transaction_no_def_error; 628 } 629 } 630 631 err = xs_printf(xst, node, "request-rx-copy", "%u", 1); 632 if (err != 0) { 633 message = "writing request-rx-copy"; 634 goto abort_transaction; 635 } 636 err = xs_printf(xst, node, "feature-rx-notify", "%d", 1); 637 if (err != 0) { 638 message = "writing feature-rx-notify"; 639 goto abort_transaction; 640 } 641 err = xs_printf(xst, node, "feature-sg", "%d", 1); 642 if (err != 0) { 643 message = "writing feature-sg"; 644 goto abort_transaction; 645 } 646 if ((if_getcapenable(info->xn_ifp) & IFCAP_LRO) != 0) { 647 err = xs_printf(xst, node, "feature-gso-tcpv4", "%d", 1); 648 if (err != 0) { 649 message = "writing feature-gso-tcpv4"; 650 goto abort_transaction; 651 } 652 } 653 if ((if_getcapenable(info->xn_ifp) & IFCAP_RXCSUM) == 0) { 654 err = xs_printf(xst, node, "feature-no-csum-offload", "%d", 1); 655 if (err != 0) { 656 message = "writing feature-no-csum-offload"; 657 goto abort_transaction; 658 } 659 } 660 661 err = xs_transaction_end(xst, 0); 662 if (err != 0) { 663 if (err == EAGAIN) 664 goto again; 665 xenbus_dev_fatal(dev, err, "completing transaction"); 666 goto free; 667 } 668 669 return 0; 670 671 abort_transaction: 672 xenbus_dev_fatal(dev, err, "%s", message); 673 abort_transaction_no_def_error: 674 xs_transaction_end(xst, 1); 675 free: 676 netif_free(info); 677 out: 678 return (err); 679 } 680 681 static void 682 xn_rxq_intr(struct netfront_rxq *rxq) 683 { 684 685 XN_RX_LOCK(rxq); 686 xn_rxeof(rxq); 687 XN_RX_UNLOCK(rxq); 688 } 689 690 static void 691 xn_txq_start(struct netfront_txq *txq) 692 { 693 struct netfront_info *np = txq->info; 694 if_t ifp = np->xn_ifp; 695 696 XN_TX_LOCK_ASSERT(txq); 697 if (!drbr_empty(ifp, txq->br)) 698 xn_txq_mq_start_locked(txq, NULL); 699 } 700 701 static void 702 xn_txq_intr(struct netfront_txq *txq) 703 { 704 705 XN_TX_LOCK(txq); 706 if (RING_HAS_UNCONSUMED_RESPONSES(&txq->ring)) 707 xn_txeof(txq); 708 xn_txq_start(txq); 709 XN_TX_UNLOCK(txq); 710 } 711 712 static void 713 xn_txq_tq_deferred(void *xtxq, int pending) 714 { 715 struct netfront_txq *txq = xtxq; 716 717 XN_TX_LOCK(txq); 718 xn_txq_start(txq); 719 XN_TX_UNLOCK(txq); 720 } 721 722 static void 723 disconnect_rxq(struct netfront_rxq *rxq) 724 { 725 726 xn_release_rx_bufs(rxq); 727 gnttab_free_grant_references(rxq->gref_head); 728 gnttab_end_foreign_access(rxq->ring_ref, NULL); 729 /* 730 * No split event channel support at the moment, handle will 731 * be unbound in tx. So no need to call xen_intr_unbind here, 732 * but we do want to reset the handler to 0. 733 */ 734 rxq->xen_intr_handle = 0; 735 } 736 737 static void 738 destroy_rxq(struct netfront_rxq *rxq) 739 { 740 741 callout_drain(&rxq->rx_refill); 742 free(rxq->ring.sring, M_DEVBUF); 743 } 744 745 static void 746 destroy_rxqs(struct netfront_info *np) 747 { 748 int i; 749 750 for (i = 0; i < np->num_queues; i++) 751 destroy_rxq(&np->rxq[i]); 752 753 free(np->rxq, M_DEVBUF); 754 np->rxq = NULL; 755 } 756 757 static int 758 setup_rxqs(device_t dev, struct netfront_info *info, 759 unsigned long num_queues) 760 { 761 int q, i; 762 int error; 763 netif_rx_sring_t *rxs; 764 struct netfront_rxq *rxq; 765 766 info->rxq = malloc(sizeof(struct netfront_rxq) * num_queues, 767 M_DEVBUF, M_WAITOK|M_ZERO); 768 769 for (q = 0; q < num_queues; q++) { 770 rxq = &info->rxq[q]; 771 772 rxq->id = q; 773 rxq->info = info; 774 rxq->ring_ref = GRANT_REF_INVALID; 775 rxq->ring.sring = NULL; 776 snprintf(rxq->name, XN_QUEUE_NAME_LEN, "xnrx_%u", q); 777 mtx_init(&rxq->lock, rxq->name, "netfront receive lock", 778 MTX_DEF); 779 780 for (i = 0; i <= NET_RX_RING_SIZE; i++) { 781 rxq->mbufs[i] = NULL; 782 rxq->grant_ref[i] = GRANT_REF_INVALID; 783 } 784 785 /* Start resources allocation */ 786 787 if (gnttab_alloc_grant_references(NET_RX_RING_SIZE, 788 &rxq->gref_head) != 0) { 789 device_printf(dev, "allocating rx gref"); 790 error = ENOMEM; 791 goto fail; 792 } 793 794 rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, 795 M_WAITOK|M_ZERO); 796 SHARED_RING_INIT(rxs); 797 FRONT_RING_INIT(&rxq->ring, rxs, PAGE_SIZE); 798 799 error = xenbus_grant_ring(dev, virt_to_mfn(rxs), 800 &rxq->ring_ref); 801 if (error != 0) { 802 device_printf(dev, "granting rx ring page"); 803 goto fail_grant_ring; 804 } 805 806 callout_init(&rxq->rx_refill, 1); 807 } 808 809 return (0); 810 811 fail_grant_ring: 812 gnttab_free_grant_references(rxq->gref_head); 813 free(rxq->ring.sring, M_DEVBUF); 814 fail: 815 for (; q >= 0; q--) { 816 disconnect_rxq(&info->rxq[q]); 817 destroy_rxq(&info->rxq[q]); 818 } 819 820 free(info->rxq, M_DEVBUF); 821 return (error); 822 } 823 824 static void 825 disconnect_txq(struct netfront_txq *txq) 826 { 827 828 xn_release_tx_bufs(txq); 829 gnttab_free_grant_references(txq->gref_head); 830 gnttab_end_foreign_access(txq->ring_ref, NULL); 831 xen_intr_unbind(&txq->xen_intr_handle); 832 } 833 834 static void 835 destroy_txq(struct netfront_txq *txq) 836 { 837 unsigned int i; 838 839 free(txq->ring.sring, M_DEVBUF); 840 buf_ring_free(txq->br, M_DEVBUF); 841 taskqueue_drain_all(txq->tq); 842 taskqueue_free(txq->tq); 843 844 for (i = 0; i <= NET_TX_RING_SIZE; i++) { 845 bus_dmamap_destroy(txq->info->dma_tag, 846 txq->xennet_tag[i].dma_map); 847 txq->xennet_tag[i].dma_map = NULL; 848 } 849 } 850 851 static void 852 destroy_txqs(struct netfront_info *np) 853 { 854 int i; 855 856 for (i = 0; i < np->num_queues; i++) 857 destroy_txq(&np->txq[i]); 858 859 free(np->txq, M_DEVBUF); 860 np->txq = NULL; 861 } 862 863 static int 864 setup_txqs(device_t dev, struct netfront_info *info, 865 unsigned long num_queues) 866 { 867 int q, i; 868 int error; 869 netif_tx_sring_t *txs; 870 struct netfront_txq *txq; 871 872 info->txq = malloc(sizeof(struct netfront_txq) * num_queues, 873 M_DEVBUF, M_WAITOK|M_ZERO); 874 875 for (q = 0; q < num_queues; q++) { 876 txq = &info->txq[q]; 877 878 txq->id = q; 879 txq->info = info; 880 881 txq->ring_ref = GRANT_REF_INVALID; 882 txq->ring.sring = NULL; 883 884 snprintf(txq->name, XN_QUEUE_NAME_LEN, "xntx_%u", q); 885 886 mtx_init(&txq->lock, txq->name, "netfront transmit lock", 887 MTX_DEF); 888 SLIST_INIT(&txq->tags); 889 890 for (i = 0; i <= NET_TX_RING_SIZE; i++) { 891 txq->mbufs[i] = (void *) ((u_long) i+1); 892 txq->grant_ref[i] = GRANT_REF_INVALID; 893 txq->xennet_tag[i].txq = txq; 894 txq->xennet_tag[i].dma_tag = info->dma_tag; 895 error = bus_dmamap_create(info->dma_tag, 0, 896 &txq->xennet_tag[i].dma_map); 897 if (error != 0) { 898 device_printf(dev, 899 "failed to allocate dma map\n"); 900 goto fail; 901 } 902 m_tag_setup(&txq->xennet_tag[i].tag, 903 MTAG_COOKIE, MTAG_XENNET, 904 sizeof(txq->xennet_tag[i]) - 905 sizeof(txq->xennet_tag[i].tag)); 906 txq->xennet_tag[i].tag.m_tag_free = &tag_free; 907 SLIST_INSERT_HEAD(&txq->tags, &txq->xennet_tag[i], 908 next); 909 } 910 txq->mbufs[NET_TX_RING_SIZE] = (void *)0; 911 912 /* Start resources allocation. */ 913 914 if (gnttab_alloc_grant_references(NET_TX_RING_SIZE, 915 &txq->gref_head) != 0) { 916 device_printf(dev, "failed to allocate tx grant refs\n"); 917 error = ENOMEM; 918 goto fail; 919 } 920 921 txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, 922 M_WAITOK|M_ZERO); 923 SHARED_RING_INIT(txs); 924 FRONT_RING_INIT(&txq->ring, txs, PAGE_SIZE); 925 926 error = xenbus_grant_ring(dev, virt_to_mfn(txs), 927 &txq->ring_ref); 928 if (error != 0) { 929 device_printf(dev, "failed to grant tx ring\n"); 930 goto fail_grant_ring; 931 } 932 933 txq->br = buf_ring_alloc(NET_TX_RING_SIZE, M_DEVBUF, 934 M_WAITOK, &txq->lock); 935 TASK_INIT(&txq->defrtask, 0, xn_txq_tq_deferred, txq); 936 937 txq->tq = taskqueue_create(txq->name, M_WAITOK, 938 taskqueue_thread_enqueue, &txq->tq); 939 940 error = taskqueue_start_threads(&txq->tq, 1, PI_NET, 941 "%s txq %d", device_get_nameunit(dev), txq->id); 942 if (error != 0) { 943 device_printf(dev, "failed to start tx taskq %d\n", 944 txq->id); 945 goto fail_start_thread; 946 } 947 948 error = xen_intr_alloc_and_bind_local_port(dev, 949 xenbus_get_otherend_id(dev), /* filter */ NULL, xn_intr, 950 &info->txq[q], INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, 951 &txq->xen_intr_handle); 952 953 if (error != 0) { 954 device_printf(dev, "xen_intr_alloc_and_bind_local_port failed\n"); 955 goto fail_bind_port; 956 } 957 } 958 959 return (0); 960 961 fail_bind_port: 962 taskqueue_drain_all(txq->tq); 963 fail_start_thread: 964 buf_ring_free(txq->br, M_DEVBUF); 965 taskqueue_free(txq->tq); 966 gnttab_end_foreign_access(txq->ring_ref, NULL); 967 fail_grant_ring: 968 gnttab_free_grant_references(txq->gref_head); 969 free(txq->ring.sring, M_DEVBUF); 970 fail: 971 for (; q >= 0; q--) { 972 disconnect_txq(&info->txq[q]); 973 destroy_txq(&info->txq[q]); 974 } 975 976 free(info->txq, M_DEVBUF); 977 return (error); 978 } 979 980 static int 981 setup_device(device_t dev, struct netfront_info *info, 982 unsigned long num_queues) 983 { 984 int error; 985 int q; 986 987 if (info->txq) 988 destroy_txqs(info); 989 990 if (info->rxq) 991 destroy_rxqs(info); 992 993 info->num_queues = 0; 994 995 error = setup_rxqs(dev, info, num_queues); 996 if (error != 0) 997 goto out; 998 error = setup_txqs(dev, info, num_queues); 999 if (error != 0) 1000 goto out; 1001 1002 info->num_queues = num_queues; 1003 1004 /* No split event channel at the moment. */ 1005 for (q = 0; q < num_queues; q++) 1006 info->rxq[q].xen_intr_handle = info->txq[q].xen_intr_handle; 1007 1008 return (0); 1009 1010 out: 1011 KASSERT(error != 0, ("Error path taken without providing an error code")); 1012 return (error); 1013 } 1014 1015 #ifdef INET 1016 static u_int 1017 netfront_addr_cb(void *arg, struct ifaddr *a, u_int count) 1018 { 1019 arp_ifinit((if_t)arg, a); 1020 return (1); 1021 } 1022 /** 1023 * If this interface has an ipv4 address, send an arp for it. This 1024 * helps to get the network going again after migrating hosts. 1025 */ 1026 static void 1027 netfront_send_fake_arp(device_t dev, struct netfront_info *info) 1028 { 1029 if_t ifp; 1030 1031 ifp = info->xn_ifp; 1032 if_foreach_addr_type(ifp, AF_INET, netfront_addr_cb, ifp); 1033 } 1034 #endif 1035 1036 /** 1037 * Callback received when the backend's state changes. 1038 */ 1039 static void 1040 netfront_backend_changed(device_t dev, XenbusState newstate) 1041 { 1042 struct netfront_info *sc = device_get_softc(dev); 1043 1044 DPRINTK("newstate=%d\n", newstate); 1045 1046 CURVNET_SET(if_getvnet(sc->xn_ifp)); 1047 1048 switch (newstate) { 1049 case XenbusStateInitialising: 1050 case XenbusStateInitialised: 1051 case XenbusStateUnknown: 1052 case XenbusStateReconfigured: 1053 case XenbusStateReconfiguring: 1054 break; 1055 case XenbusStateInitWait: 1056 if (xenbus_get_state(dev) != XenbusStateInitialising) 1057 break; 1058 if (xn_connect(sc) != 0) 1059 break; 1060 /* Switch to connected state before kicking the rings. */ 1061 xenbus_set_state(sc->xbdev, XenbusStateConnected); 1062 xn_kick_rings(sc); 1063 break; 1064 case XenbusStateClosing: 1065 xenbus_set_state(dev, XenbusStateClosed); 1066 break; 1067 case XenbusStateClosed: 1068 if (sc->xn_reset) { 1069 netif_disconnect_backend(sc); 1070 xenbus_set_state(dev, XenbusStateInitialising); 1071 sc->xn_reset = false; 1072 } 1073 break; 1074 case XenbusStateConnected: 1075 #ifdef INET 1076 netfront_send_fake_arp(dev, sc); 1077 #endif 1078 break; 1079 } 1080 1081 CURVNET_RESTORE(); 1082 } 1083 1084 /** 1085 * \brief Verify that there is sufficient space in the Tx ring 1086 * buffer for a maximally sized request to be enqueued. 1087 * 1088 * A transmit request requires a transmit descriptor for each packet 1089 * fragment, plus up to 2 entries for "options" (e.g. TSO). 1090 */ 1091 static inline int 1092 xn_tx_slot_available(struct netfront_txq *txq) 1093 { 1094 1095 return (RING_FREE_REQUESTS(&txq->ring) > (MAX_TX_REQ_FRAGS + 2)); 1096 } 1097 1098 static void 1099 xn_release_tx_bufs(struct netfront_txq *txq) 1100 { 1101 int i; 1102 1103 for (i = 1; i <= NET_TX_RING_SIZE; i++) { 1104 struct mbuf *m; 1105 1106 m = txq->mbufs[i]; 1107 1108 /* 1109 * We assume that no kernel addresses are 1110 * less than NET_TX_RING_SIZE. Any entry 1111 * in the table that is below this number 1112 * must be an index from free-list tracking. 1113 */ 1114 if (((uintptr_t)m) <= NET_TX_RING_SIZE) 1115 continue; 1116 gnttab_end_foreign_access_ref(txq->grant_ref[i]); 1117 gnttab_release_grant_reference(&txq->gref_head, 1118 txq->grant_ref[i]); 1119 txq->grant_ref[i] = GRANT_REF_INVALID; 1120 add_id_to_freelist(txq->mbufs, i); 1121 txq->mbufs_cnt--; 1122 if (txq->mbufs_cnt < 0) { 1123 panic("%s: tx_chain_cnt must be >= 0", __func__); 1124 } 1125 mbuf_release(m); 1126 } 1127 } 1128 1129 static struct mbuf * 1130 xn_alloc_one_rx_buffer(struct netfront_rxq *rxq) 1131 { 1132 struct mbuf *m; 1133 1134 m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE); 1135 if (m == NULL) 1136 return NULL; 1137 m->m_len = m->m_pkthdr.len = MJUMPAGESIZE; 1138 1139 return (m); 1140 } 1141 1142 static void 1143 xn_alloc_rx_buffers(struct netfront_rxq *rxq) 1144 { 1145 RING_IDX req_prod; 1146 int notify; 1147 1148 XN_RX_LOCK_ASSERT(rxq); 1149 1150 if (__predict_false(rxq->info->carrier == 0)) 1151 return; 1152 1153 for (req_prod = rxq->ring.req_prod_pvt; 1154 req_prod - rxq->ring.rsp_cons < NET_RX_RING_SIZE; 1155 req_prod++) { 1156 struct mbuf *m; 1157 unsigned short id; 1158 grant_ref_t ref; 1159 struct netif_rx_request *req; 1160 unsigned long pfn; 1161 1162 m = xn_alloc_one_rx_buffer(rxq); 1163 if (m == NULL) 1164 break; 1165 1166 id = xn_rxidx(req_prod); 1167 1168 KASSERT(rxq->mbufs[id] == NULL, ("non-NULL xn_rx_chain")); 1169 rxq->mbufs[id] = m; 1170 1171 ref = gnttab_claim_grant_reference(&rxq->gref_head); 1172 KASSERT(ref != GNTTAB_LIST_END, 1173 ("reserved grant references exhuasted")); 1174 rxq->grant_ref[id] = ref; 1175 1176 pfn = atop(vtophys(mtod(m, vm_offset_t))); 1177 req = RING_GET_REQUEST(&rxq->ring, req_prod); 1178 1179 gnttab_grant_foreign_access_ref(ref, 1180 xenbus_get_otherend_id(rxq->info->xbdev), pfn, 0); 1181 req->id = id; 1182 req->gref = ref; 1183 } 1184 1185 rxq->ring.req_prod_pvt = req_prod; 1186 1187 /* Not enough requests? Try again later. */ 1188 if (req_prod - rxq->ring.rsp_cons < NET_RX_SLOTS_MIN) { 1189 callout_reset_curcpu(&rxq->rx_refill, hz/10, 1190 xn_alloc_rx_buffers_callout, rxq); 1191 return; 1192 } 1193 1194 wmb(); /* barrier so backend seens requests */ 1195 1196 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rxq->ring, notify); 1197 if (notify) 1198 xen_intr_signal(rxq->xen_intr_handle); 1199 } 1200 1201 static void xn_alloc_rx_buffers_callout(void *arg) 1202 { 1203 struct netfront_rxq *rxq; 1204 1205 rxq = (struct netfront_rxq *)arg; 1206 XN_RX_LOCK(rxq); 1207 xn_alloc_rx_buffers(rxq); 1208 XN_RX_UNLOCK(rxq); 1209 } 1210 1211 static void 1212 xn_release_rx_bufs(struct netfront_rxq *rxq) 1213 { 1214 int i, ref; 1215 struct mbuf *m; 1216 1217 for (i = 0; i < NET_RX_RING_SIZE; i++) { 1218 m = rxq->mbufs[i]; 1219 1220 if (m == NULL) 1221 continue; 1222 1223 ref = rxq->grant_ref[i]; 1224 if (ref == GRANT_REF_INVALID) 1225 continue; 1226 1227 gnttab_end_foreign_access_ref(ref); 1228 gnttab_release_grant_reference(&rxq->gref_head, ref); 1229 rxq->mbufs[i] = NULL; 1230 rxq->grant_ref[i] = GRANT_REF_INVALID; 1231 m_freem(m); 1232 } 1233 } 1234 1235 static void 1236 xn_rxeof(struct netfront_rxq *rxq) 1237 { 1238 if_t ifp; 1239 struct netfront_info *np = rxq->info; 1240 #if (defined(INET) || defined(INET6)) 1241 struct lro_ctrl *lro = &rxq->lro; 1242 #endif 1243 struct netfront_rx_info rinfo; 1244 struct netif_rx_response *rx = &rinfo.rx; 1245 struct netif_extra_info *extras = rinfo.extras; 1246 RING_IDX i, rp; 1247 struct mbuf *m; 1248 struct mbufq mbufq_rxq, mbufq_errq; 1249 int err, work_to_do; 1250 1251 XN_RX_LOCK_ASSERT(rxq); 1252 1253 if (!netfront_carrier_ok(np)) 1254 return; 1255 1256 /* XXX: there should be some sane limit. */ 1257 mbufq_init(&mbufq_errq, INT_MAX); 1258 mbufq_init(&mbufq_rxq, INT_MAX); 1259 1260 ifp = np->xn_ifp; 1261 1262 do { 1263 rp = rxq->ring.sring->rsp_prod; 1264 rmb(); /* Ensure we see queued responses up to 'rp'. */ 1265 1266 i = rxq->ring.rsp_cons; 1267 while ((i != rp)) { 1268 memcpy(rx, RING_GET_RESPONSE(&rxq->ring, i), sizeof(*rx)); 1269 memset(extras, 0, sizeof(rinfo.extras)); 1270 1271 m = NULL; 1272 err = xn_get_responses(rxq, &rinfo, rp, &i, &m); 1273 1274 if (__predict_false(err)) { 1275 if (m) 1276 (void )mbufq_enqueue(&mbufq_errq, m); 1277 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 1278 continue; 1279 } 1280 1281 m->m_pkthdr.rcvif = ifp; 1282 if (rx->flags & NETRXF_data_validated) { 1283 /* 1284 * According to mbuf(9) the correct way to tell 1285 * the stack that the checksum of an inbound 1286 * packet is correct, without it actually being 1287 * present (because the underlying interface 1288 * doesn't provide it), is to set the 1289 * CSUM_DATA_VALID and CSUM_PSEUDO_HDR flags, 1290 * and the csum_data field to 0xffff. 1291 */ 1292 m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID 1293 | CSUM_PSEUDO_HDR); 1294 m->m_pkthdr.csum_data = 0xffff; 1295 } 1296 if ((rx->flags & NETRXF_extra_info) != 0 && 1297 (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type == 1298 XEN_NETIF_EXTRA_TYPE_GSO)) { 1299 m->m_pkthdr.tso_segsz = 1300 extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].u.gso.size; 1301 m->m_pkthdr.csum_flags |= CSUM_TSO; 1302 } 1303 1304 (void )mbufq_enqueue(&mbufq_rxq, m); 1305 } 1306 1307 rxq->ring.rsp_cons = i; 1308 1309 xn_alloc_rx_buffers(rxq); 1310 1311 RING_FINAL_CHECK_FOR_RESPONSES(&rxq->ring, work_to_do); 1312 } while (work_to_do); 1313 1314 mbufq_drain(&mbufq_errq); 1315 /* 1316 * Process all the mbufs after the remapping is complete. 1317 * Break the mbuf chain first though. 1318 */ 1319 while ((m = mbufq_dequeue(&mbufq_rxq)) != NULL) { 1320 if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); 1321 #if (defined(INET) || defined(INET6)) 1322 /* Use LRO if possible */ 1323 if ((if_getcapenable(ifp) & IFCAP_LRO) == 0 || 1324 lro->lro_cnt == 0 || tcp_lro_rx(lro, m, 0)) { 1325 /* 1326 * If LRO fails, pass up to the stack 1327 * directly. 1328 */ 1329 if_input(ifp, m); 1330 } 1331 #else 1332 if_input(ifp, m); 1333 #endif 1334 } 1335 1336 #if (defined(INET) || defined(INET6)) 1337 /* 1338 * Flush any outstanding LRO work 1339 */ 1340 tcp_lro_flush_all(lro); 1341 #endif 1342 } 1343 1344 static void 1345 xn_txeof(struct netfront_txq *txq) 1346 { 1347 RING_IDX i, prod; 1348 unsigned short id; 1349 if_t ifp; 1350 netif_tx_response_t *txr; 1351 struct mbuf *m; 1352 struct netfront_info *np = txq->info; 1353 1354 XN_TX_LOCK_ASSERT(txq); 1355 1356 if (!netfront_carrier_ok(np)) 1357 return; 1358 1359 ifp = np->xn_ifp; 1360 1361 do { 1362 prod = txq->ring.sring->rsp_prod; 1363 rmb(); /* Ensure we see responses up to 'rp'. */ 1364 1365 for (i = txq->ring.rsp_cons; i != prod; i++) { 1366 txr = RING_GET_RESPONSE(&txq->ring, i); 1367 if (txr->status == NETIF_RSP_NULL) 1368 continue; 1369 1370 if (txr->status != NETIF_RSP_OKAY) { 1371 printf("%s: WARNING: response is %d!\n", 1372 __func__, txr->status); 1373 } 1374 id = txr->id; 1375 m = txq->mbufs[id]; 1376 KASSERT(m != NULL, ("mbuf not found in chain")); 1377 KASSERT((uintptr_t)m > NET_TX_RING_SIZE, 1378 ("mbuf already on the free list, but we're " 1379 "trying to free it again!")); 1380 M_ASSERTVALID(m); 1381 1382 if (__predict_false(gnttab_query_foreign_access( 1383 txq->grant_ref[id]) != 0)) { 1384 panic("%s: grant id %u still in use by the " 1385 "backend", __func__, id); 1386 } 1387 gnttab_end_foreign_access_ref(txq->grant_ref[id]); 1388 gnttab_release_grant_reference( 1389 &txq->gref_head, txq->grant_ref[id]); 1390 txq->grant_ref[id] = GRANT_REF_INVALID; 1391 1392 txq->mbufs[id] = NULL; 1393 add_id_to_freelist(txq->mbufs, id); 1394 txq->mbufs_cnt--; 1395 mbuf_release(m); 1396 /* Only mark the txq active if we've freed up at least one slot to try */ 1397 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 1398 } 1399 txq->ring.rsp_cons = prod; 1400 1401 /* 1402 * Set a new event, then check for race with update of 1403 * tx_cons. Note that it is essential to schedule a 1404 * callback, no matter how few buffers are pending. Even if 1405 * there is space in the transmit ring, higher layers may 1406 * be blocked because too much data is outstanding: in such 1407 * cases notification from Xen is likely to be the only kick 1408 * that we'll get. 1409 */ 1410 txq->ring.sring->rsp_event = 1411 prod + ((txq->ring.sring->req_prod - prod) >> 1) + 1; 1412 1413 mb(); 1414 } while (prod != txq->ring.sring->rsp_prod); 1415 1416 if (txq->full && 1417 ((txq->ring.sring->req_prod - prod) < NET_TX_RING_SIZE)) { 1418 txq->full = false; 1419 xn_txq_start(txq); 1420 } 1421 } 1422 1423 static void 1424 xn_intr(void *xsc) 1425 { 1426 struct netfront_txq *txq = xsc; 1427 struct netfront_info *np = txq->info; 1428 struct netfront_rxq *rxq = &np->rxq[txq->id]; 1429 1430 /* kick both tx and rx */ 1431 xn_rxq_intr(rxq); 1432 xn_txq_intr(txq); 1433 } 1434 1435 static void 1436 xn_move_rx_slot(struct netfront_rxq *rxq, struct mbuf *m, 1437 grant_ref_t ref) 1438 { 1439 int new = xn_rxidx(rxq->ring.req_prod_pvt); 1440 1441 KASSERT(rxq->mbufs[new] == NULL, ("mbufs != NULL")); 1442 rxq->mbufs[new] = m; 1443 rxq->grant_ref[new] = ref; 1444 RING_GET_REQUEST(&rxq->ring, rxq->ring.req_prod_pvt)->id = new; 1445 RING_GET_REQUEST(&rxq->ring, rxq->ring.req_prod_pvt)->gref = ref; 1446 rxq->ring.req_prod_pvt++; 1447 } 1448 1449 static int 1450 xn_get_extras(struct netfront_rxq *rxq, 1451 struct netif_extra_info *extras, RING_IDX rp, RING_IDX *cons) 1452 { 1453 struct netif_extra_info *extra; 1454 1455 int err = 0; 1456 1457 do { 1458 struct mbuf *m; 1459 grant_ref_t ref; 1460 1461 if (__predict_false(*cons + 1 == rp)) { 1462 err = EINVAL; 1463 break; 1464 } 1465 1466 extra = (struct netif_extra_info *) 1467 RING_GET_RESPONSE(&rxq->ring, ++(*cons)); 1468 1469 if (__predict_false(!extra->type || 1470 extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) { 1471 err = EINVAL; 1472 } else { 1473 memcpy(&extras[extra->type - 1], extra, sizeof(*extra)); 1474 } 1475 1476 m = xn_get_rx_mbuf(rxq, *cons); 1477 ref = xn_get_rx_ref(rxq, *cons); 1478 xn_move_rx_slot(rxq, m, ref); 1479 } while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE); 1480 1481 return err; 1482 } 1483 1484 static int 1485 xn_get_responses(struct netfront_rxq *rxq, 1486 struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons, 1487 struct mbuf **list) 1488 { 1489 struct netif_rx_response *rx = &rinfo->rx; 1490 struct netif_extra_info *extras = rinfo->extras; 1491 struct mbuf *m, *m0, *m_prev; 1492 grant_ref_t ref = xn_get_rx_ref(rxq, *cons); 1493 int frags = 1; 1494 int err = 0; 1495 u_long ret __diagused; 1496 1497 m0 = m = m_prev = xn_get_rx_mbuf(rxq, *cons); 1498 1499 if (rx->flags & NETRXF_extra_info) { 1500 err = xn_get_extras(rxq, extras, rp, cons); 1501 } 1502 1503 if (m0 != NULL) { 1504 m0->m_pkthdr.len = 0; 1505 m0->m_next = NULL; 1506 } 1507 1508 for (;;) { 1509 #if 0 1510 DPRINTK("rx->status=%hd rx->offset=%hu frags=%u\n", 1511 rx->status, rx->offset, frags); 1512 #endif 1513 if (__predict_false(rx->status < 0 || 1514 rx->offset + rx->status > PAGE_SIZE)) { 1515 xn_move_rx_slot(rxq, m, ref); 1516 if (m0 == m) 1517 m0 = NULL; 1518 m = NULL; 1519 err = EINVAL; 1520 goto next_skip_queue; 1521 } 1522 1523 /* 1524 * This definitely indicates a bug, either in this driver or in 1525 * the backend driver. In future this should flag the bad 1526 * situation to the system controller to reboot the backed. 1527 */ 1528 if (ref == GRANT_REF_INVALID) { 1529 printf("%s: Bad rx response id %d.\n", __func__, rx->id); 1530 err = EINVAL; 1531 goto next; 1532 } 1533 1534 ret = gnttab_end_foreign_access_ref(ref); 1535 KASSERT(ret, ("Unable to end access to grant references")); 1536 1537 gnttab_release_grant_reference(&rxq->gref_head, ref); 1538 1539 next: 1540 if (m == NULL) 1541 break; 1542 1543 m->m_len = rx->status; 1544 m->m_data += rx->offset; 1545 m0->m_pkthdr.len += rx->status; 1546 1547 next_skip_queue: 1548 if (!(rx->flags & NETRXF_more_data)) 1549 break; 1550 1551 if (*cons + frags == rp) { 1552 if (net_ratelimit()) 1553 WPRINTK("Need more frags\n"); 1554 err = ENOENT; 1555 printf("%s: cons %u frags %u rp %u, not enough frags\n", 1556 __func__, *cons, frags, rp); 1557 break; 1558 } 1559 /* 1560 * Note that m can be NULL, if rx->status < 0 or if 1561 * rx->offset + rx->status > PAGE_SIZE above. 1562 */ 1563 m_prev = m; 1564 1565 rx = RING_GET_RESPONSE(&rxq->ring, *cons + frags); 1566 m = xn_get_rx_mbuf(rxq, *cons + frags); 1567 1568 /* 1569 * m_prev == NULL can happen if rx->status < 0 or if 1570 * rx->offset + * rx->status > PAGE_SIZE above. 1571 */ 1572 if (m_prev != NULL) 1573 m_prev->m_next = m; 1574 1575 /* 1576 * m0 can be NULL if rx->status < 0 or if * rx->offset + 1577 * rx->status > PAGE_SIZE above. 1578 */ 1579 if (m0 == NULL) 1580 m0 = m; 1581 m->m_next = NULL; 1582 ref = xn_get_rx_ref(rxq, *cons + frags); 1583 frags++; 1584 } 1585 *list = m0; 1586 *cons += frags; 1587 1588 return (err); 1589 } 1590 1591 /** 1592 * Given an mbuf chain, make sure we have enough room and then push 1593 * it onto the transmit ring. 1594 */ 1595 static int 1596 xn_assemble_tx_request(struct netfront_txq *txq, struct mbuf *m_head) 1597 { 1598 struct netfront_info *np = txq->info; 1599 if_t ifp = np->xn_ifp; 1600 int otherend_id, error, nfrags; 1601 bus_dma_segment_t *segs = txq->segs; 1602 struct mbuf_xennet *tag; 1603 bus_dmamap_t map; 1604 unsigned int i; 1605 1606 KASSERT(!SLIST_EMPTY(&txq->tags), ("no tags available")); 1607 tag = SLIST_FIRST(&txq->tags); 1608 SLIST_REMOVE_HEAD(&txq->tags, next); 1609 KASSERT(tag->count == 0, ("tag already in-use")); 1610 map = tag->dma_map; 1611 error = bus_dmamap_load_mbuf_sg(np->dma_tag, map, m_head, segs, 1612 &nfrags, 0); 1613 if (error == EFBIG || nfrags > np->maxfrags) { 1614 struct mbuf *m; 1615 1616 bus_dmamap_unload(np->dma_tag, map); 1617 m = m_defrag(m_head, M_NOWAIT); 1618 if (!m) { 1619 /* 1620 * Defrag failed, so free the mbuf and 1621 * therefore drop the packet. 1622 */ 1623 SLIST_INSERT_HEAD(&txq->tags, tag, next); 1624 m_freem(m_head); 1625 return (EMSGSIZE); 1626 } 1627 m_head = m; 1628 error = bus_dmamap_load_mbuf_sg(np->dma_tag, map, m_head, segs, 1629 &nfrags, 0); 1630 if (error != 0 || nfrags > np->maxfrags) { 1631 bus_dmamap_unload(np->dma_tag, map); 1632 SLIST_INSERT_HEAD(&txq->tags, tag, next); 1633 m_freem(m_head); 1634 return (error ?: EFBIG); 1635 } 1636 } else if (error != 0) { 1637 SLIST_INSERT_HEAD(&txq->tags, tag, next); 1638 m_freem(m_head); 1639 return (error); 1640 } 1641 1642 /** 1643 * The FreeBSD TCP stack, with TSO enabled, can produce a chain 1644 * of mbufs longer than Linux can handle. Make sure we don't 1645 * pass a too-long chain over to the other side by dropping the 1646 * packet. It doesn't look like there is currently a way to 1647 * tell the TCP stack to generate a shorter chain of packets. 1648 */ 1649 if (nfrags > MAX_TX_REQ_FRAGS) { 1650 #ifdef DEBUG 1651 printf("%s: nfrags %d > MAX_TX_REQ_FRAGS %d, netback " 1652 "won't be able to handle it, dropping\n", 1653 __func__, nfrags, MAX_TX_REQ_FRAGS); 1654 #endif 1655 SLIST_INSERT_HEAD(&txq->tags, tag, next); 1656 bus_dmamap_unload(np->dma_tag, map); 1657 m_freem(m_head); 1658 return (EMSGSIZE); 1659 } 1660 1661 /* 1662 * This check should be redundant. We've already verified that we 1663 * have enough slots in the ring to handle a packet of maximum 1664 * size, and that our packet is less than the maximum size. Keep 1665 * it in here as an assert for now just to make certain that 1666 * chain_cnt is accurate. 1667 */ 1668 KASSERT((txq->mbufs_cnt + nfrags) <= NET_TX_RING_SIZE, 1669 ("%s: chain_cnt (%d) + nfrags (%d) > NET_TX_RING_SIZE " 1670 "(%d)!", __func__, (int) txq->mbufs_cnt, 1671 (int) nfrags, (int) NET_TX_RING_SIZE)); 1672 1673 /* 1674 * Start packing the mbufs in this chain into 1675 * the fragment pointers. Stop when we run out 1676 * of fragments or hit the end of the mbuf chain. 1677 */ 1678 otherend_id = xenbus_get_otherend_id(np->xbdev); 1679 m_tag_prepend(m_head, &tag->tag); 1680 for (i = 0; i < nfrags; i++) { 1681 netif_tx_request_t *tx; 1682 uintptr_t id; 1683 grant_ref_t ref; 1684 u_long mfn; /* XXX Wrong type? */ 1685 1686 tx = RING_GET_REQUEST(&txq->ring, txq->ring.req_prod_pvt); 1687 id = get_id_from_freelist(txq->mbufs); 1688 if (id == 0) 1689 panic("%s: was allocated the freelist head!\n", 1690 __func__); 1691 txq->mbufs_cnt++; 1692 if (txq->mbufs_cnt > NET_TX_RING_SIZE) 1693 panic("%s: tx_chain_cnt must be <= NET_TX_RING_SIZE\n", 1694 __func__); 1695 mbuf_grab(m_head); 1696 txq->mbufs[id] = m_head; 1697 tx->id = id; 1698 ref = gnttab_claim_grant_reference(&txq->gref_head); 1699 KASSERT((short)ref >= 0, ("Negative ref")); 1700 mfn = atop(segs[i].ds_addr); 1701 gnttab_grant_foreign_access_ref(ref, otherend_id, 1702 mfn, GNTMAP_readonly); 1703 tx->gref = txq->grant_ref[id] = ref; 1704 tx->offset = segs[i].ds_addr & PAGE_MASK; 1705 KASSERT(tx->offset + segs[i].ds_len <= PAGE_SIZE, 1706 ("mbuf segment crosses a page boundary")); 1707 tx->flags = 0; 1708 if (i == 0) { 1709 /* 1710 * The first fragment has the entire packet 1711 * size, subsequent fragments have just the 1712 * fragment size. The backend works out the 1713 * true size of the first fragment by 1714 * subtracting the sizes of the other 1715 * fragments. 1716 */ 1717 tx->size = m_head->m_pkthdr.len; 1718 1719 /* 1720 * The first fragment contains the checksum flags 1721 * and is optionally followed by extra data for 1722 * TSO etc. 1723 */ 1724 /** 1725 * CSUM_TSO requires checksum offloading. 1726 * Some versions of FreeBSD fail to 1727 * set CSUM_TCP in the CSUM_TSO case, 1728 * so we have to test for CSUM_TSO 1729 * explicitly. 1730 */ 1731 if (m_head->m_pkthdr.csum_flags 1732 & (CSUM_DELAY_DATA | CSUM_TSO)) { 1733 tx->flags |= (NETTXF_csum_blank 1734 | NETTXF_data_validated); 1735 } 1736 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1737 struct netif_extra_info *gso = 1738 (struct netif_extra_info *) 1739 RING_GET_REQUEST(&txq->ring, 1740 ++txq->ring.req_prod_pvt); 1741 1742 tx->flags |= NETTXF_extra_info; 1743 1744 gso->u.gso.size = m_head->m_pkthdr.tso_segsz; 1745 gso->u.gso.type = 1746 XEN_NETIF_GSO_TYPE_TCPV4; 1747 gso->u.gso.pad = 0; 1748 gso->u.gso.features = 0; 1749 1750 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; 1751 gso->flags = 0; 1752 } 1753 } else { 1754 tx->size = segs[i].ds_len; 1755 } 1756 if (i != nfrags - 1) 1757 tx->flags |= NETTXF_more_data; 1758 1759 txq->ring.req_prod_pvt++; 1760 } 1761 bus_dmamap_sync(np->dma_tag, map, BUS_DMASYNC_PREWRITE); 1762 BPF_MTAP(ifp, m_head); 1763 1764 if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); 1765 if_inc_counter(ifp, IFCOUNTER_OBYTES, m_head->m_pkthdr.len); 1766 if (m_head->m_flags & M_MCAST) 1767 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1768 1769 xn_txeof(txq); 1770 1771 return (0); 1772 } 1773 1774 /* equivalent of network_open() in Linux */ 1775 static void 1776 xn_ifinit_locked(struct netfront_info *np) 1777 { 1778 if_t ifp; 1779 int i; 1780 struct netfront_rxq *rxq; 1781 1782 XN_LOCK_ASSERT(np); 1783 1784 ifp = np->xn_ifp; 1785 1786 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING || !netfront_carrier_ok(np)) 1787 return; 1788 1789 xn_stop(np); 1790 1791 for (i = 0; i < np->num_queues; i++) { 1792 rxq = &np->rxq[i]; 1793 XN_RX_LOCK(rxq); 1794 xn_alloc_rx_buffers(rxq); 1795 rxq->ring.sring->rsp_event = rxq->ring.rsp_cons + 1; 1796 if (RING_HAS_UNCONSUMED_RESPONSES(&rxq->ring)) 1797 xn_rxeof(rxq); 1798 XN_RX_UNLOCK(rxq); 1799 } 1800 1801 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); 1802 if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); 1803 if_link_state_change(ifp, LINK_STATE_UP); 1804 } 1805 1806 static void 1807 xn_ifinit(void *xsc) 1808 { 1809 struct netfront_info *sc = xsc; 1810 1811 XN_LOCK(sc); 1812 xn_ifinit_locked(sc); 1813 XN_UNLOCK(sc); 1814 } 1815 1816 static int 1817 xn_ioctl(if_t ifp, u_long cmd, caddr_t data) 1818 { 1819 struct netfront_info *sc = if_getsoftc(ifp); 1820 struct ifreq *ifr = (struct ifreq *) data; 1821 device_t dev; 1822 #ifdef INET 1823 struct ifaddr *ifa = (struct ifaddr *)data; 1824 #endif 1825 int mask, error = 0, reinit; 1826 1827 dev = sc->xbdev; 1828 1829 switch(cmd) { 1830 case SIOCSIFADDR: 1831 #ifdef INET 1832 XN_LOCK(sc); 1833 if (ifa->ifa_addr->sa_family == AF_INET) { 1834 if_setflagbits(ifp, IFF_UP, 0); 1835 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) 1836 xn_ifinit_locked(sc); 1837 arp_ifinit(ifp, ifa); 1838 XN_UNLOCK(sc); 1839 } else { 1840 XN_UNLOCK(sc); 1841 #endif 1842 error = ether_ioctl(ifp, cmd, data); 1843 #ifdef INET 1844 } 1845 #endif 1846 break; 1847 case SIOCSIFMTU: 1848 if (if_getmtu(ifp) == ifr->ifr_mtu) 1849 break; 1850 1851 if_setmtu(ifp, ifr->ifr_mtu); 1852 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); 1853 xn_ifinit(sc); 1854 break; 1855 case SIOCSIFFLAGS: 1856 XN_LOCK(sc); 1857 if (if_getflags(ifp) & IFF_UP) { 1858 /* 1859 * If only the state of the PROMISC flag changed, 1860 * then just use the 'set promisc mode' command 1861 * instead of reinitializing the entire NIC. Doing 1862 * a full re-init means reloading the firmware and 1863 * waiting for it to start up, which may take a 1864 * second or two. 1865 */ 1866 xn_ifinit_locked(sc); 1867 } else { 1868 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { 1869 xn_stop(sc); 1870 } 1871 } 1872 sc->xn_if_flags = if_getflags(ifp); 1873 XN_UNLOCK(sc); 1874 break; 1875 case SIOCSIFCAP: 1876 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); 1877 reinit = 0; 1878 1879 if (mask & IFCAP_TXCSUM) { 1880 if_togglecapenable(ifp, IFCAP_TXCSUM); 1881 if_togglehwassist(ifp, XN_CSUM_FEATURES); 1882 } 1883 if (mask & IFCAP_TSO4) { 1884 if_togglecapenable(ifp, IFCAP_TSO4); 1885 if_togglehwassist(ifp, CSUM_TSO); 1886 } 1887 1888 if (mask & (IFCAP_RXCSUM | IFCAP_LRO)) { 1889 /* These Rx features require us to renegotiate. */ 1890 reinit = 1; 1891 1892 if (mask & IFCAP_RXCSUM) 1893 if_togglecapenable(ifp, IFCAP_RXCSUM); 1894 if (mask & IFCAP_LRO) 1895 if_togglecapenable(ifp, IFCAP_LRO); 1896 } 1897 1898 if (reinit == 0) 1899 break; 1900 1901 /* 1902 * We must reset the interface so the backend picks up the 1903 * new features. 1904 */ 1905 device_printf(sc->xbdev, 1906 "performing interface reset due to feature change\n"); 1907 XN_LOCK(sc); 1908 netfront_carrier_off(sc); 1909 sc->xn_reset = true; 1910 /* 1911 * NB: the pending packet queue is not flushed, since 1912 * the interface should still support the old options. 1913 */ 1914 XN_UNLOCK(sc); 1915 /* 1916 * Delete the xenstore nodes that export features. 1917 * 1918 * NB: There's a xenbus state called 1919 * "XenbusStateReconfiguring", which is what we should set 1920 * here. Sadly none of the backends know how to handle it, 1921 * and simply disconnect from the frontend, so we will just 1922 * switch back to XenbusStateInitialising in order to force 1923 * a reconnection. 1924 */ 1925 xs_rm(XST_NIL, xenbus_get_node(dev), "feature-gso-tcpv4"); 1926 xs_rm(XST_NIL, xenbus_get_node(dev), "feature-no-csum-offload"); 1927 xenbus_set_state(dev, XenbusStateClosing); 1928 1929 /* 1930 * Wait for the frontend to reconnect before returning 1931 * from the ioctl. 30s should be more than enough for any 1932 * sane backend to reconnect. 1933 */ 1934 error = tsleep(sc, 0, "xn_rst", 30*hz); 1935 break; 1936 case SIOCADDMULTI: 1937 case SIOCDELMULTI: 1938 break; 1939 case SIOCSIFMEDIA: 1940 case SIOCGIFMEDIA: 1941 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 1942 break; 1943 default: 1944 error = ether_ioctl(ifp, cmd, data); 1945 } 1946 1947 return (error); 1948 } 1949 1950 static void 1951 xn_stop(struct netfront_info *sc) 1952 { 1953 if_t ifp; 1954 1955 XN_LOCK_ASSERT(sc); 1956 1957 ifp = sc->xn_ifp; 1958 1959 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1960 if_link_state_change(ifp, LINK_STATE_DOWN); 1961 } 1962 1963 static void 1964 xn_rebuild_rx_bufs(struct netfront_rxq *rxq) 1965 { 1966 int requeue_idx, i; 1967 grant_ref_t ref; 1968 netif_rx_request_t *req; 1969 1970 for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) { 1971 struct mbuf *m; 1972 u_long pfn; 1973 1974 if (rxq->mbufs[i] == NULL) 1975 continue; 1976 1977 m = rxq->mbufs[requeue_idx] = xn_get_rx_mbuf(rxq, i); 1978 ref = rxq->grant_ref[requeue_idx] = xn_get_rx_ref(rxq, i); 1979 1980 req = RING_GET_REQUEST(&rxq->ring, requeue_idx); 1981 pfn = vtophys(mtod(m, vm_offset_t)) >> PAGE_SHIFT; 1982 1983 gnttab_grant_foreign_access_ref(ref, 1984 xenbus_get_otherend_id(rxq->info->xbdev), 1985 pfn, 0); 1986 1987 req->gref = ref; 1988 req->id = requeue_idx; 1989 1990 requeue_idx++; 1991 } 1992 1993 rxq->ring.req_prod_pvt = requeue_idx; 1994 } 1995 1996 /* START of Xenolinux helper functions adapted to FreeBSD */ 1997 static int 1998 xn_connect(struct netfront_info *np) 1999 { 2000 int i, error; 2001 u_int feature_rx_copy; 2002 struct netfront_rxq *rxq; 2003 struct netfront_txq *txq; 2004 2005 error = xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev), 2006 "feature-rx-copy", NULL, "%u", &feature_rx_copy); 2007 if (error != 0) 2008 feature_rx_copy = 0; 2009 2010 /* We only support rx copy. */ 2011 if (!feature_rx_copy) 2012 return (EPROTONOSUPPORT); 2013 2014 /* Recovery procedure: */ 2015 error = talk_to_backend(np->xbdev, np); 2016 if (error != 0) 2017 return (error); 2018 2019 /* Step 1: Reinitialise variables. */ 2020 xn_query_features(np); 2021 xn_configure_features(np); 2022 2023 /* Step 2: Release TX buffer */ 2024 for (i = 0; i < np->num_queues; i++) { 2025 txq = &np->txq[i]; 2026 xn_release_tx_bufs(txq); 2027 } 2028 2029 /* Step 3: Rebuild the RX buffer freelist and the RX ring itself. */ 2030 for (i = 0; i < np->num_queues; i++) { 2031 rxq = &np->rxq[i]; 2032 xn_rebuild_rx_bufs(rxq); 2033 } 2034 2035 /* Step 4: All public and private state should now be sane. Get 2036 * ready to start sending and receiving packets and give the driver 2037 * domain a kick because we've probably just requeued some 2038 * packets. 2039 */ 2040 netfront_carrier_on(np); 2041 wakeup(np); 2042 2043 return (0); 2044 } 2045 2046 static void 2047 xn_kick_rings(struct netfront_info *np) 2048 { 2049 struct netfront_rxq *rxq; 2050 struct netfront_txq *txq; 2051 int i; 2052 2053 for (i = 0; i < np->num_queues; i++) { 2054 txq = &np->txq[i]; 2055 rxq = &np->rxq[i]; 2056 xen_intr_signal(txq->xen_intr_handle); 2057 XN_TX_LOCK(txq); 2058 xn_txeof(txq); 2059 XN_TX_UNLOCK(txq); 2060 XN_RX_LOCK(rxq); 2061 xn_alloc_rx_buffers(rxq); 2062 XN_RX_UNLOCK(rxq); 2063 } 2064 } 2065 2066 static void 2067 xn_query_features(struct netfront_info *np) 2068 { 2069 int val; 2070 2071 device_printf(np->xbdev, "backend features:"); 2072 2073 if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev), 2074 "feature-sg", NULL, "%d", &val) != 0) 2075 val = 0; 2076 2077 np->maxfrags = 1; 2078 if (val) { 2079 np->maxfrags = MAX_TX_REQ_FRAGS; 2080 printf(" feature-sg"); 2081 } 2082 2083 if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev), 2084 "feature-gso-tcpv4", NULL, "%d", &val) != 0) 2085 val = 0; 2086 2087 if_setcapabilitiesbit(np->xn_ifp, 0, IFCAP_TSO4 | IFCAP_LRO); 2088 if (val) { 2089 if_setcapabilitiesbit(np->xn_ifp, IFCAP_TSO4 | IFCAP_LRO, 0); 2090 printf(" feature-gso-tcp4"); 2091 } 2092 2093 /* 2094 * HW CSUM offload is assumed to be available unless 2095 * feature-no-csum-offload is set in xenstore. 2096 */ 2097 if (xs_scanf(XST_NIL, xenbus_get_otherend_path(np->xbdev), 2098 "feature-no-csum-offload", NULL, "%d", &val) != 0) 2099 val = 0; 2100 2101 if_setcapabilitiesbit(np->xn_ifp, IFCAP_HWCSUM, 0); 2102 if (val) { 2103 if_setcapabilitiesbit(np->xn_ifp, 0, IFCAP_HWCSUM); 2104 printf(" feature-no-csum-offload"); 2105 } 2106 2107 printf("\n"); 2108 } 2109 2110 static int 2111 xn_configure_features(struct netfront_info *np) 2112 { 2113 int err, cap_enabled; 2114 #if (defined(INET) || defined(INET6)) 2115 int i; 2116 #endif 2117 if_t ifp; 2118 2119 ifp = np->xn_ifp; 2120 err = 0; 2121 2122 if ((if_getcapenable(ifp) & if_getcapabilities(ifp)) == if_getcapenable(ifp)) { 2123 /* Current options are available, no need to do anything. */ 2124 return (0); 2125 } 2126 2127 /* Try to preserve as many options as possible. */ 2128 cap_enabled = if_getcapenable(ifp); 2129 if_setcapenable(ifp, 0); 2130 if_sethwassist(ifp, 0); 2131 2132 #if (defined(INET) || defined(INET6)) 2133 if ((cap_enabled & IFCAP_LRO) != 0) 2134 for (i = 0; i < np->num_queues; i++) 2135 tcp_lro_free(&np->rxq[i].lro); 2136 if (xn_enable_lro && 2137 (if_getcapabilities(ifp) & cap_enabled & IFCAP_LRO) != 0) { 2138 if_setcapenablebit(ifp, IFCAP_LRO, 0); 2139 for (i = 0; i < np->num_queues; i++) { 2140 err = tcp_lro_init(&np->rxq[i].lro); 2141 if (err != 0) { 2142 device_printf(np->xbdev, 2143 "LRO initialization failed\n"); 2144 if_setcapenablebit(ifp, 0, IFCAP_LRO); 2145 break; 2146 } 2147 np->rxq[i].lro.ifp = ifp; 2148 } 2149 } 2150 if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_TSO4) != 0) { 2151 if_setcapenablebit(ifp, IFCAP_TSO4, 0); 2152 if_sethwassistbits(ifp, CSUM_TSO, 0); 2153 } 2154 #endif 2155 if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_TXCSUM) != 0) { 2156 if_setcapenablebit(ifp, IFCAP_TXCSUM, 0); 2157 if_sethwassistbits(ifp, XN_CSUM_FEATURES, 0); 2158 } 2159 if ((if_getcapabilities(ifp) & cap_enabled & IFCAP_RXCSUM) != 0) 2160 if_setcapenablebit(ifp, IFCAP_RXCSUM, 0); 2161 2162 return (err); 2163 } 2164 2165 static int 2166 xn_txq_mq_start_locked(struct netfront_txq *txq, struct mbuf *m) 2167 { 2168 struct netfront_info *np; 2169 if_t ifp; 2170 struct buf_ring *br; 2171 int error, notify; 2172 2173 np = txq->info; 2174 br = txq->br; 2175 ifp = np->xn_ifp; 2176 error = 0; 2177 2178 XN_TX_LOCK_ASSERT(txq); 2179 2180 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0 || 2181 !netfront_carrier_ok(np)) { 2182 if (m != NULL) 2183 error = drbr_enqueue(ifp, br, m); 2184 return (error); 2185 } 2186 2187 if (m != NULL) { 2188 error = drbr_enqueue(ifp, br, m); 2189 if (error != 0) 2190 return (error); 2191 } 2192 2193 while ((m = drbr_peek(ifp, br)) != NULL) { 2194 if (!xn_tx_slot_available(txq)) { 2195 drbr_putback(ifp, br, m); 2196 break; 2197 } 2198 2199 error = xn_assemble_tx_request(txq, m); 2200 /* xn_assemble_tx_request always consumes the mbuf*/ 2201 if (error != 0) { 2202 drbr_advance(ifp, br); 2203 break; 2204 } 2205 2206 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&txq->ring, notify); 2207 if (notify) 2208 xen_intr_signal(txq->xen_intr_handle); 2209 2210 drbr_advance(ifp, br); 2211 } 2212 2213 if (RING_FULL(&txq->ring)) 2214 txq->full = true; 2215 2216 return (0); 2217 } 2218 2219 static int 2220 xn_txq_mq_start(if_t ifp, struct mbuf *m) 2221 { 2222 struct netfront_info *np; 2223 struct netfront_txq *txq; 2224 int i, npairs, error; 2225 2226 np = if_getsoftc(ifp); 2227 npairs = np->num_queues; 2228 2229 if (!netfront_carrier_ok(np)) 2230 return (ENOBUFS); 2231 2232 KASSERT(npairs != 0, ("called with 0 available queues")); 2233 2234 /* check if flowid is set */ 2235 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 2236 i = m->m_pkthdr.flowid % npairs; 2237 else 2238 i = curcpu % npairs; 2239 2240 txq = &np->txq[i]; 2241 2242 if (XN_TX_TRYLOCK(txq) != 0) { 2243 error = xn_txq_mq_start_locked(txq, m); 2244 XN_TX_UNLOCK(txq); 2245 } else { 2246 error = drbr_enqueue(ifp, txq->br, m); 2247 taskqueue_enqueue(txq->tq, &txq->defrtask); 2248 } 2249 2250 return (error); 2251 } 2252 2253 static void 2254 xn_qflush(if_t ifp) 2255 { 2256 struct netfront_info *np; 2257 struct netfront_txq *txq; 2258 struct mbuf *m; 2259 int i; 2260 2261 np = if_getsoftc(ifp); 2262 2263 for (i = 0; i < np->num_queues; i++) { 2264 txq = &np->txq[i]; 2265 2266 XN_TX_LOCK(txq); 2267 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 2268 m_freem(m); 2269 XN_TX_UNLOCK(txq); 2270 } 2271 2272 if_qflush(ifp); 2273 } 2274 2275 /** 2276 * Create a network device. 2277 * @param dev Newbus device representing this virtual NIC. 2278 */ 2279 int 2280 create_netdev(device_t dev) 2281 { 2282 struct netfront_info *np; 2283 int err, cap_enabled; 2284 if_t ifp; 2285 2286 np = device_get_softc(dev); 2287 2288 np->xbdev = dev; 2289 2290 mtx_init(&np->sc_lock, "xnsc", "netfront softc lock", MTX_DEF); 2291 2292 ifmedia_init(&np->sc_media, 0, xn_ifmedia_upd, xn_ifmedia_sts); 2293 ifmedia_add(&np->sc_media, IFM_ETHER|IFM_MANUAL, 0, NULL); 2294 ifmedia_set(&np->sc_media, IFM_ETHER|IFM_MANUAL); 2295 2296 err = xen_net_read_mac(dev, np->mac); 2297 if (err != 0) 2298 goto error; 2299 2300 /* Set up ifnet structure */ 2301 ifp = np->xn_ifp = if_alloc(IFT_ETHER); 2302 if_setsoftc(ifp, np); 2303 if_initname(ifp, "xn", device_get_unit(dev)); 2304 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2305 if_setioctlfn(ifp, xn_ioctl); 2306 2307 if_settransmitfn(ifp, xn_txq_mq_start); 2308 if_setqflushfn(ifp, xn_qflush); 2309 2310 if_setinitfn(ifp, xn_ifinit); 2311 2312 if_sethwassist(ifp, XN_CSUM_FEATURES); 2313 /* Enable all supported features at device creation. */ 2314 if_setcapabilities(ifp, IFCAP_HWCSUM|IFCAP_TSO4|IFCAP_LRO); 2315 cap_enabled = if_getcapabilities(ifp); 2316 if (!xn_enable_lro) { 2317 cap_enabled &= ~IFCAP_LRO; 2318 } 2319 if_setcapenable(ifp, cap_enabled); 2320 2321 if_sethwtsomax(ifp, 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); 2322 if_sethwtsomaxsegcount(ifp, MAX_TX_REQ_FRAGS); 2323 if_sethwtsomaxsegsize(ifp, PAGE_SIZE); 2324 2325 ether_ifattach(ifp, np->mac); 2326 netfront_carrier_off(np); 2327 2328 err = bus_dma_tag_create( 2329 bus_get_dma_tag(dev), /* parent */ 2330 1, PAGE_SIZE, /* algnmnt, boundary */ 2331 BUS_SPACE_MAXADDR, /* lowaddr */ 2332 BUS_SPACE_MAXADDR, /* highaddr */ 2333 NULL, NULL, /* filter, filterarg */ 2334 PAGE_SIZE * MAX_TX_REQ_FRAGS, /* max request size */ 2335 MAX_TX_REQ_FRAGS, /* max segments */ 2336 PAGE_SIZE, /* maxsegsize */ 2337 BUS_DMA_ALLOCNOW, /* flags */ 2338 NULL, NULL, /* lockfunc, lockarg */ 2339 &np->dma_tag); 2340 2341 return (err); 2342 2343 error: 2344 KASSERT(err != 0, ("Error path with no error code specified")); 2345 return (err); 2346 } 2347 2348 static int 2349 netfront_detach(device_t dev) 2350 { 2351 struct netfront_info *info = device_get_softc(dev); 2352 2353 DPRINTK("%s\n", xenbus_get_node(dev)); 2354 2355 netif_free(info); 2356 2357 return 0; 2358 } 2359 2360 static void 2361 netif_free(struct netfront_info *np) 2362 { 2363 2364 XN_LOCK(np); 2365 xn_stop(np); 2366 XN_UNLOCK(np); 2367 netif_disconnect_backend(np); 2368 ether_ifdetach(np->xn_ifp); 2369 free(np->rxq, M_DEVBUF); 2370 free(np->txq, M_DEVBUF); 2371 if_free(np->xn_ifp); 2372 np->xn_ifp = NULL; 2373 ifmedia_removeall(&np->sc_media); 2374 bus_dma_tag_destroy(np->dma_tag); 2375 } 2376 2377 static void 2378 netif_disconnect_backend(struct netfront_info *np) 2379 { 2380 u_int i; 2381 2382 for (i = 0; i < np->num_queues; i++) { 2383 XN_RX_LOCK(&np->rxq[i]); 2384 XN_TX_LOCK(&np->txq[i]); 2385 } 2386 netfront_carrier_off(np); 2387 for (i = 0; i < np->num_queues; i++) { 2388 XN_RX_UNLOCK(&np->rxq[i]); 2389 XN_TX_UNLOCK(&np->txq[i]); 2390 } 2391 2392 for (i = 0; i < np->num_queues; i++) { 2393 disconnect_rxq(&np->rxq[i]); 2394 disconnect_txq(&np->txq[i]); 2395 } 2396 } 2397 2398 static int 2399 xn_ifmedia_upd(if_t ifp) 2400 { 2401 2402 return (0); 2403 } 2404 2405 static void 2406 xn_ifmedia_sts(if_t ifp, struct ifmediareq *ifmr) 2407 { 2408 2409 ifmr->ifm_status = IFM_AVALID|IFM_ACTIVE; 2410 ifmr->ifm_active = IFM_ETHER|IFM_MANUAL; 2411 } 2412 2413 /* ** Driver registration ** */ 2414 static device_method_t netfront_methods[] = { 2415 /* Device interface */ 2416 DEVMETHOD(device_probe, netfront_probe), 2417 DEVMETHOD(device_attach, netfront_attach), 2418 DEVMETHOD(device_detach, netfront_detach), 2419 DEVMETHOD(device_shutdown, bus_generic_shutdown), 2420 DEVMETHOD(device_suspend, netfront_suspend), 2421 DEVMETHOD(device_resume, netfront_resume), 2422 2423 /* Xenbus interface */ 2424 DEVMETHOD(xenbus_otherend_changed, netfront_backend_changed), 2425 2426 DEVMETHOD_END 2427 }; 2428 2429 static driver_t netfront_driver = { 2430 "xn", 2431 netfront_methods, 2432 sizeof(struct netfront_info), 2433 }; 2434 2435 DRIVER_MODULE(xe, xenbusb_front, netfront_driver, NULL, NULL); 2436