1 /* 2 * Copyright (C) 2014-2018 Vincenzo Maffione, Luigi Rizzo. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 */ 29 30 #include <net/netmap.h> 31 #include <sys/selinfo.h> 32 #include <vm/vm.h> 33 #include <vm/pmap.h> /* vtophys ? */ 34 #include <dev/netmap/netmap_kern.h> 35 36 /* 37 * Return 1 if the queue identified by 't' and 'idx' is in netmap mode. 38 */ 39 static int 40 vtnet_netmap_queue_on(struct vtnet_softc *sc, enum txrx t, int idx) 41 { 42 struct netmap_adapter *na = NA(sc->vtnet_ifp); 43 44 if (!nm_native_on(na)) 45 return 0; 46 47 if (t == NR_RX) 48 return !!(idx < na->num_rx_rings && 49 na->rx_rings[idx]->nr_mode == NKR_NETMAP_ON); 50 51 return !!(idx < na->num_tx_rings && 52 na->tx_rings[idx]->nr_mode == NKR_NETMAP_ON); 53 } 54 55 static void 56 vtnet_free_used(struct virtqueue *vq, int netmap_bufs, enum txrx t, int idx) 57 { 58 void *cookie; 59 int deq = 0; 60 61 while ((cookie = virtqueue_dequeue(vq, NULL)) != NULL) { 62 if (netmap_bufs) { 63 /* These are netmap buffers: there is nothing to do. */ 64 } else { 65 /* These are mbufs that we need to free. */ 66 struct mbuf *m; 67 68 if (t == NR_TX) { 69 struct vtnet_tx_header *txhdr = cookie; 70 m = txhdr->vth_mbuf; 71 m_freem(m); 72 uma_zfree(vtnet_tx_header_zone, txhdr); 73 } else { 74 m = cookie; 75 m_freem(m); 76 } 77 } 78 deq++; 79 } 80 81 if (deq) 82 nm_prinf("%d sgs dequeued from %s-%d (netmap=%d)", 83 deq, nm_txrx2str(t), idx, netmap_bufs); 84 } 85 86 /* Register and unregister. */ 87 static int 88 vtnet_netmap_reg(struct netmap_adapter *na, int state) 89 { 90 struct ifnet *ifp = na->ifp; 91 struct vtnet_softc *sc = ifp->if_softc; 92 int success; 93 int i; 94 95 /* Drain the taskqueues to make sure that there are no worker threads 96 * accessing the virtqueues. */ 97 vtnet_drain_taskqueues(sc); 98 99 VTNET_CORE_LOCK(sc); 100 101 /* We need nm_netmap_on() to return true when called by 102 * vtnet_init_locked() below. */ 103 if (state) 104 nm_set_native_flags(na); 105 106 /* We need to trigger a device reset in order to unexpose guest buffers 107 * published to the host. */ 108 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 109 /* Get pending used buffers. The way they are freed depends on whether 110 * they are netmap buffer or they are mbufs. We can tell apart the two 111 * cases by looking at kring->nr_mode, before this is possibly updated 112 * in the loop below. */ 113 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 114 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 115 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 116 struct netmap_kring *kring; 117 118 VTNET_TXQ_LOCK(txq); 119 kring = NMR(na, NR_TX)[i]; 120 vtnet_free_used(txq->vtntx_vq, 121 kring->nr_mode == NKR_NETMAP_ON, NR_TX, i); 122 VTNET_TXQ_UNLOCK(txq); 123 124 VTNET_RXQ_LOCK(rxq); 125 kring = NMR(na, NR_RX)[i]; 126 vtnet_free_used(rxq->vtnrx_vq, 127 kring->nr_mode == NKR_NETMAP_ON, NR_RX, i); 128 VTNET_RXQ_UNLOCK(rxq); 129 } 130 vtnet_init_locked(sc); 131 success = (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO; 132 133 if (state) { 134 netmap_krings_mode_commit(na, state); 135 } else { 136 nm_clear_native_flags(na); 137 netmap_krings_mode_commit(na, state); 138 } 139 140 VTNET_CORE_UNLOCK(sc); 141 142 return success; 143 } 144 145 146 /* Reconcile kernel and user view of the transmit ring. */ 147 static int 148 vtnet_netmap_txsync(struct netmap_kring *kring, int flags) 149 { 150 struct netmap_adapter *na = kring->na; 151 struct ifnet *ifp = na->ifp; 152 struct netmap_ring *ring = kring->ring; 153 u_int ring_nr = kring->ring_id; 154 u_int nm_i; /* index into the netmap ring */ 155 u_int const lim = kring->nkr_num_slots - 1; 156 u_int const head = kring->rhead; 157 158 /* device-specific */ 159 struct vtnet_softc *sc = ifp->if_softc; 160 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr]; 161 struct virtqueue *vq = txq->vtntx_vq; 162 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 163 u_int n; 164 165 /* 166 * First part: process new packets to send. 167 */ 168 rmb(); 169 170 nm_i = kring->nr_hwcur; 171 if (nm_i != head) { /* we have new packets to send */ 172 struct sglist *sg = txq->vtntx_sg; 173 174 for (; nm_i != head; nm_i = nm_next(nm_i, lim)) { 175 /* we use an empty header here */ 176 struct netmap_slot *slot = &ring->slot[nm_i]; 177 u_int len = slot->len; 178 uint64_t paddr; 179 void *addr = PNMB(na, slot, &paddr); 180 int err; 181 182 NM_CHECK_ADDR_LEN(na, addr, len); 183 184 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 185 /* Initialize the scatterlist, expose it to the hypervisor, 186 * and kick the hypervisor (if necessary). 187 */ 188 sglist_reset(sg); // cheap 189 err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size); 190 err |= sglist_append_phys(sg, paddr, len); 191 KASSERT(err == 0, ("%s: cannot append to sglist %d", 192 __func__, err)); 193 err = virtqueue_enqueue(vq, /*cookie=*/txq, sg, 194 /*readable=*/sg->sg_nseg, 195 /*writeable=*/0); 196 if (unlikely(err)) { 197 if (err != ENOSPC) 198 nm_prerr("virtqueue_enqueue(%s) failed: %d", 199 kring->name, err); 200 break; 201 } 202 } 203 204 virtqueue_notify(vq); 205 206 /* Update hwcur depending on where we stopped. */ 207 kring->nr_hwcur = nm_i; /* note we migth break early */ 208 } 209 210 /* Free used slots. We only consider our own used buffers, recognized 211 * by the token we passed to virtqueue_enqueue. 212 */ 213 n = 0; 214 for (;;) { 215 void *token = virtqueue_dequeue(vq, NULL); 216 if (token == NULL) 217 break; 218 if (unlikely(token != (void *)txq)) 219 nm_prerr("BUG: TX token mismatch"); 220 else 221 n++; 222 } 223 if (n > 0) { 224 kring->nr_hwtail += n; 225 if (kring->nr_hwtail > lim) 226 kring->nr_hwtail -= lim + 1; 227 } 228 229 if (interrupts && virtqueue_nfree(vq) < 32) 230 virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG); 231 232 return 0; 233 } 234 235 /* 236 * Publish (up to) num netmap receive buffers to the host, 237 * starting from the first one that the user made available 238 * (kring->nr_hwcur). 239 */ 240 static int 241 vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num) 242 { 243 struct netmap_adapter *na = kring->na; 244 struct ifnet *ifp = na->ifp; 245 struct netmap_ring *ring = kring->ring; 246 u_int ring_nr = kring->ring_id; 247 u_int const lim = kring->nkr_num_slots - 1; 248 u_int nm_i = kring->nr_hwcur; 249 250 /* device-specific */ 251 struct vtnet_softc *sc = ifp->if_softc; 252 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 253 struct virtqueue *vq = rxq->vtnrx_vq; 254 255 /* use a local sglist, default might be short */ 256 struct sglist_seg ss[2]; 257 struct sglist sg = { ss, 0, 0, 2 }; 258 259 for (; num > 0; nm_i = nm_next(nm_i, lim), num--) { 260 struct netmap_slot *slot = &ring->slot[nm_i]; 261 uint64_t paddr; 262 void *addr = PNMB(na, slot, &paddr); 263 int err; 264 265 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */ 266 if (netmap_ring_reinit(kring)) 267 return -1; 268 } 269 270 slot->flags &= ~NS_BUF_CHANGED; 271 sglist_reset(&sg); 272 err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size); 273 err |= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na)); 274 KASSERT(err == 0, ("%s: cannot append to sglist %d", 275 __func__, err)); 276 /* writable for the host */ 277 err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg, 278 /*readable=*/0, /*writeable=*/sg.sg_nseg); 279 if (unlikely(err)) { 280 if (err != ENOSPC) 281 nm_prerr("virtqueue_enqueue(%s) failed: %d", 282 kring->name, err); 283 break; 284 } 285 } 286 287 return nm_i; 288 } 289 290 /* 291 * Publish netmap buffers on a RX virtqueue. 292 * Returns -1 if this virtqueue is not being opened in netmap mode. 293 * If the virtqueue is being opened in netmap mode, return 0 on success and 294 * a positive error code on failure. 295 */ 296 static int 297 vtnet_netmap_rxq_populate(struct vtnet_rxq *rxq) 298 { 299 struct netmap_adapter *na = NA(rxq->vtnrx_sc->vtnet_ifp); 300 struct netmap_kring *kring; 301 int error; 302 303 if (!nm_native_on(na) || rxq->vtnrx_id >= na->num_rx_rings) 304 return -1; 305 306 kring = na->rx_rings[rxq->vtnrx_id]; 307 if (!(nm_kring_pending_on(kring) || 308 kring->nr_pending_mode == NKR_NETMAP_ON)) 309 return -1; 310 311 /* Expose all the RX netmap buffers we can. In case of no indirect 312 * buffers, the number of netmap slots in the RX ring matches the 313 * maximum number of 2-elements sglist that the RX virtqueue can 314 * accommodate. */ 315 error = vtnet_netmap_kring_refill(kring, na->num_rx_desc); 316 virtqueue_notify(rxq->vtnrx_vq); 317 318 return error < 0 ? ENXIO : 0; 319 } 320 321 /* Reconcile kernel and user view of the receive ring. */ 322 static int 323 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags) 324 { 325 struct netmap_adapter *na = kring->na; 326 struct ifnet *ifp = na->ifp; 327 struct netmap_ring *ring = kring->ring; 328 u_int ring_nr = kring->ring_id; 329 u_int nm_i; /* index into the netmap ring */ 330 u_int const lim = kring->nkr_num_slots - 1; 331 u_int const head = kring->rhead; 332 int force_update = (flags & NAF_FORCE_READ) || 333 (kring->nr_kflags & NKR_PENDINTR); 334 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 335 336 /* device-specific */ 337 struct vtnet_softc *sc = ifp->if_softc; 338 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 339 struct virtqueue *vq = rxq->vtnrx_vq; 340 341 rmb(); 342 /* 343 * First part: import newly received packets. 344 * Only accept our own buffers (matching the token). We should only get 345 * matching buffers. We may need to stop early to avoid hwtail to overrun 346 * hwcur. 347 */ 348 if (netmap_no_pendintr || force_update) { 349 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); 350 void *token; 351 352 vtnet_rxq_disable_intr(rxq); 353 354 nm_i = kring->nr_hwtail; 355 while (nm_i != hwtail_lim) { 356 int len; 357 token = virtqueue_dequeue(vq, &len); 358 if (token == NULL) { 359 if (interrupts && vtnet_rxq_enable_intr(rxq)) { 360 vtnet_rxq_disable_intr(rxq); 361 continue; 362 } 363 break; 364 } 365 if (unlikely(token != (void *)rxq)) { 366 nm_prerr("BUG: RX token mismatch"); 367 } else { 368 /* Skip the virtio-net header. */ 369 len -= sc->vtnet_hdr_size; 370 if (unlikely(len < 0)) { 371 nm_prlim(1, "Truncated virtio-net-header, " 372 "missing %d bytes", -len); 373 len = 0; 374 } 375 ring->slot[nm_i].len = len; 376 ring->slot[nm_i].flags = 0; 377 nm_i = nm_next(nm_i, lim); 378 } 379 } 380 kring->nr_hwtail = nm_i; 381 kring->nr_kflags &= ~NKR_PENDINTR; 382 } 383 nm_prdis("[B] h %d c %d hwcur %d hwtail %d", ring->head, ring->cur, 384 kring->nr_hwcur, kring->nr_hwtail); 385 386 /* 387 * Second part: skip past packets that userspace has released. 388 */ 389 nm_i = kring->nr_hwcur; /* netmap ring index */ 390 if (nm_i != head) { 391 int howmany = head - nm_i; 392 int nm_j; 393 394 if (howmany < 0) 395 howmany += kring->nkr_num_slots; 396 nm_j = vtnet_netmap_kring_refill(kring, howmany); 397 if (nm_j < 0) 398 return nm_j; 399 kring->nr_hwcur = nm_j; 400 virtqueue_notify(vq); 401 } 402 403 nm_prdis("[C] h %d c %d t %d hwcur %d hwtail %d", ring->head, ring->cur, 404 ring->tail, kring->nr_hwcur, kring->nr_hwtail); 405 406 return 0; 407 } 408 409 410 /* Enable/disable interrupts on all virtqueues. */ 411 static void 412 vtnet_netmap_intr(struct netmap_adapter *na, int state) 413 { 414 struct vtnet_softc *sc = na->ifp->if_softc; 415 int i; 416 417 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 418 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 419 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 420 struct virtqueue *txvq = txq->vtntx_vq; 421 422 if (state) { 423 vtnet_rxq_enable_intr(rxq); 424 virtqueue_enable_intr(txvq); 425 } else { 426 vtnet_rxq_disable_intr(rxq); 427 virtqueue_disable_intr(txvq); 428 } 429 } 430 } 431 432 static int 433 vtnet_netmap_tx_slots(struct vtnet_softc *sc) 434 { 435 int div; 436 437 /* We need to prepend a virtio-net header to each netmap buffer to be 438 * transmitted, therefore calling virtqueue_enqueue() passing sglist 439 * with 2 elements. 440 * TX virtqueues use indirect descriptors if the feature was negotiated 441 * with the host, and if sc->vtnet_tx_nsegs > 1. With indirect 442 * descriptors, a single virtio descriptor is sufficient to reference 443 * each TX sglist. Without them, we need two separate virtio descriptors 444 * for each TX sglist. We therefore compute the number of netmap TX 445 * slots according to these assumptions. 446 */ 447 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_tx_nsegs > 1) 448 div = 1; 449 else 450 div = 2; 451 452 return virtqueue_size(sc->vtnet_txqs[0].vtntx_vq) / div; 453 } 454 455 static int 456 vtnet_netmap_rx_slots(struct vtnet_softc *sc) 457 { 458 int div; 459 460 /* We need to prepend a virtio-net header to each netmap buffer to be 461 * received, therefore calling virtqueue_enqueue() passing sglist 462 * with 2 elements. 463 * RX virtqueues use indirect descriptors if the feature was negotiated 464 * with the host, and if sc->vtnet_rx_nsegs > 1. With indirect 465 * descriptors, a single virtio descriptor is sufficient to reference 466 * each RX sglist. Without them, we need two separate virtio descriptors 467 * for each RX sglist. We therefore compute the number of netmap RX 468 * slots according to these assumptions. 469 */ 470 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_rx_nsegs > 1) 471 div = 1; 472 else 473 div = 2; 474 475 return virtqueue_size(sc->vtnet_rxqs[0].vtnrx_vq) / div; 476 } 477 478 static int 479 vtnet_netmap_config(struct netmap_adapter *na, struct nm_config_info *info) 480 { 481 struct vtnet_softc *sc = na->ifp->if_softc; 482 483 info->num_tx_rings = sc->vtnet_act_vq_pairs; 484 info->num_rx_rings = sc->vtnet_act_vq_pairs; 485 info->num_tx_descs = vtnet_netmap_tx_slots(sc); 486 info->num_rx_descs = vtnet_netmap_rx_slots(sc); 487 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na); 488 489 return 0; 490 } 491 492 static void 493 vtnet_netmap_attach(struct vtnet_softc *sc) 494 { 495 struct netmap_adapter na; 496 497 bzero(&na, sizeof(na)); 498 499 na.ifp = sc->vtnet_ifp; 500 na.na_flags = 0; 501 na.num_tx_desc = vtnet_netmap_tx_slots(sc); 502 na.num_rx_desc = vtnet_netmap_rx_slots(sc); 503 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs; 504 na.rx_buf_maxsize = 0; 505 na.nm_register = vtnet_netmap_reg; 506 na.nm_txsync = vtnet_netmap_txsync; 507 na.nm_rxsync = vtnet_netmap_rxsync; 508 na.nm_intr = vtnet_netmap_intr; 509 na.nm_config = vtnet_netmap_config; 510 511 netmap_attach(&na); 512 513 nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d", 514 na.num_tx_rings, na.num_tx_desc, 515 na.num_tx_rings, na.num_rx_desc); 516 } 517 /* end of file */ 518