1 /* 2 * Copyright (C) 2014-2018 Vincenzo Maffione, Luigi Rizzo. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* 27 * $FreeBSD$ 28 */ 29 30 #include <net/netmap.h> 31 #include <sys/selinfo.h> 32 #include <vm/vm.h> 33 #include <vm/pmap.h> /* vtophys ? */ 34 #include <dev/netmap/netmap_kern.h> 35 36 /* Register and unregister. */ 37 static int 38 vtnet_netmap_reg(struct netmap_adapter *na, int state) 39 { 40 struct ifnet *ifp = na->ifp; 41 struct vtnet_softc *sc = ifp->if_softc; 42 int success; 43 int i; 44 45 /* Drain the taskqueues to make sure that there are no worker threads 46 * accessing the virtqueues. */ 47 vtnet_drain_taskqueues(sc); 48 49 VTNET_CORE_LOCK(sc); 50 51 /* We need nm_netmap_on() to return true when called by 52 * vtnet_init_locked() below. */ 53 if (state) 54 nm_set_native_flags(na); 55 56 /* We need to trigger a device reset in order to unexpose guest buffers 57 * published to the host. */ 58 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 59 /* Get pending used buffers. The way they are freed depends on whether 60 * they are netmap buffer or they are mbufs. We can tell apart the two 61 * cases by looking at kring->nr_mode, before this is possibly updated 62 * in the loop below. */ 63 for (i = 0; i < sc->vtnet_act_vq_pairs; i++) { 64 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 65 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 66 67 VTNET_TXQ_LOCK(txq); 68 vtnet_txq_free_mbufs(txq); 69 VTNET_TXQ_UNLOCK(txq); 70 71 VTNET_RXQ_LOCK(rxq); 72 vtnet_rxq_free_mbufs(rxq); 73 VTNET_RXQ_UNLOCK(rxq); 74 } 75 vtnet_init_locked(sc); 76 success = (ifp->if_drv_flags & IFF_DRV_RUNNING) ? 0 : ENXIO; 77 78 if (state) { 79 netmap_krings_mode_commit(na, state); 80 } else { 81 nm_clear_native_flags(na); 82 netmap_krings_mode_commit(na, state); 83 } 84 85 VTNET_CORE_UNLOCK(sc); 86 87 return success; 88 } 89 90 91 /* Reconcile kernel and user view of the transmit ring. */ 92 static int 93 vtnet_netmap_txsync(struct netmap_kring *kring, int flags) 94 { 95 struct netmap_adapter *na = kring->na; 96 struct ifnet *ifp = na->ifp; 97 struct netmap_ring *ring = kring->ring; 98 u_int ring_nr = kring->ring_id; 99 u_int nm_i; /* index into the netmap ring */ 100 u_int const lim = kring->nkr_num_slots - 1; 101 u_int const head = kring->rhead; 102 103 /* device-specific */ 104 struct vtnet_softc *sc = ifp->if_softc; 105 struct vtnet_txq *txq = &sc->vtnet_txqs[ring_nr]; 106 struct virtqueue *vq = txq->vtntx_vq; 107 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 108 u_int n; 109 110 /* 111 * First part: process new packets to send. 112 */ 113 114 nm_i = kring->nr_hwcur; 115 if (nm_i != head) { /* we have new packets to send */ 116 struct sglist *sg = txq->vtntx_sg; 117 118 for (; nm_i != head; nm_i = nm_next(nm_i, lim)) { 119 /* we use an empty header here */ 120 struct netmap_slot *slot = &ring->slot[nm_i]; 121 u_int len = slot->len; 122 uint64_t paddr; 123 void *addr = PNMB(na, slot, &paddr); 124 int err; 125 126 NM_CHECK_ADDR_LEN(na, addr, len); 127 128 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 129 /* Initialize the scatterlist, expose it to the hypervisor, 130 * and kick the hypervisor (if necessary). 131 */ 132 sglist_reset(sg); // cheap 133 err = sglist_append(sg, &txq->vtntx_shrhdr, sc->vtnet_hdr_size); 134 err |= sglist_append_phys(sg, paddr, len); 135 KASSERT(err == 0, ("%s: cannot append to sglist %d", 136 __func__, err)); 137 err = virtqueue_enqueue(vq, /*cookie=*/txq, sg, 138 /*readable=*/sg->sg_nseg, 139 /*writeable=*/0); 140 if (unlikely(err)) { 141 if (err != ENOSPC) 142 nm_prerr("virtqueue_enqueue(%s) failed: %d", 143 kring->name, err); 144 break; 145 } 146 } 147 148 virtqueue_notify(vq); 149 150 /* Update hwcur depending on where we stopped. */ 151 kring->nr_hwcur = nm_i; /* note we migth break early */ 152 } 153 154 /* Free used slots. We only consider our own used buffers, recognized 155 * by the token we passed to virtqueue_enqueue. 156 */ 157 n = 0; 158 for (;;) { 159 void *token = virtqueue_dequeue(vq, NULL); 160 if (token == NULL) 161 break; 162 if (unlikely(token != (void *)txq)) 163 nm_prerr("BUG: TX token mismatch"); 164 else 165 n++; 166 } 167 if (n > 0) { 168 kring->nr_hwtail += n; 169 if (kring->nr_hwtail > lim) 170 kring->nr_hwtail -= lim + 1; 171 } 172 173 if (interrupts && virtqueue_nfree(vq) < 32) 174 virtqueue_postpone_intr(vq, VQ_POSTPONE_LONG); 175 176 return 0; 177 } 178 179 /* 180 * Publish 'num 'netmap receive buffers to the host, starting 181 * from the next available one (rx->vtnrx_nm_refill). 182 * Return a positive error code on error, and 0 on success. 183 * If we could not publish all of the buffers that's an error, 184 * since the netmap ring and the virtqueue would go out of sync. 185 */ 186 static int 187 vtnet_netmap_kring_refill(struct netmap_kring *kring, u_int num) 188 { 189 struct netmap_adapter *na = kring->na; 190 struct ifnet *ifp = na->ifp; 191 struct netmap_ring *ring = kring->ring; 192 u_int ring_nr = kring->ring_id; 193 u_int const lim = kring->nkr_num_slots - 1; 194 u_int nm_i; 195 196 /* device-specific */ 197 struct vtnet_softc *sc = ifp->if_softc; 198 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 199 struct virtqueue *vq = rxq->vtnrx_vq; 200 201 /* use a local sglist, default might be short */ 202 struct sglist_seg ss[2]; 203 struct sglist sg = { ss, 0, 0, 2 }; 204 205 for (nm_i = rxq->vtnrx_nm_refill; num > 0; 206 nm_i = nm_next(nm_i, lim), num--) { 207 struct netmap_slot *slot = &ring->slot[nm_i]; 208 uint64_t paddr; 209 void *addr = PNMB(na, slot, &paddr); 210 int err; 211 212 if (addr == NETMAP_BUF_BASE(na)) { /* bad buf */ 213 if (netmap_ring_reinit(kring)) 214 return EFAULT; 215 } 216 217 slot->flags &= ~NS_BUF_CHANGED; 218 sglist_reset(&sg); 219 err = sglist_append(&sg, &rxq->vtnrx_shrhdr, sc->vtnet_hdr_size); 220 err |= sglist_append_phys(&sg, paddr, NETMAP_BUF_SIZE(na)); 221 KASSERT(err == 0, ("%s: cannot append to sglist %d", 222 __func__, err)); 223 /* writable for the host */ 224 err = virtqueue_enqueue(vq, /*cookie=*/rxq, &sg, 225 /*readable=*/0, /*writeable=*/sg.sg_nseg); 226 if (unlikely(err)) { 227 nm_prerr("virtqueue_enqueue(%s) failed: %d", 228 kring->name, err); 229 break; 230 } 231 } 232 rxq->vtnrx_nm_refill = nm_i; 233 234 return num == 0 ? 0 : ENOSPC; 235 } 236 237 /* 238 * Publish netmap buffers on a RX virtqueue. 239 * Returns -1 if this virtqueue is not being opened in netmap mode. 240 * If the virtqueue is being opened in netmap mode, return 0 on success and 241 * a positive error code on failure. 242 */ 243 static int 244 vtnet_netmap_rxq_populate(struct vtnet_rxq *rxq) 245 { 246 struct netmap_adapter *na = NA(rxq->vtnrx_sc->vtnet_ifp); 247 struct netmap_kring *kring; 248 int error; 249 250 if (!nm_native_on(na) || rxq->vtnrx_id >= na->num_rx_rings) 251 return -1; 252 253 kring = na->rx_rings[rxq->vtnrx_id]; 254 if (!(nm_kring_pending_on(kring) || 255 kring->nr_pending_mode == NKR_NETMAP_ON)) 256 return -1; 257 258 /* Expose all the RX netmap buffers we can. In case of no indirect 259 * buffers, the number of netmap slots in the RX ring matches the 260 * maximum number of 2-elements sglist that the RX virtqueue can 261 * accommodate. We need to start from kring->nr_hwcur, which is 0 262 * on netmap register and may be different from 0 if a virtio 263 * re-init happens while the device is in use by netmap. */ 264 rxq->vtnrx_nm_refill = kring->nr_hwcur; 265 error = vtnet_netmap_kring_refill(kring, na->num_rx_desc - 1); 266 virtqueue_notify(rxq->vtnrx_vq); 267 268 return error; 269 } 270 271 /* Reconcile kernel and user view of the receive ring. */ 272 static int 273 vtnet_netmap_rxsync(struct netmap_kring *kring, int flags) 274 { 275 struct netmap_adapter *na = kring->na; 276 struct ifnet *ifp = na->ifp; 277 struct netmap_ring *ring = kring->ring; 278 u_int ring_nr = kring->ring_id; 279 u_int nm_i; /* index into the netmap ring */ 280 u_int const lim = kring->nkr_num_slots - 1; 281 u_int const head = kring->rhead; 282 int force_update = (flags & NAF_FORCE_READ) || 283 (kring->nr_kflags & NKR_PENDINTR); 284 int interrupts = !(kring->nr_kflags & NKR_NOINTR); 285 286 /* device-specific */ 287 struct vtnet_softc *sc = ifp->if_softc; 288 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[ring_nr]; 289 struct virtqueue *vq = rxq->vtnrx_vq; 290 291 /* 292 * First part: import newly received packets. 293 * Only accept our own buffers (matching the token). We should only get 294 * matching buffers. The hwtail should never overrun hwcur, because 295 * we publish only N-1 receive buffers (and non N). 296 * In any case we must not leave this routine with the interrupts 297 * disabled, pending packets in the VQ and hwtail == (hwcur - 1), 298 * otherwise the pending packets could stall. 299 */ 300 if (netmap_no_pendintr || force_update) { 301 uint32_t hwtail_lim = nm_prev(kring->nr_hwcur, lim); 302 void *token; 303 304 vtnet_rxq_disable_intr(rxq); 305 306 nm_i = kring->nr_hwtail; 307 for (;;) { 308 int len; 309 token = virtqueue_dequeue(vq, &len); 310 if (token == NULL) { 311 /* 312 * Enable the interrupts again and double-check 313 * for more work. We can go on until we win the 314 * race condition, since we are not replenishing 315 * in the meanwhile, and thus we will process at 316 * most N-1 slots. 317 */ 318 if (interrupts && vtnet_rxq_enable_intr(rxq)) { 319 vtnet_rxq_disable_intr(rxq); 320 continue; 321 } 322 break; 323 } 324 if (unlikely(token != (void *)rxq)) { 325 nm_prerr("BUG: RX token mismatch"); 326 } else { 327 if (nm_i == hwtail_lim) { 328 KASSERT(false, ("hwtail would " 329 "overrun hwcur")); 330 } 331 332 /* Skip the virtio-net header. */ 333 len -= sc->vtnet_hdr_size; 334 if (unlikely(len < 0)) { 335 nm_prlim(1, "Truncated virtio-net-header, " 336 "missing %d bytes", -len); 337 len = 0; 338 } 339 ring->slot[nm_i].len = len; 340 ring->slot[nm_i].flags = 0; 341 nm_i = nm_next(nm_i, lim); 342 } 343 } 344 kring->nr_hwtail = nm_i; 345 kring->nr_kflags &= ~NKR_PENDINTR; 346 } 347 348 /* 349 * Second part: skip past packets that userspace has released. 350 */ 351 nm_i = kring->nr_hwcur; /* netmap ring index */ 352 if (nm_i != head) { 353 int released; 354 int error; 355 356 released = head - nm_i; 357 if (released < 0) 358 released += kring->nkr_num_slots; 359 error = vtnet_netmap_kring_refill(kring, released); 360 if (error) { 361 nm_prerr("Failed to replenish RX VQ with %u sgs", 362 released); 363 return error; 364 } 365 kring->nr_hwcur = head; 366 virtqueue_notify(vq); 367 } 368 369 nm_prdis("h %d c %d t %d hwcur %d hwtail %d", kring->rhead, 370 kring->rcur, kring->rtail, kring->nr_hwcur, kring->nr_hwtail); 371 372 return 0; 373 } 374 375 376 /* Enable/disable interrupts on all virtqueues. */ 377 static void 378 vtnet_netmap_intr(struct netmap_adapter *na, int state) 379 { 380 struct vtnet_softc *sc = na->ifp->if_softc; 381 int i; 382 383 for (i = 0; i < sc->vtnet_max_vq_pairs; i++) { 384 struct vtnet_rxq *rxq = &sc->vtnet_rxqs[i]; 385 struct vtnet_txq *txq = &sc->vtnet_txqs[i]; 386 struct virtqueue *txvq = txq->vtntx_vq; 387 388 if (state) { 389 vtnet_rxq_enable_intr(rxq); 390 virtqueue_enable_intr(txvq); 391 } else { 392 vtnet_rxq_disable_intr(rxq); 393 virtqueue_disable_intr(txvq); 394 } 395 } 396 } 397 398 static int 399 vtnet_netmap_tx_slots(struct vtnet_softc *sc) 400 { 401 int div; 402 403 /* We need to prepend a virtio-net header to each netmap buffer to be 404 * transmitted, therefore calling virtqueue_enqueue() passing sglist 405 * with 2 elements. 406 * TX virtqueues use indirect descriptors if the feature was negotiated 407 * with the host, and if sc->vtnet_tx_nsegs > 1. With indirect 408 * descriptors, a single virtio descriptor is sufficient to reference 409 * each TX sglist. Without them, we need two separate virtio descriptors 410 * for each TX sglist. We therefore compute the number of netmap TX 411 * slots according to these assumptions. 412 */ 413 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_tx_nsegs > 1) 414 div = 1; 415 else 416 div = 2; 417 418 return virtqueue_size(sc->vtnet_txqs[0].vtntx_vq) / div; 419 } 420 421 static int 422 vtnet_netmap_rx_slots(struct vtnet_softc *sc) 423 { 424 int div; 425 426 /* We need to prepend a virtio-net header to each netmap buffer to be 427 * received, therefore calling virtqueue_enqueue() passing sglist 428 * with 2 elements. 429 * RX virtqueues use indirect descriptors if the feature was negotiated 430 * with the host, and if sc->vtnet_rx_nsegs > 1. With indirect 431 * descriptors, a single virtio descriptor is sufficient to reference 432 * each RX sglist. Without them, we need two separate virtio descriptors 433 * for each RX sglist. We therefore compute the number of netmap RX 434 * slots according to these assumptions. 435 */ 436 if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) && sc->vtnet_rx_nsegs > 1) 437 div = 1; 438 else 439 div = 2; 440 441 return virtqueue_size(sc->vtnet_rxqs[0].vtnrx_vq) / div; 442 } 443 444 static int 445 vtnet_netmap_config(struct netmap_adapter *na, struct nm_config_info *info) 446 { 447 struct vtnet_softc *sc = na->ifp->if_softc; 448 449 info->num_tx_rings = sc->vtnet_act_vq_pairs; 450 info->num_rx_rings = sc->vtnet_act_vq_pairs; 451 info->num_tx_descs = vtnet_netmap_tx_slots(sc); 452 info->num_rx_descs = vtnet_netmap_rx_slots(sc); 453 info->rx_buf_maxsize = NETMAP_BUF_SIZE(na); 454 455 return 0; 456 } 457 458 static void 459 vtnet_netmap_attach(struct vtnet_softc *sc) 460 { 461 struct netmap_adapter na; 462 463 bzero(&na, sizeof(na)); 464 465 na.ifp = sc->vtnet_ifp; 466 na.na_flags = 0; 467 na.num_tx_desc = vtnet_netmap_tx_slots(sc); 468 na.num_rx_desc = vtnet_netmap_rx_slots(sc); 469 na.num_tx_rings = na.num_rx_rings = sc->vtnet_max_vq_pairs; 470 na.rx_buf_maxsize = 0; 471 na.nm_register = vtnet_netmap_reg; 472 na.nm_txsync = vtnet_netmap_txsync; 473 na.nm_rxsync = vtnet_netmap_rxsync; 474 na.nm_intr = vtnet_netmap_intr; 475 na.nm_config = vtnet_netmap_config; 476 477 netmap_attach(&na); 478 479 nm_prinf("vtnet attached txq=%d, txd=%d rxq=%d, rxd=%d", 480 na.num_tx_rings, na.num_tx_desc, 481 na.num_tx_rings, na.num_rx_desc); 482 } 483 /* end of file */ 484