1 /*- 2 * Copyright (c) 2014 Ruslan Bukin <[email protected]> 3 * All rights reserved. 4 * 5 * This software was developed by SRI International and the University of 6 * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237) 7 * ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* 32 * BERI virtio block backend driver 33 */ 34 35 #include <sys/cdefs.h> 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/conf.h> 43 #include <sys/stat.h> 44 #include <sys/endian.h> 45 #include <sys/disk.h> 46 #include <sys/vnode.h> 47 #include <sys/fcntl.h> 48 #include <sys/kthread.h> 49 #include <sys/buf.h> 50 #include <sys/mdioctl.h> 51 #include <sys/namei.h> 52 53 #include <machine/bus.h> 54 #include <machine/fdt.h> 55 #include <machine/cpu.h> 56 #include <machine/intr.h> 57 58 #include <dev/fdt/fdt_common.h> 59 #include <dev/ofw/openfirm.h> 60 #include <dev/ofw/ofw_bus.h> 61 #include <dev/ofw/ofw_bus_subr.h> 62 63 #include <dev/beri/virtio/virtio.h> 64 #include <dev/beri/virtio/virtio_mmio_platform.h> 65 #include <dev/altera/pio/pio.h> 66 #include <dev/virtio/mmio/virtio_mmio.h> 67 #include <dev/virtio/block/virtio_blk.h> 68 #include <dev/virtio/virtio_ids.h> 69 #include <dev/virtio/virtio_config.h> 70 #include <dev/virtio/virtio_ring.h> 71 72 #include "pio_if.h" 73 74 #define DPRINTF(fmt, ...) 75 76 /* We use indirect descriptors */ 77 #define NUM_DESCS 1 78 #define NUM_QUEUES 1 79 80 #define VTBLK_BLK_ID_BYTES 20 81 #define VTBLK_MAXSEGS 256 82 83 struct beri_vtblk_softc { 84 struct resource *res[1]; 85 bus_space_tag_t bst; 86 bus_space_handle_t bsh; 87 struct cdev *cdev; 88 device_t dev; 89 int opened; 90 device_t pio_recv; 91 device_t pio_send; 92 struct vqueue_info vs_queues[NUM_QUEUES]; 93 char ident[VTBLK_BLK_ID_BYTES]; 94 struct ucred *cred; 95 struct vnode *vnode; 96 struct thread *vtblk_ktd; 97 struct sx sc_mtx; 98 int beri_mem_offset; 99 struct md_ioctl *mdio; 100 struct virtio_blk_config *cfg; 101 }; 102 103 static struct resource_spec beri_spec[] = { 104 { SYS_RES_MEMORY, 0, RF_ACTIVE }, 105 { -1, 0 } 106 }; 107 108 static int 109 vtblk_rdwr(struct beri_vtblk_softc *sc, struct iovec *iov, 110 int cnt, int offset, int operation, int iolen) 111 { 112 struct vnode *vp; 113 struct mount *mp; 114 struct uio auio; 115 int error; 116 117 bzero(&auio, sizeof(auio)); 118 119 vp = sc->vnode; 120 121 KASSERT(vp != NULL, ("file not opened")); 122 123 auio.uio_iov = iov; 124 auio.uio_iovcnt = cnt; 125 auio.uio_offset = offset; 126 auio.uio_segflg = UIO_SYSSPACE; 127 auio.uio_rw = operation; 128 auio.uio_resid = iolen; 129 auio.uio_td = curthread; 130 131 if (operation == 0) { 132 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 133 error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); 134 VOP_UNLOCK(vp); 135 } else { 136 (void) vn_start_write(vp, &mp, V_WAIT); 137 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 138 error = VOP_WRITE(vp, &auio, IO_SYNC, sc->cred); 139 VOP_UNLOCK(vp); 140 vn_finished_write(mp); 141 } 142 143 return (error); 144 } 145 146 static void 147 vtblk_proc(struct beri_vtblk_softc *sc, struct vqueue_info *vq) 148 { 149 struct iovec iov[VTBLK_MAXSEGS + 2]; 150 uint16_t flags[VTBLK_MAXSEGS + 2]; 151 struct virtio_blk_outhdr *vbh; 152 struct iovec *tiov; 153 uint8_t *status; 154 off_t offset; 155 int iolen; 156 int type; 157 int i, n; 158 int err; 159 160 n = vq_getchain(sc->beri_mem_offset, vq, iov, 161 VTBLK_MAXSEGS + 2, flags); 162 KASSERT(n >= 2 && n <= VTBLK_MAXSEGS + 2, 163 ("wrong n value %d", n)); 164 165 tiov = getcopy(iov, n); 166 vbh = iov[0].iov_base; 167 168 status = iov[n-1].iov_base; 169 KASSERT(iov[n-1].iov_len == 1, 170 ("iov_len == %d", iov[n-1].iov_len)); 171 172 type = be32toh(vbh->type) & ~VIRTIO_BLK_T_BARRIER; 173 offset = be64toh(vbh->sector) * DEV_BSIZE; 174 175 iolen = 0; 176 for (i = 1; i < (n-1); i++) { 177 iolen += iov[i].iov_len; 178 } 179 180 switch (type) { 181 case VIRTIO_BLK_T_OUT: 182 case VIRTIO_BLK_T_IN: 183 err = vtblk_rdwr(sc, tiov + 1, i - 1, 184 offset, type, iolen); 185 break; 186 case VIRTIO_BLK_T_GET_ID: 187 /* Assume a single buffer */ 188 strncpy(iov[1].iov_base, sc->ident, 189 MIN(iov[1].iov_len, sizeof(sc->ident))); 190 err = 0; 191 break; 192 case VIRTIO_BLK_T_FLUSH: 193 /* Possible? */ 194 default: 195 err = -ENOSYS; 196 break; 197 } 198 199 if (err < 0) { 200 if (err == -ENOSYS) { 201 *status = VIRTIO_BLK_S_UNSUPP; 202 } else 203 *status = VIRTIO_BLK_S_IOERR; 204 } else 205 *status = VIRTIO_BLK_S_OK; 206 207 free(tiov, M_DEVBUF); 208 vq_relchain(vq, iov, n, 1); 209 } 210 211 static int 212 close_file(struct beri_vtblk_softc *sc, struct thread *td) 213 { 214 int error; 215 216 if (sc->vnode != NULL) { 217 vn_lock(sc->vnode, LK_EXCLUSIVE | LK_RETRY); 218 sc->vnode->v_vflag &= ~VV_MD; 219 VOP_UNLOCK(sc->vnode); 220 error = vn_close(sc->vnode, (FREAD|FWRITE), 221 sc->cred, td); 222 if (error != 0) 223 return (error); 224 sc->vnode = NULL; 225 } 226 227 if (sc->cred != NULL) 228 crfree(sc->cred); 229 230 return (0); 231 } 232 233 static int 234 open_file(struct beri_vtblk_softc *sc, struct thread *td) 235 { 236 struct nameidata nd; 237 struct vattr vattr; 238 int error; 239 int flags; 240 241 flags = (FREAD | FWRITE); 242 NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, sc->mdio->md_file); 243 error = vn_open(&nd, &flags, 0, NULL); 244 if (error != 0) 245 return (error); 246 NDFREE_PNBUF(&nd); 247 248 if (nd.ni_vp->v_type != VREG) { 249 return (EINVAL); 250 } 251 252 error = VOP_GETATTR(nd.ni_vp, &vattr, td->td_ucred); 253 if (error != 0) 254 return (error); 255 256 if (VOP_ISLOCKED(nd.ni_vp) != LK_EXCLUSIVE) { 257 vn_lock(nd.ni_vp, LK_UPGRADE | LK_RETRY); 258 if (VN_IS_DOOMED(nd.ni_vp)) { 259 return (1); 260 } 261 } 262 nd.ni_vp->v_vflag |= VV_MD; 263 VOP_UNLOCK(nd.ni_vp); 264 265 sc->vnode = nd.ni_vp; 266 sc->cred = crhold(td->td_ucred); 267 268 return (0); 269 } 270 271 static int 272 vtblk_notify(struct beri_vtblk_softc *sc) 273 { 274 struct vqueue_info *vq; 275 int queue; 276 int reg; 277 278 vq = &sc->vs_queues[0]; 279 if (!vq_ring_ready(vq)) 280 return (0); 281 282 if (!sc->opened) 283 return (0); 284 285 reg = READ2(sc, VIRTIO_MMIO_QUEUE_NOTIFY); 286 queue = be16toh(reg); 287 288 KASSERT(queue == 0, ("we support single queue only")); 289 290 /* Process new descriptors */ 291 vq = &sc->vs_queues[queue]; 292 vq->vq_save_used = be16toh(vq->vq_used->idx); 293 while (vq_has_descs(vq)) 294 vtblk_proc(sc, vq); 295 296 /* Interrupt the other side */ 297 if ((be16toh(vq->vq_avail->flags) & VRING_AVAIL_F_NO_INTERRUPT) == 0) { 298 reg = htobe32(VIRTIO_MMIO_INT_VRING); 299 WRITE4(sc, VIRTIO_MMIO_INTERRUPT_STATUS, reg); 300 PIO_SET(sc->pio_send, Q_INTR, 1); 301 } 302 303 return (0); 304 } 305 306 static int 307 vq_init(struct beri_vtblk_softc *sc) 308 { 309 struct vqueue_info *vq; 310 uint8_t *base; 311 int size; 312 int reg; 313 int pfn; 314 315 vq = &sc->vs_queues[0]; 316 vq->vq_qsize = NUM_DESCS; 317 318 reg = READ4(sc, VIRTIO_MMIO_QUEUE_PFN); 319 pfn = be32toh(reg); 320 vq->vq_pfn = pfn; 321 322 size = vring_size(vq->vq_qsize, VRING_ALIGN); 323 base = paddr_map(sc->beri_mem_offset, 324 (pfn << PAGE_SHIFT), size); 325 326 /* First pages are descriptors */ 327 vq->vq_desc = (struct vring_desc *)base; 328 base += vq->vq_qsize * sizeof(struct vring_desc); 329 330 /* Then avail ring */ 331 vq->vq_avail = (struct vring_avail *)base; 332 base += (2 + vq->vq_qsize + 1) * sizeof(uint16_t); 333 334 /* Then it's rounded up to the next page */ 335 base = (uint8_t *)roundup2((uintptr_t)base, VRING_ALIGN); 336 337 /* And the last pages are the used ring */ 338 vq->vq_used = (struct vring_used *)base; 339 340 /* Mark queue as allocated, and start at 0 when we use it. */ 341 vq->vq_flags = VQ_ALLOC; 342 vq->vq_last_avail = 0; 343 344 return (0); 345 } 346 347 static void 348 vtblk_thread(void *arg) 349 { 350 struct beri_vtblk_softc *sc; 351 int err; 352 353 sc = arg; 354 355 sx_xlock(&sc->sc_mtx); 356 for (;;) { 357 err = msleep(sc, &sc->sc_mtx, PCATCH | PZERO, "prd", hz); 358 vtblk_notify(sc); 359 } 360 sx_xunlock(&sc->sc_mtx); 361 362 kthread_exit(); 363 } 364 365 static int 366 backend_info(struct beri_vtblk_softc *sc) 367 { 368 struct virtio_blk_config *cfg; 369 uint32_t *s; 370 int reg; 371 int i; 372 373 /* Specify that we provide block device */ 374 reg = htobe32(VIRTIO_ID_BLOCK); 375 WRITE4(sc, VIRTIO_MMIO_DEVICE_ID, reg); 376 377 /* Queue size */ 378 reg = htobe32(NUM_DESCS); 379 WRITE4(sc, VIRTIO_MMIO_QUEUE_NUM_MAX, reg); 380 381 /* Our features */ 382 reg = htobe32(VIRTIO_RING_F_INDIRECT_DESC 383 | VIRTIO_BLK_F_BLK_SIZE 384 | VIRTIO_BLK_F_SEG_MAX); 385 WRITE4(sc, VIRTIO_MMIO_HOST_FEATURES, reg); 386 387 cfg = sc->cfg; 388 cfg->capacity = htobe64(sc->mdio->md_mediasize / DEV_BSIZE); 389 cfg->size_max = 0; /* not negotiated */ 390 cfg->seg_max = htobe32(VTBLK_MAXSEGS); 391 cfg->blk_size = htobe32(DEV_BSIZE); 392 393 s = (uint32_t *)cfg; 394 395 for (i = 0; i < sizeof(struct virtio_blk_config); i+=4) { 396 WRITE4(sc, VIRTIO_MMIO_CONFIG + i, *s); 397 s+=1; 398 } 399 400 strncpy(sc->ident, "Virtio block backend", sizeof(sc->ident)); 401 402 return (0); 403 } 404 405 static void 406 vtblk_intr(void *arg) 407 { 408 struct beri_vtblk_softc *sc; 409 int pending; 410 int reg; 411 412 sc = arg; 413 414 reg = PIO_READ(sc->pio_recv); 415 416 /* Ack */ 417 PIO_SET(sc->pio_recv, reg, 0); 418 419 pending = htobe32(reg); 420 421 if (pending & Q_PFN) { 422 vq_init(sc); 423 } 424 425 if (pending & Q_NOTIFY) { 426 wakeup(sc); 427 } 428 } 429 430 static int 431 beri_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, 432 int flags, struct thread *td) 433 { 434 struct beri_vtblk_softc *sc; 435 int err; 436 437 sc = dev->si_drv1; 438 439 switch (cmd) { 440 case MDIOCATTACH: 441 /* take file as argument */ 442 if (sc->vnode != NULL) { 443 /* Already opened */ 444 return (1); 445 } 446 sc->mdio = (struct md_ioctl *)addr; 447 backend_info(sc); 448 DPRINTF("opening file, td 0x%08x\n", (int)td); 449 err = open_file(sc, td); 450 if (err) 451 return (err); 452 PIO_SETUP_IRQ(sc->pio_recv, vtblk_intr, sc); 453 sc->opened = 1; 454 break; 455 case MDIOCDETACH: 456 if (sc->vnode == NULL) { 457 /* File not opened */ 458 return (1); 459 } 460 sc->opened = 0; 461 DPRINTF("closing file, td 0x%08x\n", (int)td); 462 err = close_file(sc, td); 463 if (err) 464 return (err); 465 PIO_TEARDOWN_IRQ(sc->pio_recv); 466 break; 467 default: 468 break; 469 } 470 471 return (0); 472 } 473 474 static struct cdevsw beri_cdevsw = { 475 .d_version = D_VERSION, 476 .d_ioctl = beri_ioctl, 477 .d_name = "virtio block backend", 478 }; 479 480 static int 481 beri_vtblk_probe(device_t dev) 482 { 483 484 if (!ofw_bus_status_okay(dev)) 485 return (ENXIO); 486 487 if (!ofw_bus_is_compatible(dev, "sri-cambridge,beri-vtblk")) 488 return (ENXIO); 489 490 device_set_desc(dev, "SRI-Cambridge BERI block"); 491 return (BUS_PROBE_DEFAULT); 492 } 493 494 static int 495 beri_vtblk_attach(device_t dev) 496 { 497 struct beri_vtblk_softc *sc; 498 int error; 499 500 sc = device_get_softc(dev); 501 sc->dev = dev; 502 503 if (bus_alloc_resources(dev, beri_spec, sc->res)) { 504 device_printf(dev, "could not allocate resources\n"); 505 return (ENXIO); 506 } 507 508 /* Memory interface */ 509 sc->bst = rman_get_bustag(sc->res[0]); 510 sc->bsh = rman_get_bushandle(sc->res[0]); 511 512 sc->cfg = malloc(sizeof(struct virtio_blk_config), 513 M_DEVBUF, M_NOWAIT|M_ZERO); 514 515 sx_init(&sc->sc_mtx, device_get_nameunit(sc->dev)); 516 517 error = kthread_add(vtblk_thread, sc, NULL, &sc->vtblk_ktd, 518 0, 0, "beri_virtio_block"); 519 if (error) { 520 device_printf(dev, "cannot create kthread\n"); 521 return (ENXIO); 522 } 523 524 if (setup_offset(dev, &sc->beri_mem_offset) != 0) 525 return (ENXIO); 526 if (setup_pio(dev, "pio-send", &sc->pio_send) != 0) 527 return (ENXIO); 528 if (setup_pio(dev, "pio-recv", &sc->pio_recv) != 0) 529 return (ENXIO); 530 531 sc->cdev = make_dev(&beri_cdevsw, 0, UID_ROOT, GID_WHEEL, 532 S_IRWXU, "beri_vtblk"); 533 if (sc->cdev == NULL) { 534 device_printf(dev, "Failed to create character device.\n"); 535 return (ENXIO); 536 } 537 538 sc->cdev->si_drv1 = sc; 539 return (0); 540 } 541 542 static device_method_t beri_vtblk_methods[] = { 543 DEVMETHOD(device_probe, beri_vtblk_probe), 544 DEVMETHOD(device_attach, beri_vtblk_attach), 545 { 0, 0 } 546 }; 547 548 static driver_t beri_vtblk_driver = { 549 "beri_vtblk", 550 beri_vtblk_methods, 551 sizeof(struct beri_vtblk_softc), 552 }; 553 554 DRIVER_MODULE(beri_vtblk, simplebus, beri_vtblk_driver, 0, 0); 555