1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_bhyve_snapshot.h" 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/jail.h> 39 #include <sys/queue.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/malloc.h> 43 #include <sys/conf.h> 44 #include <sys/sysctl.h> 45 #include <sys/libkern.h> 46 #include <sys/ioccom.h> 47 #include <sys/mman.h> 48 #include <sys/uio.h> 49 #include <sys/proc.h> 50 51 #include <vm/vm.h> 52 #include <vm/pmap.h> 53 #include <vm/vm_map.h> 54 #include <vm/vm_object.h> 55 56 #include <machine/vmparam.h> 57 #include <machine/vmm.h> 58 #include <machine/vmm_dev.h> 59 #include <machine/vmm_instruction_emul.h> 60 #include <machine/vmm_snapshot.h> 61 #include <x86/apicreg.h> 62 63 #include "vmm_lapic.h" 64 #include "vmm_stat.h" 65 #include "vmm_mem.h" 66 #include "io/ppt.h" 67 #include "io/vatpic.h" 68 #include "io/vioapic.h" 69 #include "io/vhpet.h" 70 #include "io/vrtc.h" 71 72 struct devmem_softc { 73 int segid; 74 char *name; 75 struct cdev *cdev; 76 struct vmmdev_softc *sc; 77 SLIST_ENTRY(devmem_softc) link; 78 }; 79 80 struct vmmdev_softc { 81 struct vm *vm; /* vm instance cookie */ 82 struct cdev *cdev; 83 SLIST_ENTRY(vmmdev_softc) link; 84 SLIST_HEAD(, devmem_softc) devmem; 85 int flags; 86 }; 87 #define VSC_LINKED 0x01 88 89 static SLIST_HEAD(, vmmdev_softc) head; 90 91 static unsigned pr_allow_flag; 92 static struct mtx vmmdev_mtx; 93 94 static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev"); 95 96 SYSCTL_DECL(_hw_vmm); 97 98 static int vmm_priv_check(struct ucred *ucred); 99 static int devmem_create_cdev(const char *vmname, int id, char *devmem); 100 static void devmem_destroy(void *arg); 101 102 static int 103 vmm_priv_check(struct ucred *ucred) 104 { 105 106 if (jailed(ucred) && 107 !(ucred->cr_prison->pr_allow & pr_allow_flag)) 108 return (EPERM); 109 110 return (0); 111 } 112 113 static int 114 vcpu_lock_one(struct vmmdev_softc *sc, int vcpu) 115 { 116 int error; 117 118 if (vcpu < 0 || vcpu >= vm_get_maxcpus(sc->vm)) 119 return (EINVAL); 120 121 error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN, true); 122 return (error); 123 } 124 125 static void 126 vcpu_unlock_one(struct vmmdev_softc *sc, int vcpu) 127 { 128 enum vcpu_state state; 129 130 state = vcpu_get_state(sc->vm, vcpu, NULL); 131 if (state != VCPU_FROZEN) { 132 panic("vcpu %s(%d) has invalid state %d", vm_name(sc->vm), 133 vcpu, state); 134 } 135 136 vcpu_set_state(sc->vm, vcpu, VCPU_IDLE, false); 137 } 138 139 static int 140 vcpu_lock_all(struct vmmdev_softc *sc) 141 { 142 int error, vcpu; 143 uint16_t maxcpus; 144 145 maxcpus = vm_get_maxcpus(sc->vm); 146 for (vcpu = 0; vcpu < maxcpus; vcpu++) { 147 error = vcpu_lock_one(sc, vcpu); 148 if (error) 149 break; 150 } 151 152 if (error) { 153 while (--vcpu >= 0) 154 vcpu_unlock_one(sc, vcpu); 155 } 156 157 return (error); 158 } 159 160 static void 161 vcpu_unlock_all(struct vmmdev_softc *sc) 162 { 163 int vcpu; 164 uint16_t maxcpus; 165 166 maxcpus = vm_get_maxcpus(sc->vm); 167 for (vcpu = 0; vcpu < maxcpus; vcpu++) 168 vcpu_unlock_one(sc, vcpu); 169 } 170 171 static struct vmmdev_softc * 172 vmmdev_lookup(const char *name) 173 { 174 struct vmmdev_softc *sc; 175 176 #ifdef notyet /* XXX kernel is not compiled with invariants */ 177 mtx_assert(&vmmdev_mtx, MA_OWNED); 178 #endif 179 180 SLIST_FOREACH(sc, &head, link) { 181 if (strcmp(name, vm_name(sc->vm)) == 0) 182 break; 183 } 184 185 return (sc); 186 } 187 188 static struct vmmdev_softc * 189 vmmdev_lookup2(struct cdev *cdev) 190 { 191 192 return (cdev->si_drv1); 193 } 194 195 static int 196 vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags) 197 { 198 int error, off, c, prot; 199 vm_paddr_t gpa, maxaddr; 200 void *hpa, *cookie; 201 struct vmmdev_softc *sc; 202 uint16_t lastcpu; 203 204 error = vmm_priv_check(curthread->td_ucred); 205 if (error) 206 return (error); 207 208 sc = vmmdev_lookup2(cdev); 209 if (sc == NULL) 210 return (ENXIO); 211 212 /* 213 * Get a read lock on the guest memory map by freezing any vcpu. 214 */ 215 lastcpu = vm_get_maxcpus(sc->vm) - 1; 216 error = vcpu_lock_one(sc, lastcpu); 217 if (error) 218 return (error); 219 220 prot = (uio->uio_rw == UIO_WRITE ? VM_PROT_WRITE : VM_PROT_READ); 221 maxaddr = vmm_sysmem_maxaddr(sc->vm); 222 while (uio->uio_resid > 0 && error == 0) { 223 gpa = uio->uio_offset; 224 off = gpa & PAGE_MASK; 225 c = min(uio->uio_resid, PAGE_SIZE - off); 226 227 /* 228 * The VM has a hole in its physical memory map. If we want to 229 * use 'dd' to inspect memory beyond the hole we need to 230 * provide bogus data for memory that lies in the hole. 231 * 232 * Since this device does not support lseek(2), dd(1) will 233 * read(2) blocks of data to simulate the lseek(2). 234 */ 235 hpa = vm_gpa_hold(sc->vm, lastcpu, gpa, c, 236 prot, &cookie); 237 if (hpa == NULL) { 238 if (uio->uio_rw == UIO_READ && gpa < maxaddr) 239 error = uiomove(__DECONST(void *, zero_region), 240 c, uio); 241 else 242 error = EFAULT; 243 } else { 244 error = uiomove(hpa, c, uio); 245 vm_gpa_release(cookie); 246 } 247 } 248 vcpu_unlock_one(sc, lastcpu); 249 return (error); 250 } 251 252 CTASSERT(sizeof(((struct vm_memseg *)0)->name) >= VM_MAX_SUFFIXLEN + 1); 253 254 static int 255 get_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len) 256 { 257 struct devmem_softc *dsc; 258 int error; 259 bool sysmem; 260 261 error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL); 262 if (error || mseg->len == 0) 263 return (error); 264 265 if (!sysmem) { 266 SLIST_FOREACH(dsc, &sc->devmem, link) { 267 if (dsc->segid == mseg->segid) 268 break; 269 } 270 KASSERT(dsc != NULL, ("%s: devmem segment %d not found", 271 __func__, mseg->segid)); 272 error = copystr(dsc->name, mseg->name, len, NULL); 273 } else { 274 bzero(mseg->name, len); 275 } 276 277 return (error); 278 } 279 280 static int 281 alloc_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len) 282 { 283 char *name; 284 int error; 285 bool sysmem; 286 287 error = 0; 288 name = NULL; 289 sysmem = true; 290 291 /* 292 * The allocation is lengthened by 1 to hold a terminating NUL. It'll 293 * by stripped off when devfs processes the full string. 294 */ 295 if (VM_MEMSEG_NAME(mseg)) { 296 sysmem = false; 297 name = malloc(len, M_VMMDEV, M_WAITOK); 298 error = copystr(mseg->name, name, len, NULL); 299 if (error) 300 goto done; 301 } 302 303 error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem); 304 if (error) 305 goto done; 306 307 if (VM_MEMSEG_NAME(mseg)) { 308 error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name); 309 if (error) 310 vm_free_memseg(sc->vm, mseg->segid); 311 else 312 name = NULL; /* freed when 'cdev' is destroyed */ 313 } 314 done: 315 free(name, M_VMMDEV); 316 return (error); 317 } 318 319 static int 320 vm_get_register_set(struct vm *vm, int vcpu, unsigned int count, int *regnum, 321 uint64_t *regval) 322 { 323 int error, i; 324 325 error = 0; 326 for (i = 0; i < count; i++) { 327 error = vm_get_register(vm, vcpu, regnum[i], ®val[i]); 328 if (error) 329 break; 330 } 331 return (error); 332 } 333 334 static int 335 vm_set_register_set(struct vm *vm, int vcpu, unsigned int count, int *regnum, 336 uint64_t *regval) 337 { 338 int error, i; 339 340 error = 0; 341 for (i = 0; i < count; i++) { 342 error = vm_set_register(vm, vcpu, regnum[i], regval[i]); 343 if (error) 344 break; 345 } 346 return (error); 347 } 348 349 static int 350 vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, 351 struct thread *td) 352 { 353 int error, vcpu, state_changed, size; 354 cpuset_t *cpuset; 355 struct vmmdev_softc *sc; 356 struct vm_register *vmreg; 357 struct vm_seg_desc *vmsegdesc; 358 struct vm_register_set *vmregset; 359 struct vm_run *vmrun; 360 struct vm_exception *vmexc; 361 struct vm_lapic_irq *vmirq; 362 struct vm_lapic_msi *vmmsi; 363 struct vm_ioapic_irq *ioapic_irq; 364 struct vm_isa_irq *isa_irq; 365 struct vm_isa_irq_trigger *isa_irq_trigger; 366 struct vm_capability *vmcap; 367 struct vm_pptdev *pptdev; 368 struct vm_pptdev_mmio *pptmmio; 369 struct vm_pptdev_msi *pptmsi; 370 struct vm_pptdev_msix *pptmsix; 371 struct vm_nmi *vmnmi; 372 struct vm_stats *vmstats; 373 struct vm_stat_desc *statdesc; 374 struct vm_x2apic *x2apic; 375 struct vm_gpa_pte *gpapte; 376 struct vm_suspend *vmsuspend; 377 struct vm_gla2gpa *gg; 378 struct vm_activate_cpu *vac; 379 struct vm_cpuset *vm_cpuset; 380 struct vm_intinfo *vmii; 381 struct vm_rtc_time *rtctime; 382 struct vm_rtc_data *rtcdata; 383 struct vm_memmap *mm; 384 struct vm_munmap *mu; 385 struct vm_cpu_topology *topology; 386 struct vm_readwrite_kernemu_device *kernemu; 387 uint64_t *regvals; 388 int *regnums; 389 #ifdef BHYVE_SNAPSHOT 390 struct vm_snapshot_meta *snapshot_meta; 391 #endif 392 393 error = vmm_priv_check(curthread->td_ucred); 394 if (error) 395 return (error); 396 397 sc = vmmdev_lookup2(cdev); 398 if (sc == NULL) 399 return (ENXIO); 400 401 vcpu = -1; 402 state_changed = 0; 403 404 /* 405 * Some VMM ioctls can operate only on vcpus that are not running. 406 */ 407 switch (cmd) { 408 case VM_RUN: 409 case VM_GET_REGISTER: 410 case VM_SET_REGISTER: 411 case VM_GET_SEGMENT_DESCRIPTOR: 412 case VM_SET_SEGMENT_DESCRIPTOR: 413 case VM_GET_REGISTER_SET: 414 case VM_SET_REGISTER_SET: 415 case VM_INJECT_EXCEPTION: 416 case VM_GET_CAPABILITY: 417 case VM_SET_CAPABILITY: 418 case VM_PPTDEV_MSI: 419 case VM_PPTDEV_MSIX: 420 case VM_SET_X2APIC_STATE: 421 case VM_GLA2GPA: 422 case VM_GLA2GPA_NOFAULT: 423 case VM_ACTIVATE_CPU: 424 case VM_SET_INTINFO: 425 case VM_GET_INTINFO: 426 case VM_RESTART_INSTRUCTION: 427 /* 428 * XXX fragile, handle with care 429 * Assumes that the first field of the ioctl data is the vcpu. 430 */ 431 vcpu = *(int *)data; 432 error = vcpu_lock_one(sc, vcpu); 433 if (error) 434 goto done; 435 state_changed = 1; 436 break; 437 438 case VM_MAP_PPTDEV_MMIO: 439 case VM_UNMAP_PPTDEV_MMIO: 440 case VM_BIND_PPTDEV: 441 case VM_UNBIND_PPTDEV: 442 #ifdef COMPAT_FREEBSD12 443 case VM_ALLOC_MEMSEG_FBSD12: 444 #endif 445 case VM_ALLOC_MEMSEG: 446 case VM_MMAP_MEMSEG: 447 case VM_MUNMAP_MEMSEG: 448 case VM_REINIT: 449 /* 450 * ioctls that operate on the entire virtual machine must 451 * prevent all vcpus from running. 452 */ 453 error = vcpu_lock_all(sc); 454 if (error) 455 goto done; 456 state_changed = 2; 457 break; 458 459 #ifdef COMPAT_FREEBSD12 460 case VM_GET_MEMSEG_FBSD12: 461 #endif 462 case VM_GET_MEMSEG: 463 case VM_MMAP_GETNEXT: 464 /* 465 * Lock a vcpu to make sure that the memory map cannot be 466 * modified while it is being inspected. 467 */ 468 vcpu = vm_get_maxcpus(sc->vm) - 1; 469 error = vcpu_lock_one(sc, vcpu); 470 if (error) 471 goto done; 472 state_changed = 1; 473 break; 474 475 default: 476 break; 477 } 478 479 switch(cmd) { 480 case VM_RUN: 481 vmrun = (struct vm_run *)data; 482 error = vm_run(sc->vm, vmrun); 483 break; 484 case VM_SUSPEND: 485 vmsuspend = (struct vm_suspend *)data; 486 error = vm_suspend(sc->vm, vmsuspend->how); 487 break; 488 case VM_REINIT: 489 error = vm_reinit(sc->vm); 490 break; 491 case VM_STAT_DESC: { 492 statdesc = (struct vm_stat_desc *)data; 493 error = vmm_stat_desc_copy(statdesc->index, 494 statdesc->desc, sizeof(statdesc->desc)); 495 break; 496 } 497 case VM_STATS: { 498 CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_ELEMS); 499 vmstats = (struct vm_stats *)data; 500 getmicrotime(&vmstats->tv); 501 error = vmm_stat_copy(sc->vm, vmstats->cpuid, 502 &vmstats->num_entries, vmstats->statbuf); 503 break; 504 } 505 case VM_PPTDEV_MSI: 506 pptmsi = (struct vm_pptdev_msi *)data; 507 error = ppt_setup_msi(sc->vm, pptmsi->vcpu, 508 pptmsi->bus, pptmsi->slot, pptmsi->func, 509 pptmsi->addr, pptmsi->msg, 510 pptmsi->numvec); 511 break; 512 case VM_PPTDEV_MSIX: 513 pptmsix = (struct vm_pptdev_msix *)data; 514 error = ppt_setup_msix(sc->vm, pptmsix->vcpu, 515 pptmsix->bus, pptmsix->slot, 516 pptmsix->func, pptmsix->idx, 517 pptmsix->addr, pptmsix->msg, 518 pptmsix->vector_control); 519 break; 520 case VM_PPTDEV_DISABLE_MSIX: 521 pptdev = (struct vm_pptdev *)data; 522 error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot, 523 pptdev->func); 524 break; 525 case VM_MAP_PPTDEV_MMIO: 526 pptmmio = (struct vm_pptdev_mmio *)data; 527 error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot, 528 pptmmio->func, pptmmio->gpa, pptmmio->len, 529 pptmmio->hpa); 530 break; 531 case VM_UNMAP_PPTDEV_MMIO: 532 pptmmio = (struct vm_pptdev_mmio *)data; 533 error = ppt_unmap_mmio(sc->vm, pptmmio->bus, pptmmio->slot, 534 pptmmio->func, pptmmio->gpa, pptmmio->len); 535 break; 536 case VM_BIND_PPTDEV: 537 pptdev = (struct vm_pptdev *)data; 538 error = vm_assign_pptdev(sc->vm, pptdev->bus, pptdev->slot, 539 pptdev->func); 540 break; 541 case VM_UNBIND_PPTDEV: 542 pptdev = (struct vm_pptdev *)data; 543 error = vm_unassign_pptdev(sc->vm, pptdev->bus, pptdev->slot, 544 pptdev->func); 545 break; 546 case VM_INJECT_EXCEPTION: 547 vmexc = (struct vm_exception *)data; 548 error = vm_inject_exception(sc->vm, vmexc->cpuid, 549 vmexc->vector, vmexc->error_code_valid, vmexc->error_code, 550 vmexc->restart_instruction); 551 break; 552 case VM_INJECT_NMI: 553 vmnmi = (struct vm_nmi *)data; 554 error = vm_inject_nmi(sc->vm, vmnmi->cpuid); 555 break; 556 case VM_LAPIC_IRQ: 557 vmirq = (struct vm_lapic_irq *)data; 558 error = lapic_intr_edge(sc->vm, vmirq->cpuid, vmirq->vector); 559 break; 560 case VM_LAPIC_LOCAL_IRQ: 561 vmirq = (struct vm_lapic_irq *)data; 562 error = lapic_set_local_intr(sc->vm, vmirq->cpuid, 563 vmirq->vector); 564 break; 565 case VM_LAPIC_MSI: 566 vmmsi = (struct vm_lapic_msi *)data; 567 error = lapic_intr_msi(sc->vm, vmmsi->addr, vmmsi->msg); 568 break; 569 case VM_IOAPIC_ASSERT_IRQ: 570 ioapic_irq = (struct vm_ioapic_irq *)data; 571 error = vioapic_assert_irq(sc->vm, ioapic_irq->irq); 572 break; 573 case VM_IOAPIC_DEASSERT_IRQ: 574 ioapic_irq = (struct vm_ioapic_irq *)data; 575 error = vioapic_deassert_irq(sc->vm, ioapic_irq->irq); 576 break; 577 case VM_IOAPIC_PULSE_IRQ: 578 ioapic_irq = (struct vm_ioapic_irq *)data; 579 error = vioapic_pulse_irq(sc->vm, ioapic_irq->irq); 580 break; 581 case VM_IOAPIC_PINCOUNT: 582 *(int *)data = vioapic_pincount(sc->vm); 583 break; 584 case VM_SET_KERNEMU_DEV: 585 case VM_GET_KERNEMU_DEV: { 586 mem_region_write_t mwrite; 587 mem_region_read_t mread; 588 bool arg; 589 590 kernemu = (void *)data; 591 592 if (kernemu->access_width > 0) 593 size = (1u << kernemu->access_width); 594 else 595 size = 1; 596 597 if (kernemu->gpa >= DEFAULT_APIC_BASE && kernemu->gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 598 mread = lapic_mmio_read; 599 mwrite = lapic_mmio_write; 600 } else if (kernemu->gpa >= VIOAPIC_BASE && kernemu->gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 601 mread = vioapic_mmio_read; 602 mwrite = vioapic_mmio_write; 603 } else if (kernemu->gpa >= VHPET_BASE && kernemu->gpa < VHPET_BASE + VHPET_SIZE) { 604 mread = vhpet_mmio_read; 605 mwrite = vhpet_mmio_write; 606 } else { 607 error = EINVAL; 608 break; 609 } 610 611 if (cmd == VM_SET_KERNEMU_DEV) 612 error = mwrite(sc->vm, kernemu->vcpuid, kernemu->gpa, 613 kernemu->value, size, &arg); 614 else 615 error = mread(sc->vm, kernemu->vcpuid, kernemu->gpa, 616 &kernemu->value, size, &arg); 617 break; 618 } 619 case VM_ISA_ASSERT_IRQ: 620 isa_irq = (struct vm_isa_irq *)data; 621 error = vatpic_assert_irq(sc->vm, isa_irq->atpic_irq); 622 if (error == 0 && isa_irq->ioapic_irq != -1) 623 error = vioapic_assert_irq(sc->vm, 624 isa_irq->ioapic_irq); 625 break; 626 case VM_ISA_DEASSERT_IRQ: 627 isa_irq = (struct vm_isa_irq *)data; 628 error = vatpic_deassert_irq(sc->vm, isa_irq->atpic_irq); 629 if (error == 0 && isa_irq->ioapic_irq != -1) 630 error = vioapic_deassert_irq(sc->vm, 631 isa_irq->ioapic_irq); 632 break; 633 case VM_ISA_PULSE_IRQ: 634 isa_irq = (struct vm_isa_irq *)data; 635 error = vatpic_pulse_irq(sc->vm, isa_irq->atpic_irq); 636 if (error == 0 && isa_irq->ioapic_irq != -1) 637 error = vioapic_pulse_irq(sc->vm, isa_irq->ioapic_irq); 638 break; 639 case VM_ISA_SET_IRQ_TRIGGER: 640 isa_irq_trigger = (struct vm_isa_irq_trigger *)data; 641 error = vatpic_set_irq_trigger(sc->vm, 642 isa_irq_trigger->atpic_irq, isa_irq_trigger->trigger); 643 break; 644 case VM_MMAP_GETNEXT: 645 mm = (struct vm_memmap *)data; 646 error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid, 647 &mm->segoff, &mm->len, &mm->prot, &mm->flags); 648 break; 649 case VM_MMAP_MEMSEG: 650 mm = (struct vm_memmap *)data; 651 error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff, 652 mm->len, mm->prot, mm->flags); 653 break; 654 case VM_MUNMAP_MEMSEG: 655 mu = (struct vm_munmap *)data; 656 error = vm_munmap_memseg(sc->vm, mu->gpa, mu->len); 657 break; 658 #ifdef COMPAT_FREEBSD12 659 case VM_ALLOC_MEMSEG_FBSD12: 660 error = alloc_memseg(sc, (struct vm_memseg *)data, 661 sizeof(((struct vm_memseg_fbsd12 *)0)->name)); 662 break; 663 #endif 664 case VM_ALLOC_MEMSEG: 665 error = alloc_memseg(sc, (struct vm_memseg *)data, 666 sizeof(((struct vm_memseg *)0)->name)); 667 break; 668 #ifdef COMPAT_FREEBSD12 669 case VM_GET_MEMSEG_FBSD12: 670 error = get_memseg(sc, (struct vm_memseg *)data, 671 sizeof(((struct vm_memseg_fbsd12 *)0)->name)); 672 break; 673 #endif 674 case VM_GET_MEMSEG: 675 error = get_memseg(sc, (struct vm_memseg *)data, 676 sizeof(((struct vm_memseg *)0)->name)); 677 break; 678 case VM_GET_REGISTER: 679 vmreg = (struct vm_register *)data; 680 error = vm_get_register(sc->vm, vmreg->cpuid, vmreg->regnum, 681 &vmreg->regval); 682 break; 683 case VM_SET_REGISTER: 684 vmreg = (struct vm_register *)data; 685 error = vm_set_register(sc->vm, vmreg->cpuid, vmreg->regnum, 686 vmreg->regval); 687 break; 688 case VM_SET_SEGMENT_DESCRIPTOR: 689 vmsegdesc = (struct vm_seg_desc *)data; 690 error = vm_set_seg_desc(sc->vm, vmsegdesc->cpuid, 691 vmsegdesc->regnum, 692 &vmsegdesc->desc); 693 break; 694 case VM_GET_SEGMENT_DESCRIPTOR: 695 vmsegdesc = (struct vm_seg_desc *)data; 696 error = vm_get_seg_desc(sc->vm, vmsegdesc->cpuid, 697 vmsegdesc->regnum, 698 &vmsegdesc->desc); 699 break; 700 case VM_GET_REGISTER_SET: 701 vmregset = (struct vm_register_set *)data; 702 if (vmregset->count > VM_REG_LAST) { 703 error = EINVAL; 704 break; 705 } 706 regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV, 707 M_WAITOK); 708 regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV, 709 M_WAITOK); 710 error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) * 711 vmregset->count); 712 if (error == 0) 713 error = vm_get_register_set(sc->vm, vmregset->cpuid, 714 vmregset->count, regnums, regvals); 715 if (error == 0) 716 error = copyout(regvals, vmregset->regvals, 717 sizeof(regvals[0]) * vmregset->count); 718 free(regvals, M_VMMDEV); 719 free(regnums, M_VMMDEV); 720 break; 721 case VM_SET_REGISTER_SET: 722 vmregset = (struct vm_register_set *)data; 723 if (vmregset->count > VM_REG_LAST) { 724 error = EINVAL; 725 break; 726 } 727 regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV, 728 M_WAITOK); 729 regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV, 730 M_WAITOK); 731 error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) * 732 vmregset->count); 733 if (error == 0) 734 error = copyin(vmregset->regvals, regvals, 735 sizeof(regvals[0]) * vmregset->count); 736 if (error == 0) 737 error = vm_set_register_set(sc->vm, vmregset->cpuid, 738 vmregset->count, regnums, regvals); 739 free(regvals, M_VMMDEV); 740 free(regnums, M_VMMDEV); 741 break; 742 case VM_GET_CAPABILITY: 743 vmcap = (struct vm_capability *)data; 744 error = vm_get_capability(sc->vm, vmcap->cpuid, 745 vmcap->captype, 746 &vmcap->capval); 747 break; 748 case VM_SET_CAPABILITY: 749 vmcap = (struct vm_capability *)data; 750 error = vm_set_capability(sc->vm, vmcap->cpuid, 751 vmcap->captype, 752 vmcap->capval); 753 break; 754 case VM_SET_X2APIC_STATE: 755 x2apic = (struct vm_x2apic *)data; 756 error = vm_set_x2apic_state(sc->vm, 757 x2apic->cpuid, x2apic->state); 758 break; 759 case VM_GET_X2APIC_STATE: 760 x2apic = (struct vm_x2apic *)data; 761 error = vm_get_x2apic_state(sc->vm, 762 x2apic->cpuid, &x2apic->state); 763 break; 764 case VM_GET_GPA_PMAP: 765 gpapte = (struct vm_gpa_pte *)data; 766 pmap_get_mapping(vmspace_pmap(vm_get_vmspace(sc->vm)), 767 gpapte->gpa, gpapte->pte, &gpapte->ptenum); 768 error = 0; 769 break; 770 case VM_GET_HPET_CAPABILITIES: 771 error = vhpet_getcap((struct vm_hpet_cap *)data); 772 break; 773 case VM_GLA2GPA: { 774 CTASSERT(PROT_READ == VM_PROT_READ); 775 CTASSERT(PROT_WRITE == VM_PROT_WRITE); 776 CTASSERT(PROT_EXEC == VM_PROT_EXECUTE); 777 gg = (struct vm_gla2gpa *)data; 778 error = vm_gla2gpa(sc->vm, gg->vcpuid, &gg->paging, gg->gla, 779 gg->prot, &gg->gpa, &gg->fault); 780 KASSERT(error == 0 || error == EFAULT, 781 ("%s: vm_gla2gpa unknown error %d", __func__, error)); 782 break; 783 } 784 case VM_GLA2GPA_NOFAULT: 785 gg = (struct vm_gla2gpa *)data; 786 error = vm_gla2gpa_nofault(sc->vm, gg->vcpuid, &gg->paging, 787 gg->gla, gg->prot, &gg->gpa, &gg->fault); 788 KASSERT(error == 0 || error == EFAULT, 789 ("%s: vm_gla2gpa unknown error %d", __func__, error)); 790 break; 791 case VM_ACTIVATE_CPU: 792 vac = (struct vm_activate_cpu *)data; 793 error = vm_activate_cpu(sc->vm, vac->vcpuid); 794 break; 795 case VM_GET_CPUS: 796 error = 0; 797 vm_cpuset = (struct vm_cpuset *)data; 798 size = vm_cpuset->cpusetsize; 799 if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) { 800 error = ERANGE; 801 break; 802 } 803 cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO); 804 if (vm_cpuset->which == VM_ACTIVE_CPUS) 805 *cpuset = vm_active_cpus(sc->vm); 806 else if (vm_cpuset->which == VM_SUSPENDED_CPUS) 807 *cpuset = vm_suspended_cpus(sc->vm); 808 else if (vm_cpuset->which == VM_DEBUG_CPUS) 809 *cpuset = vm_debug_cpus(sc->vm); 810 else 811 error = EINVAL; 812 if (error == 0) 813 error = copyout(cpuset, vm_cpuset->cpus, size); 814 free(cpuset, M_TEMP); 815 break; 816 case VM_SUSPEND_CPU: 817 vac = (struct vm_activate_cpu *)data; 818 error = vm_suspend_cpu(sc->vm, vac->vcpuid); 819 break; 820 case VM_RESUME_CPU: 821 vac = (struct vm_activate_cpu *)data; 822 error = vm_resume_cpu(sc->vm, vac->vcpuid); 823 break; 824 case VM_SET_INTINFO: 825 vmii = (struct vm_intinfo *)data; 826 error = vm_exit_intinfo(sc->vm, vmii->vcpuid, vmii->info1); 827 break; 828 case VM_GET_INTINFO: 829 vmii = (struct vm_intinfo *)data; 830 error = vm_get_intinfo(sc->vm, vmii->vcpuid, &vmii->info1, 831 &vmii->info2); 832 break; 833 case VM_RTC_WRITE: 834 rtcdata = (struct vm_rtc_data *)data; 835 error = vrtc_nvram_write(sc->vm, rtcdata->offset, 836 rtcdata->value); 837 break; 838 case VM_RTC_READ: 839 rtcdata = (struct vm_rtc_data *)data; 840 error = vrtc_nvram_read(sc->vm, rtcdata->offset, 841 &rtcdata->value); 842 break; 843 case VM_RTC_SETTIME: 844 rtctime = (struct vm_rtc_time *)data; 845 error = vrtc_set_time(sc->vm, rtctime->secs); 846 break; 847 case VM_RTC_GETTIME: 848 error = 0; 849 rtctime = (struct vm_rtc_time *)data; 850 rtctime->secs = vrtc_get_time(sc->vm); 851 break; 852 case VM_RESTART_INSTRUCTION: 853 error = vm_restart_instruction(sc->vm, vcpu); 854 break; 855 case VM_SET_TOPOLOGY: 856 topology = (struct vm_cpu_topology *)data; 857 error = vm_set_topology(sc->vm, topology->sockets, 858 topology->cores, topology->threads, topology->maxcpus); 859 break; 860 case VM_GET_TOPOLOGY: 861 topology = (struct vm_cpu_topology *)data; 862 vm_get_topology(sc->vm, &topology->sockets, &topology->cores, 863 &topology->threads, &topology->maxcpus); 864 error = 0; 865 break; 866 #ifdef BHYVE_SNAPSHOT 867 case VM_SNAPSHOT_REQ: 868 snapshot_meta = (struct vm_snapshot_meta *)data; 869 error = vm_snapshot_req(sc->vm, snapshot_meta); 870 break; 871 case VM_RESTORE_TIME: 872 error = vm_restore_time(sc->vm); 873 break; 874 #endif 875 default: 876 error = ENOTTY; 877 break; 878 } 879 880 if (state_changed == 1) 881 vcpu_unlock_one(sc, vcpu); 882 else if (state_changed == 2) 883 vcpu_unlock_all(sc); 884 885 done: 886 /* 887 * Make sure that no handler returns a kernel-internal 888 * error value to userspace. 889 */ 890 KASSERT(error == ERESTART || error >= 0, 891 ("vmmdev_ioctl: invalid error return %d", error)); 892 return (error); 893 } 894 895 static int 896 vmmdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize, 897 struct vm_object **objp, int nprot) 898 { 899 struct vmmdev_softc *sc; 900 vm_paddr_t gpa; 901 size_t len; 902 vm_ooffset_t segoff, first, last; 903 int error, found, segid; 904 uint16_t lastcpu; 905 bool sysmem; 906 907 error = vmm_priv_check(curthread->td_ucred); 908 if (error) 909 return (error); 910 911 first = *offset; 912 last = first + mapsize; 913 if ((nprot & PROT_EXEC) || first < 0 || first >= last) 914 return (EINVAL); 915 916 sc = vmmdev_lookup2(cdev); 917 if (sc == NULL) { 918 /* virtual machine is in the process of being created */ 919 return (EINVAL); 920 } 921 922 /* 923 * Get a read lock on the guest memory map by freezing any vcpu. 924 */ 925 lastcpu = vm_get_maxcpus(sc->vm) - 1; 926 error = vcpu_lock_one(sc, lastcpu); 927 if (error) 928 return (error); 929 930 gpa = 0; 931 found = 0; 932 while (!found) { 933 error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len, 934 NULL, NULL); 935 if (error) 936 break; 937 938 if (first >= gpa && last <= gpa + len) 939 found = 1; 940 else 941 gpa += len; 942 } 943 944 if (found) { 945 error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp); 946 KASSERT(error == 0 && *objp != NULL, 947 ("%s: invalid memory segment %d", __func__, segid)); 948 if (sysmem) { 949 vm_object_reference(*objp); 950 *offset = segoff + (first - gpa); 951 } else { 952 error = EINVAL; 953 } 954 } 955 vcpu_unlock_one(sc, lastcpu); 956 return (error); 957 } 958 959 static void 960 vmmdev_destroy(void *arg) 961 { 962 struct vmmdev_softc *sc = arg; 963 struct devmem_softc *dsc; 964 int error; 965 966 error = vcpu_lock_all(sc); 967 KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error)); 968 969 while ((dsc = SLIST_FIRST(&sc->devmem)) != NULL) { 970 KASSERT(dsc->cdev == NULL, ("%s: devmem not free", __func__)); 971 SLIST_REMOVE_HEAD(&sc->devmem, link); 972 free(dsc->name, M_VMMDEV); 973 free(dsc, M_VMMDEV); 974 } 975 976 if (sc->cdev != NULL) 977 destroy_dev(sc->cdev); 978 979 if (sc->vm != NULL) 980 vm_destroy(sc->vm); 981 982 if ((sc->flags & VSC_LINKED) != 0) { 983 mtx_lock(&vmmdev_mtx); 984 SLIST_REMOVE(&head, sc, vmmdev_softc, link); 985 mtx_unlock(&vmmdev_mtx); 986 } 987 988 free(sc, M_VMMDEV); 989 } 990 991 static int 992 sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS) 993 { 994 struct devmem_softc *dsc; 995 struct vmmdev_softc *sc; 996 struct cdev *cdev; 997 char *buf; 998 int error, buflen; 999 1000 error = vmm_priv_check(req->td->td_ucred); 1001 if (error) 1002 return (error); 1003 1004 buflen = VM_MAX_NAMELEN + 1; 1005 buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO); 1006 strlcpy(buf, "beavis", buflen); 1007 error = sysctl_handle_string(oidp, buf, buflen, req); 1008 if (error != 0 || req->newptr == NULL) 1009 goto out; 1010 1011 mtx_lock(&vmmdev_mtx); 1012 sc = vmmdev_lookup(buf); 1013 if (sc == NULL || sc->cdev == NULL) { 1014 mtx_unlock(&vmmdev_mtx); 1015 error = EINVAL; 1016 goto out; 1017 } 1018 1019 /* 1020 * The 'cdev' will be destroyed asynchronously when 'si_threadcount' 1021 * goes down to 0 so we should not do it again in the callback. 1022 * 1023 * Setting 'sc->cdev' to NULL is also used to indicate that the VM 1024 * is scheduled for destruction. 1025 */ 1026 cdev = sc->cdev; 1027 sc->cdev = NULL; 1028 mtx_unlock(&vmmdev_mtx); 1029 1030 /* 1031 * Schedule all cdevs to be destroyed: 1032 * 1033 * - any new operations on the 'cdev' will return an error (ENXIO). 1034 * 1035 * - when the 'si_threadcount' dwindles down to zero the 'cdev' will 1036 * be destroyed and the callback will be invoked in a taskqueue 1037 * context. 1038 * 1039 * - the 'devmem' cdevs are destroyed before the virtual machine 'cdev' 1040 */ 1041 SLIST_FOREACH(dsc, &sc->devmem, link) { 1042 KASSERT(dsc->cdev != NULL, ("devmem cdev already destroyed")); 1043 destroy_dev_sched_cb(dsc->cdev, devmem_destroy, dsc); 1044 } 1045 destroy_dev_sched_cb(cdev, vmmdev_destroy, sc); 1046 error = 0; 1047 1048 out: 1049 free(buf, M_VMMDEV); 1050 return (error); 1051 } 1052 SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy, 1053 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, 1054 NULL, 0, sysctl_vmm_destroy, "A", 1055 NULL); 1056 1057 static struct cdevsw vmmdevsw = { 1058 .d_name = "vmmdev", 1059 .d_version = D_VERSION, 1060 .d_ioctl = vmmdev_ioctl, 1061 .d_mmap_single = vmmdev_mmap_single, 1062 .d_read = vmmdev_rw, 1063 .d_write = vmmdev_rw, 1064 }; 1065 1066 static int 1067 sysctl_vmm_create(SYSCTL_HANDLER_ARGS) 1068 { 1069 struct vm *vm; 1070 struct cdev *cdev; 1071 struct vmmdev_softc *sc, *sc2; 1072 char *buf; 1073 int error, buflen; 1074 1075 error = vmm_priv_check(req->td->td_ucred); 1076 if (error) 1077 return (error); 1078 1079 buflen = VM_MAX_NAMELEN + 1; 1080 buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO); 1081 strlcpy(buf, "beavis", buflen); 1082 error = sysctl_handle_string(oidp, buf, buflen, req); 1083 if (error != 0 || req->newptr == NULL) 1084 goto out; 1085 1086 mtx_lock(&vmmdev_mtx); 1087 sc = vmmdev_lookup(buf); 1088 mtx_unlock(&vmmdev_mtx); 1089 if (sc != NULL) { 1090 error = EEXIST; 1091 goto out; 1092 } 1093 1094 error = vm_create(buf, &vm); 1095 if (error != 0) 1096 goto out; 1097 1098 sc = malloc(sizeof(struct vmmdev_softc), M_VMMDEV, M_WAITOK | M_ZERO); 1099 sc->vm = vm; 1100 SLIST_INIT(&sc->devmem); 1101 1102 /* 1103 * Lookup the name again just in case somebody sneaked in when we 1104 * dropped the lock. 1105 */ 1106 mtx_lock(&vmmdev_mtx); 1107 sc2 = vmmdev_lookup(buf); 1108 if (sc2 == NULL) { 1109 SLIST_INSERT_HEAD(&head, sc, link); 1110 sc->flags |= VSC_LINKED; 1111 } 1112 mtx_unlock(&vmmdev_mtx); 1113 1114 if (sc2 != NULL) { 1115 vmmdev_destroy(sc); 1116 error = EEXIST; 1117 goto out; 1118 } 1119 1120 error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, NULL, 1121 UID_ROOT, GID_WHEEL, 0600, "vmm/%s", buf); 1122 if (error != 0) { 1123 vmmdev_destroy(sc); 1124 goto out; 1125 } 1126 1127 mtx_lock(&vmmdev_mtx); 1128 sc->cdev = cdev; 1129 sc->cdev->si_drv1 = sc; 1130 mtx_unlock(&vmmdev_mtx); 1131 1132 out: 1133 free(buf, M_VMMDEV); 1134 return (error); 1135 } 1136 SYSCTL_PROC(_hw_vmm, OID_AUTO, create, 1137 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, 1138 NULL, 0, sysctl_vmm_create, "A", 1139 NULL); 1140 1141 void 1142 vmmdev_init(void) 1143 { 1144 mtx_init(&vmmdev_mtx, "vmm device mutex", NULL, MTX_DEF); 1145 pr_allow_flag = prison_add_allow(NULL, "vmm", NULL, 1146 "Allow use of vmm in a jail."); 1147 } 1148 1149 int 1150 vmmdev_cleanup(void) 1151 { 1152 int error; 1153 1154 if (SLIST_EMPTY(&head)) 1155 error = 0; 1156 else 1157 error = EBUSY; 1158 1159 return (error); 1160 } 1161 1162 static int 1163 devmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t len, 1164 struct vm_object **objp, int nprot) 1165 { 1166 struct devmem_softc *dsc; 1167 vm_ooffset_t first, last; 1168 size_t seglen; 1169 int error; 1170 uint16_t lastcpu; 1171 bool sysmem; 1172 1173 dsc = cdev->si_drv1; 1174 if (dsc == NULL) { 1175 /* 'cdev' has been created but is not ready for use */ 1176 return (ENXIO); 1177 } 1178 1179 first = *offset; 1180 last = *offset + len; 1181 if ((nprot & PROT_EXEC) || first < 0 || first >= last) 1182 return (EINVAL); 1183 1184 lastcpu = vm_get_maxcpus(dsc->sc->vm) - 1; 1185 error = vcpu_lock_one(dsc->sc, lastcpu); 1186 if (error) 1187 return (error); 1188 1189 error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp); 1190 KASSERT(error == 0 && !sysmem && *objp != NULL, 1191 ("%s: invalid devmem segment %d", __func__, dsc->segid)); 1192 1193 vcpu_unlock_one(dsc->sc, lastcpu); 1194 1195 if (seglen >= last) { 1196 vm_object_reference(*objp); 1197 return (0); 1198 } else { 1199 return (EINVAL); 1200 } 1201 } 1202 1203 static struct cdevsw devmemsw = { 1204 .d_name = "devmem", 1205 .d_version = D_VERSION, 1206 .d_mmap_single = devmem_mmap_single, 1207 }; 1208 1209 static int 1210 devmem_create_cdev(const char *vmname, int segid, char *devname) 1211 { 1212 struct devmem_softc *dsc; 1213 struct vmmdev_softc *sc; 1214 struct cdev *cdev; 1215 int error; 1216 1217 error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL, 1218 UID_ROOT, GID_WHEEL, 0600, "vmm.io/%s.%s", vmname, devname); 1219 if (error) 1220 return (error); 1221 1222 dsc = malloc(sizeof(struct devmem_softc), M_VMMDEV, M_WAITOK | M_ZERO); 1223 1224 mtx_lock(&vmmdev_mtx); 1225 sc = vmmdev_lookup(vmname); 1226 KASSERT(sc != NULL, ("%s: vm %s softc not found", __func__, vmname)); 1227 if (sc->cdev == NULL) { 1228 /* virtual machine is being created or destroyed */ 1229 mtx_unlock(&vmmdev_mtx); 1230 free(dsc, M_VMMDEV); 1231 destroy_dev_sched_cb(cdev, NULL, 0); 1232 return (ENODEV); 1233 } 1234 1235 dsc->segid = segid; 1236 dsc->name = devname; 1237 dsc->cdev = cdev; 1238 dsc->sc = sc; 1239 SLIST_INSERT_HEAD(&sc->devmem, dsc, link); 1240 mtx_unlock(&vmmdev_mtx); 1241 1242 /* The 'cdev' is ready for use after 'si_drv1' is initialized */ 1243 cdev->si_drv1 = dsc; 1244 return (0); 1245 } 1246 1247 static void 1248 devmem_destroy(void *arg) 1249 { 1250 struct devmem_softc *dsc = arg; 1251 1252 KASSERT(dsc->cdev, ("%s: devmem cdev already destroyed", __func__)); 1253 dsc->cdev = NULL; 1254 dsc->sc = NULL; 1255 } 1256