112 	int error;  in vcpu_lock_one()  local
117 	error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN, true);  in vcpu_lock_one()
118 	return (error);  in vcpu_lock_one()
138 	int error, vcpu;  in vcpu_lock_all()  local
143 		error = vcpu_lock_one(sc, vcpu);  in vcpu_lock_all()
144 		if (error)  in vcpu_lock_all()
148 	if (error) {  in vcpu_lock_all()
153 	return (error);  in vcpu_lock_all()
194 	int error, off, c, prot;  in vmmdev_rw()  local
200 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_rw()
201 	if (error)  in vmmdev_rw()
202 		return (error);  in vmmdev_rw()
212 	error = vcpu_lock_one(sc, lastcpu);  in vmmdev_rw()
213 	if (error)  in vmmdev_rw()
214 		return (error);  in vmmdev_rw()
218 	while (uio->uio_resid > 0 && error == 0) {  in vmmdev_rw()
235 				error = uiomove(__DECONST(void *, zero_region),  in vmmdev_rw()
238 				error = EFAULT;  in vmmdev_rw()
240 			error = uiomove(hpa, c, uio);  in vmmdev_rw()
245 	return (error);  in vmmdev_rw()
254 	int error;  in get_memseg()  local
257 	error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL);  in get_memseg()
258 	if (error || mseg->len == 0)  in get_memseg()
259 		return (error);  in get_memseg()
268 		error = copystr(dsc->name, mseg->name, SPECNAMELEN + 1, NULL);  in get_memseg()
273 	return (error);  in get_memseg()
280 	int error;  in alloc_memseg()  local
283 	error = 0;  in alloc_memseg()
290 		error = copystr(mseg->name, name, SPECNAMELEN + 1, 0);  in alloc_memseg()
291 		if (error)  in alloc_memseg()
295 	error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem);  in alloc_memseg()
296 	if (error)  in alloc_memseg()
300 		error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name);  in alloc_memseg()
301 		if (error)  in alloc_memseg()
308 	return (error);  in alloc_memseg()
315 	int error, i;  in vm_get_register_set()  local
317 	error = 0;  in vm_get_register_set()
319 		error = vm_get_register(vm, vcpu, regnum[i], &regval[i]);  in vm_get_register_set()
320 		if (error)  in vm_get_register_set()
323 	return (error);  in vm_get_register_set()
330 	int error, i;  in vm_set_register_set()  local
332 	error = 0;  in vm_set_register_set()
334 		error = vm_set_register(vm, vcpu, regnum[i], regval[i]);  in vm_set_register_set()
335 		if (error)  in vm_set_register_set()
338 	return (error);  in vm_set_register_set()
345 	int error, vcpu, state_changed, size;  in vmmdev_ioctl()  local
380 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_ioctl()
381 	if (error)  in vmmdev_ioctl()
382 		return (error);  in vmmdev_ioctl()
419 		error = vcpu_lock_one(sc, vcpu);  in vmmdev_ioctl()
420 		if (error)  in vmmdev_ioctl()
435 		error = vcpu_lock_all(sc);  in vmmdev_ioctl()
436 		if (error)  in vmmdev_ioctl()
448 		error = vcpu_lock_one(sc, vcpu);  in vmmdev_ioctl()
449 		if (error)  in vmmdev_ioctl()
461 		error = vm_run(sc->vm, vmrun);  in vmmdev_ioctl()
465 		error = vm_suspend(sc->vm, vmsuspend->how);  in vmmdev_ioctl()
468 		error = vm_reinit(sc->vm);  in vmmdev_ioctl()
472 		error = vmm_stat_desc_copy(statdesc->index,  in vmmdev_ioctl()
480 		error = vmm_stat_copy(sc->vm, vmstats->cpuid,  in vmmdev_ioctl()
486 		error = ppt_setup_msi(sc->vm, pptmsi->vcpu,  in vmmdev_ioctl()
493 		error = ppt_setup_msix(sc->vm, pptmsix->vcpu,  in vmmdev_ioctl()
501 		error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,  in vmmdev_ioctl()
507 		error = vm_assign_pptdev(sc->vm, pptdev->bus, pptdev->slot,  in vmmdev_ioctl()
512 		error = vm_unassign_pptdev(sc->vm, pptdev->bus, pptdev->slot,  in vmmdev_ioctl()
517 		error = vm_inject_exception(sc->vm, vmexc->cpuid,  in vmmdev_ioctl()
523 		error = vm_inject_nmi(sc->vm, vmnmi->cpuid);  in vmmdev_ioctl()
527 		error = lapic_intr_edge(sc->vm, vmirq->cpuid, vmirq->vector);  in vmmdev_ioctl()
531 		error = lapic_set_local_intr(sc->vm, vmirq->cpuid,  in vmmdev_ioctl()
536 		error = lapic_intr_msi(sc->vm, vmmsi->addr, vmmsi->msg);  in vmmdev_ioctl()
540 		error = vioapic_assert_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
544 		error = vioapic_deassert_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
548 		error = vioapic_pulse_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
555 		error = vatpic_assert_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
556 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
557 			error = vioapic_assert_irq(sc->vm,  in vmmdev_ioctl()
562 		error = vatpic_deassert_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
563 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
564 			error = vioapic_deassert_irq(sc->vm,  in vmmdev_ioctl()
569 		error = vatpic_pulse_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
570 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
571 			error = vioapic_pulse_irq(sc->vm, isa_irq->ioapic_irq);  in vmmdev_ioctl()
575 		error = vatpic_set_irq_trigger(sc->vm,  in vmmdev_ioctl()
580 		error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid,  in vmmdev_ioctl()
585 		error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff,  in vmmdev_ioctl()
589 		error = alloc_memseg(sc, (struct vm_memseg *)data);  in vmmdev_ioctl()
592 		error = get_memseg(sc, (struct vm_memseg *)data);  in vmmdev_ioctl()
596 		error = vm_get_register(sc->vm, vmreg->cpuid, vmreg->regnum,  in vmmdev_ioctl()
601 		error = vm_set_register(sc->vm, vmreg->cpuid, vmreg->regnum,  in vmmdev_ioctl()
606 		error = vm_set_seg_desc(sc->vm, vmsegdesc->cpuid,  in vmmdev_ioctl()
612 		error = vm_get_seg_desc(sc->vm, vmsegdesc->cpuid,  in vmmdev_ioctl()
619 			error = EINVAL;  in vmmdev_ioctl()
626 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *  in vmmdev_ioctl()
628 		if (error == 0)  in vmmdev_ioctl()
629 			error = vm_get_register_set(sc->vm, vmregset->cpuid,  in vmmdev_ioctl()
631 		if (error == 0)  in vmmdev_ioctl()
632 			error = copyout(regvals, vmregset->regvals,  in vmmdev_ioctl()
640 			error = EINVAL;  in vmmdev_ioctl()
647 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *  in vmmdev_ioctl()
649 		if (error == 0)  in vmmdev_ioctl()
650 			error = copyin(vmregset->regvals, regvals,  in vmmdev_ioctl()
652 		if (error == 0)  in vmmdev_ioctl()
653 			error = vm_set_register_set(sc->vm, vmregset->cpuid,  in vmmdev_ioctl()
660 		error = vm_get_capability(sc->vm, vmcap->cpuid,  in vmmdev_ioctl()
666 		error = vm_set_capability(sc->vm, vmcap->cpuid,  in vmmdev_ioctl()
672 		error = vm_set_x2apic_state(sc->vm,  in vmmdev_ioctl()
677 		error = vm_get_x2apic_state(sc->vm,  in vmmdev_ioctl()
684 		error = 0;  in vmmdev_ioctl()
687 		error = vhpet_getcap((struct vm_hpet_cap *)data);  in vmmdev_ioctl()
694 		error = vm_gla2gpa(sc->vm, gg->vcpuid, &gg->paging, gg->gla,  in vmmdev_ioctl()
696 		KASSERT(error == 0 || error == EFAULT,  in vmmdev_ioctl()
697 		    ("%s: vm_gla2gpa unknown error %d", __func__, error));  in vmmdev_ioctl()
702 		error = vm_gla2gpa_nofault(sc->vm, gg->vcpuid, &gg->paging,  in vmmdev_ioctl()
704 		KASSERT(error == 0 || error == EFAULT,  in vmmdev_ioctl()
705 		    ("%s: vm_gla2gpa unknown error %d", __func__, error));  in vmmdev_ioctl()
709 		error = vm_activate_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
712 		error = 0;  in vmmdev_ioctl()
716 			error = ERANGE;  in vmmdev_ioctl()
727 			error = EINVAL;  in vmmdev_ioctl()
728 		if (error == 0)  in vmmdev_ioctl()
729 			error = copyout(cpuset, vm_cpuset->cpus, size);  in vmmdev_ioctl()
734 		error = vm_suspend_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
738 		error = vm_resume_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
742 		error = vm_exit_intinfo(sc->vm, vmii->vcpuid, vmii->info1);  in vmmdev_ioctl()
746 		error = vm_get_intinfo(sc->vm, vmii->vcpuid, &vmii->info1,  in vmmdev_ioctl()
751 		error = vrtc_nvram_write(sc->vm, rtcdata->offset,  in vmmdev_ioctl()
756 		error = vrtc_nvram_read(sc->vm, rtcdata->offset,  in vmmdev_ioctl()
761 		error = vrtc_set_time(sc->vm, rtctime->secs);  in vmmdev_ioctl()
764 		error = 0;  in vmmdev_ioctl()
769 		error = vm_restart_instruction(sc->vm, vcpu);  in vmmdev_ioctl()
773 		error = vm_set_topology(sc->vm, topology->sockets,  in vmmdev_ioctl()
780 		error = 0;  in vmmdev_ioctl()
783 		error = ENOTTY;  in vmmdev_ioctl()
794 	KASSERT(error >= 0, ("vmmdev_ioctl: invalid error return %d", error));  in vmmdev_ioctl()
795 	return (error);  in vmmdev_ioctl()
806 	int error, found, segid;  in vmmdev_mmap_single()  local
810 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_mmap_single()
811 	if (error)  in vmmdev_mmap_single()
812 		return (error);  in vmmdev_mmap_single()
829 	error = vcpu_lock_one(sc, lastcpu);  in vmmdev_mmap_single()
830 	if (error)  in vmmdev_mmap_single()
831 		return (error);  in vmmdev_mmap_single()
836 		error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len,  in vmmdev_mmap_single()
838 		if (error)  in vmmdev_mmap_single()
848 		error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp);  in vmmdev_mmap_single()
849 		KASSERT(error == 0 && *objp != NULL,  in vmmdev_mmap_single()
855 			error = EINVAL;  in vmmdev_mmap_single()
859 	return (error);  in vmmdev_mmap_single()
867 	int error;  in vmmdev_destroy()  local
869 	error = vcpu_lock_all(sc);  in vmmdev_destroy()
870 	KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error));  in vmmdev_destroy()
897 	int error;  in sysctl_vmm_destroy()  local
903 	error = vmm_priv_check(req->td->td_ucred);  in sysctl_vmm_destroy()
904 	if (error)  in sysctl_vmm_destroy()
905 		return (error);  in sysctl_vmm_destroy()
908 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);  in sysctl_vmm_destroy()
909 	if (error != 0 || req->newptr == NULL)  in sysctl_vmm_destroy()
910 		return (error);  in sysctl_vmm_destroy()
964 	int error;  in sysctl_vmm_create()  local
970 	error = vmm_priv_check(req->td->td_ucred);  in sysctl_vmm_create()
971 	if (error)  in sysctl_vmm_create()
972 		return (error);  in sysctl_vmm_create()
975 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);  in sysctl_vmm_create()
976 	if (error != 0 || req->newptr == NULL)  in sysctl_vmm_create()
977 		return (error);  in sysctl_vmm_create()
985 	error = vm_create(buf, &vm);  in sysctl_vmm_create()
986 	if (error != 0)  in sysctl_vmm_create()
987 		return (error);  in sysctl_vmm_create()
1010 	error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, NULL,  in sysctl_vmm_create()
1012 	if (error != 0) {  in sysctl_vmm_create()
1014 		return (error);  in sysctl_vmm_create()
1039 	int error;  in vmmdev_cleanup()  local
1042 		error = 0;  in vmmdev_cleanup()
1044 		error = EBUSY;  in vmmdev_cleanup()
1046 	return (error);  in vmmdev_cleanup()
1056 	int error;  in devmem_mmap_single()  local
1072 	error = vcpu_lock_one(dsc->sc, lastcpu);  in devmem_mmap_single()
1073 	if (error)  in devmem_mmap_single()
1074 		return (error);  in devmem_mmap_single()
1076 	error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp);  in devmem_mmap_single()
1077 	KASSERT(error == 0 && !sysmem && *objp != NULL,  in devmem_mmap_single()
1102 	int error;  in devmem_create_cdev()  local
1104 	error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL,  in devmem_create_cdev()
1106 	if (error)  in devmem_create_cdev()
1107 		return (error);  in devmem_create_cdev()