116 	int error;  in vcpu_lock_one()  local
121 	error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN, true);  in vcpu_lock_one()
122 	return (error);  in vcpu_lock_one()
142 	int error, vcpu;  in vcpu_lock_all()  local
147 		error = vcpu_lock_one(sc, vcpu);  in vcpu_lock_all()
148 		if (error)  in vcpu_lock_all()
152 	if (error) {  in vcpu_lock_all()
157 	return (error);  in vcpu_lock_all()
198 	int error, off, c, prot;  in vmmdev_rw()  local
204 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_rw()
205 	if (error)  in vmmdev_rw()
206 		return (error);  in vmmdev_rw()
216 	error = vcpu_lock_one(sc, lastcpu);  in vmmdev_rw()
217 	if (error)  in vmmdev_rw()
218 		return (error);  in vmmdev_rw()
222 	while (uio->uio_resid > 0 && error == 0) {  in vmmdev_rw()
239 				error = uiomove(__DECONST(void *, zero_region),  in vmmdev_rw()
242 				error = EFAULT;  in vmmdev_rw()
244 			error = uiomove(hpa, c, uio);  in vmmdev_rw()
249 	return (error);  in vmmdev_rw()
258 	int error;  in get_memseg()  local
261 	error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL);  in get_memseg()
262 	if (error || mseg->len == 0)  in get_memseg()
263 		return (error);  in get_memseg()
272 		error = copystr(dsc->name, mseg->name, len, NULL);  in get_memseg()
277 	return (error);  in get_memseg()
284 	int error;  in alloc_memseg()  local
287 	error = 0;  in alloc_memseg()
298 		error = copystr(mseg->name, name, len, NULL);  in alloc_memseg()
299 		if (error)  in alloc_memseg()
303 	error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem);  in alloc_memseg()
304 	if (error)  in alloc_memseg()
308 		error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name);  in alloc_memseg()
309 		if (error)  in alloc_memseg()
316 	return (error);  in alloc_memseg()
323 	int error, i;  in vm_get_register_set()  local
325 	error = 0;  in vm_get_register_set()
327 		error = vm_get_register(vm, vcpu, regnum[i], &regval[i]);  in vm_get_register_set()
328 		if (error)  in vm_get_register_set()
331 	return (error);  in vm_get_register_set()
338 	int error, i;  in vm_set_register_set()  local
340 	error = 0;  in vm_set_register_set()
342 		error = vm_set_register(vm, vcpu, regnum[i], regval[i]);  in vm_set_register_set()
343 		if (error)  in vm_set_register_set()
346 	return (error);  in vm_set_register_set()
353 	int error, vcpu, state_changed, size;  in vmmdev_ioctl()  local
392 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_ioctl()
393 	if (error)  in vmmdev_ioctl()
394 		return (error);  in vmmdev_ioctl()
431 		error = vcpu_lock_one(sc, vcpu);  in vmmdev_ioctl()
432 		if (error)  in vmmdev_ioctl()
450 		error = vcpu_lock_all(sc);  in vmmdev_ioctl()
451 		if (error)  in vmmdev_ioctl()
466 		error = vcpu_lock_one(sc, vcpu);  in vmmdev_ioctl()
467 		if (error)  in vmmdev_ioctl()
479 		error = vm_run(sc->vm, vmrun);  in vmmdev_ioctl()
483 		error = vm_suspend(sc->vm, vmsuspend->how);  in vmmdev_ioctl()
486 		error = vm_reinit(sc->vm);  in vmmdev_ioctl()
490 		error = vmm_stat_desc_copy(statdesc->index,  in vmmdev_ioctl()
498 		error = vmm_stat_copy(sc->vm, vmstats->cpuid,  in vmmdev_ioctl()
504 		error = ppt_setup_msi(sc->vm, pptmsi->vcpu,  in vmmdev_ioctl()
511 		error = ppt_setup_msix(sc->vm, pptmsix->vcpu,  in vmmdev_ioctl()
519 		error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot,  in vmmdev_ioctl()
524 		error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,  in vmmdev_ioctl()
530 		error = vm_assign_pptdev(sc->vm, pptdev->bus, pptdev->slot,  in vmmdev_ioctl()
535 		error = vm_unassign_pptdev(sc->vm, pptdev->bus, pptdev->slot,  in vmmdev_ioctl()
540 		error = vm_inject_exception(sc->vm, vmexc->cpuid,  in vmmdev_ioctl()
546 		error = vm_inject_nmi(sc->vm, vmnmi->cpuid);  in vmmdev_ioctl()
550 		error = lapic_intr_edge(sc->vm, vmirq->cpuid, vmirq->vector);  in vmmdev_ioctl()
554 		error = lapic_set_local_intr(sc->vm, vmirq->cpuid,  in vmmdev_ioctl()
559 		error = lapic_intr_msi(sc->vm, vmmsi->addr, vmmsi->msg);  in vmmdev_ioctl()
563 		error = vioapic_assert_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
567 		error = vioapic_deassert_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
571 		error = vioapic_pulse_irq(sc->vm, ioapic_irq->irq);  in vmmdev_ioctl()
599 			error = EINVAL;  in vmmdev_ioctl()
604 			error = mwrite(sc->vm, kernemu->vcpuid, kernemu->gpa,  in vmmdev_ioctl()
607 			error = mread(sc->vm, kernemu->vcpuid, kernemu->gpa,  in vmmdev_ioctl()
613 		error = vatpic_assert_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
614 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
615 			error = vioapic_assert_irq(sc->vm,  in vmmdev_ioctl()
620 		error = vatpic_deassert_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
621 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
622 			error = vioapic_deassert_irq(sc->vm,  in vmmdev_ioctl()
627 		error = vatpic_pulse_irq(sc->vm, isa_irq->atpic_irq);  in vmmdev_ioctl()
628 		if (error == 0 && isa_irq->ioapic_irq != -1)  in vmmdev_ioctl()
629 			error = vioapic_pulse_irq(sc->vm, isa_irq->ioapic_irq);  in vmmdev_ioctl()
633 		error = vatpic_set_irq_trigger(sc->vm,  in vmmdev_ioctl()
638 		error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid,  in vmmdev_ioctl()
643 		error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff,  in vmmdev_ioctl()
648 		error = alloc_memseg(sc, (struct vm_memseg *)data,  in vmmdev_ioctl()
653 		error = alloc_memseg(sc, (struct vm_memseg *)data,  in vmmdev_ioctl()
658 		error = get_memseg(sc, (struct vm_memseg *)data,  in vmmdev_ioctl()
663 		error = get_memseg(sc, (struct vm_memseg *)data,  in vmmdev_ioctl()
668 		error = vm_get_register(sc->vm, vmreg->cpuid, vmreg->regnum,  in vmmdev_ioctl()
673 		error = vm_set_register(sc->vm, vmreg->cpuid, vmreg->regnum,  in vmmdev_ioctl()
678 		error = vm_set_seg_desc(sc->vm, vmsegdesc->cpuid,  in vmmdev_ioctl()
684 		error = vm_get_seg_desc(sc->vm, vmsegdesc->cpuid,  in vmmdev_ioctl()
691 			error = EINVAL;  in vmmdev_ioctl()
698 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *  in vmmdev_ioctl()
700 		if (error == 0)  in vmmdev_ioctl()
701 			error = vm_get_register_set(sc->vm, vmregset->cpuid,  in vmmdev_ioctl()
703 		if (error == 0)  in vmmdev_ioctl()
704 			error = copyout(regvals, vmregset->regvals,  in vmmdev_ioctl()
712 			error = EINVAL;  in vmmdev_ioctl()
719 		error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *  in vmmdev_ioctl()
721 		if (error == 0)  in vmmdev_ioctl()
722 			error = copyin(vmregset->regvals, regvals,  in vmmdev_ioctl()
724 		if (error == 0)  in vmmdev_ioctl()
725 			error = vm_set_register_set(sc->vm, vmregset->cpuid,  in vmmdev_ioctl()
732 		error = vm_get_capability(sc->vm, vmcap->cpuid,  in vmmdev_ioctl()
738 		error = vm_set_capability(sc->vm, vmcap->cpuid,  in vmmdev_ioctl()
744 		error = vm_set_x2apic_state(sc->vm,  in vmmdev_ioctl()
749 		error = vm_get_x2apic_state(sc->vm,  in vmmdev_ioctl()
756 		error = 0;  in vmmdev_ioctl()
759 		error = vhpet_getcap((struct vm_hpet_cap *)data);  in vmmdev_ioctl()
766 		error = vm_gla2gpa(sc->vm, gg->vcpuid, &gg->paging, gg->gla,  in vmmdev_ioctl()
768 		KASSERT(error == 0 || error == EFAULT,  in vmmdev_ioctl()
769 		    ("%s: vm_gla2gpa unknown error %d", __func__, error));  in vmmdev_ioctl()
774 		error = vm_gla2gpa_nofault(sc->vm, gg->vcpuid, &gg->paging,  in vmmdev_ioctl()
776 		KASSERT(error == 0 || error == EFAULT,  in vmmdev_ioctl()
777 		    ("%s: vm_gla2gpa unknown error %d", __func__, error));  in vmmdev_ioctl()
781 		error = vm_activate_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
784 		error = 0;  in vmmdev_ioctl()
788 			error = ERANGE;  in vmmdev_ioctl()
799 			error = EINVAL;  in vmmdev_ioctl()
800 		if (error == 0)  in vmmdev_ioctl()
801 			error = copyout(cpuset, vm_cpuset->cpus, size);  in vmmdev_ioctl()
806 		error = vm_suspend_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
810 		error = vm_resume_cpu(sc->vm, vac->vcpuid);  in vmmdev_ioctl()
814 		error = vm_exit_intinfo(sc->vm, vmii->vcpuid, vmii->info1);  in vmmdev_ioctl()
818 		error = vm_get_intinfo(sc->vm, vmii->vcpuid, &vmii->info1,  in vmmdev_ioctl()
823 		error = vrtc_nvram_write(sc->vm, rtcdata->offset,  in vmmdev_ioctl()
828 		error = vrtc_nvram_read(sc->vm, rtcdata->offset,  in vmmdev_ioctl()
833 		error = vrtc_set_time(sc->vm, rtctime->secs);  in vmmdev_ioctl()
836 		error = 0;  in vmmdev_ioctl()
841 		error = vm_restart_instruction(sc->vm, vcpu);  in vmmdev_ioctl()
845 		error = vm_set_topology(sc->vm, topology->sockets,  in vmmdev_ioctl()
852 		error = 0;  in vmmdev_ioctl()
857 		error = vm_snapshot_req(sc->vm, snapshot_meta);  in vmmdev_ioctl()
860 		error = vm_restore_time(sc->vm);  in vmmdev_ioctl()
864 		error = ENOTTY;  in vmmdev_ioctl()
878 	KASSERT(error == ERESTART || error >= 0,  in vmmdev_ioctl()
879 	    ("vmmdev_ioctl: invalid error return %d", error));  in vmmdev_ioctl()
880 	return (error);  in vmmdev_ioctl()
891 	int error, found, segid;  in vmmdev_mmap_single()  local
895 	error = vmm_priv_check(curthread->td_ucred);  in vmmdev_mmap_single()
896 	if (error)  in vmmdev_mmap_single()
897 		return (error);  in vmmdev_mmap_single()
914 	error = vcpu_lock_one(sc, lastcpu);  in vmmdev_mmap_single()
915 	if (error)  in vmmdev_mmap_single()
916 		return (error);  in vmmdev_mmap_single()
921 		error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len,  in vmmdev_mmap_single()
923 		if (error)  in vmmdev_mmap_single()
933 		error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp);  in vmmdev_mmap_single()
934 		KASSERT(error == 0 && *objp != NULL,  in vmmdev_mmap_single()
940 			error = EINVAL;  in vmmdev_mmap_single()
944 	return (error);  in vmmdev_mmap_single()
952 	int error;  in vmmdev_destroy()  local
954 	error = vcpu_lock_all(sc);  in vmmdev_destroy()
955 	KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error));  in vmmdev_destroy()
986 	int error, buflen;  in sysctl_vmm_destroy()  local
988 	error = vmm_priv_check(req->td->td_ucred);  in sysctl_vmm_destroy()
989 	if (error)  in sysctl_vmm_destroy()
990 		return (error);  in sysctl_vmm_destroy()
995 	error = sysctl_handle_string(oidp, buf, buflen, req);  in sysctl_vmm_destroy()
996 	if (error != 0 || req->newptr == NULL)  in sysctl_vmm_destroy()
1003 		error = EINVAL;  in sysctl_vmm_destroy()
1034 	error = 0;  in sysctl_vmm_destroy()
1038 	return (error);  in sysctl_vmm_destroy()
1061 	int error, buflen;  in sysctl_vmm_create()  local
1063 	error = vmm_priv_check(req->td->td_ucred);  in sysctl_vmm_create()
1064 	if (error)  in sysctl_vmm_create()
1065 		return (error);  in sysctl_vmm_create()
1070 	error = sysctl_handle_string(oidp, buf, buflen, req);  in sysctl_vmm_create()
1071 	if (error != 0 || req->newptr == NULL)  in sysctl_vmm_create()
1078 		error = EEXIST;  in sysctl_vmm_create()
1082 	error = vm_create(buf, &vm);  in sysctl_vmm_create()
1083 	if (error != 0)  in sysctl_vmm_create()
1104 		error = EEXIST;  in sysctl_vmm_create()
1108 	error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, NULL,  in sysctl_vmm_create()
1110 	if (error != 0) {  in sysctl_vmm_create()
1122 	return (error);  in sysctl_vmm_create()
1140 	int error;  in vmmdev_cleanup()  local
1143 		error = 0;  in vmmdev_cleanup()
1145 		error = EBUSY;  in vmmdev_cleanup()
1147 	return (error);  in vmmdev_cleanup()
1157 	int error;  in devmem_mmap_single()  local
1173 	error = vcpu_lock_one(dsc->sc, lastcpu);  in devmem_mmap_single()
1174 	if (error)  in devmem_mmap_single()
1175 		return (error);  in devmem_mmap_single()
1177 	error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp);  in devmem_mmap_single()
1178 	KASSERT(error == 0 && !sysmem && *objp != NULL,  in devmem_mmap_single()
1203 	int error;  in devmem_create_cdev()  local
1205 	error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL,  in devmem_create_cdev()
1207 	if (error)  in devmem_create_cdev()
1208 		return (error);  in devmem_create_cdev()