1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011 NetApp, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "opt_bhyve_snapshot.h"
35
36 #include <sys/param.h>
37 #include <sys/kernel.h>
38 #include <sys/jail.h>
39 #include <sys/queue.h>
40 #include <sys/lock.h>
41 #include <sys/mutex.h>
42 #include <sys/malloc.h>
43 #include <sys/conf.h>
44 #include <sys/sysctl.h>
45 #include <sys/libkern.h>
46 #include <sys/ioccom.h>
47 #include <sys/mman.h>
48 #include <sys/uio.h>
49 #include <sys/proc.h>
50
51 #include <vm/vm.h>
52 #include <vm/pmap.h>
53 #include <vm/vm_map.h>
54 #include <vm/vm_object.h>
55
56 #include <machine/vmparam.h>
57 #include <machine/vmm.h>
58 #include <machine/vmm_dev.h>
59 #include <machine/vmm_instruction_emul.h>
60 #include <machine/vmm_snapshot.h>
61 #include <x86/apicreg.h>
62
63 #include "vmm_lapic.h"
64 #include "vmm_stat.h"
65 #include "vmm_mem.h"
66 #include "io/ppt.h"
67 #include "io/vatpic.h"
68 #include "io/vioapic.h"
69 #include "io/vhpet.h"
70 #include "io/vrtc.h"
71
72 struct devmem_softc {
73 int segid;
74 char *name;
75 struct cdev *cdev;
76 struct vmmdev_softc *sc;
77 SLIST_ENTRY(devmem_softc) link;
78 };
79
80 struct vmmdev_softc {
81 struct vm *vm; /* vm instance cookie */
82 struct cdev *cdev;
83 struct ucred *ucred;
84 SLIST_ENTRY(vmmdev_softc) link;
85 SLIST_HEAD(, devmem_softc) devmem;
86 int flags;
87 };
88 #define VSC_LINKED 0x01
89
90 static SLIST_HEAD(, vmmdev_softc) head;
91
92 static unsigned pr_allow_flag;
93 static struct mtx vmmdev_mtx;
94
95 static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev");
96
97 SYSCTL_DECL(_hw_vmm);
98
99 static int vmm_priv_check(struct ucred *ucred);
100 static int devmem_create_cdev(const char *vmname, int id, char *devmem);
101 static void devmem_destroy(void *arg);
102
103 static int
vmm_priv_check(struct ucred * ucred)104 vmm_priv_check(struct ucred *ucred)
105 {
106
107 if (jailed(ucred) &&
108 !(ucred->cr_prison->pr_allow & pr_allow_flag))
109 return (EPERM);
110
111 return (0);
112 }
113
114 static int
vcpu_lock_one(struct vmmdev_softc * sc,int vcpu)115 vcpu_lock_one(struct vmmdev_softc *sc, int vcpu)
116 {
117 int error;
118
119 if (vcpu < 0 || vcpu >= vm_get_maxcpus(sc->vm))
120 return (EINVAL);
121
122 error = vcpu_set_state(sc->vm, vcpu, VCPU_FROZEN, true);
123 return (error);
124 }
125
126 static void
vcpu_unlock_one(struct vmmdev_softc * sc,int vcpu)127 vcpu_unlock_one(struct vmmdev_softc *sc, int vcpu)
128 {
129 enum vcpu_state state;
130
131 state = vcpu_get_state(sc->vm, vcpu, NULL);
132 if (state != VCPU_FROZEN) {
133 panic("vcpu %s(%d) has invalid state %d", vm_name(sc->vm),
134 vcpu, state);
135 }
136
137 vcpu_set_state(sc->vm, vcpu, VCPU_IDLE, false);
138 }
139
140 static int
vcpu_lock_all(struct vmmdev_softc * sc)141 vcpu_lock_all(struct vmmdev_softc *sc)
142 {
143 int error, vcpu;
144 uint16_t maxcpus;
145
146 maxcpus = vm_get_maxcpus(sc->vm);
147 for (vcpu = 0; vcpu < maxcpus; vcpu++) {
148 error = vcpu_lock_one(sc, vcpu);
149 if (error)
150 break;
151 }
152
153 if (error) {
154 while (--vcpu >= 0)
155 vcpu_unlock_one(sc, vcpu);
156 }
157
158 return (error);
159 }
160
161 static void
vcpu_unlock_all(struct vmmdev_softc * sc)162 vcpu_unlock_all(struct vmmdev_softc *sc)
163 {
164 int vcpu;
165 uint16_t maxcpus;
166
167 maxcpus = vm_get_maxcpus(sc->vm);
168 for (vcpu = 0; vcpu < maxcpus; vcpu++)
169 vcpu_unlock_one(sc, vcpu);
170 }
171
172 static struct vmmdev_softc *
vmmdev_lookup(const char * name)173 vmmdev_lookup(const char *name)
174 {
175 struct vmmdev_softc *sc;
176
177 #ifdef notyet /* XXX kernel is not compiled with invariants */
178 mtx_assert(&vmmdev_mtx, MA_OWNED);
179 #endif
180
181 SLIST_FOREACH(sc, &head, link) {
182 if (strcmp(name, vm_name(sc->vm)) == 0)
183 break;
184 }
185
186 if (sc == NULL)
187 return (NULL);
188
189 if (cr_cansee(curthread->td_ucred, sc->ucred))
190 return (NULL);
191
192 return (sc);
193 }
194
195 static struct vmmdev_softc *
vmmdev_lookup2(struct cdev * cdev)196 vmmdev_lookup2(struct cdev *cdev)
197 {
198
199 return (cdev->si_drv1);
200 }
201
202 static int
vmmdev_rw(struct cdev * cdev,struct uio * uio,int flags)203 vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags)
204 {
205 int error, off, c, prot;
206 vm_paddr_t gpa, maxaddr;
207 void *hpa, *cookie;
208 struct vmmdev_softc *sc;
209 uint16_t lastcpu;
210
211 error = vmm_priv_check(curthread->td_ucred);
212 if (error)
213 return (error);
214
215 sc = vmmdev_lookup2(cdev);
216 if (sc == NULL)
217 return (ENXIO);
218
219 /*
220 * Get a read lock on the guest memory map by freezing any vcpu.
221 */
222 lastcpu = vm_get_maxcpus(sc->vm) - 1;
223 error = vcpu_lock_one(sc, lastcpu);
224 if (error)
225 return (error);
226
227 prot = (uio->uio_rw == UIO_WRITE ? VM_PROT_WRITE : VM_PROT_READ);
228 maxaddr = vmm_sysmem_maxaddr(sc->vm);
229 while (uio->uio_resid > 0 && error == 0) {
230 gpa = uio->uio_offset;
231 off = gpa & PAGE_MASK;
232 c = min(uio->uio_resid, PAGE_SIZE - off);
233
234 /*
235 * The VM has a hole in its physical memory map. If we want to
236 * use 'dd' to inspect memory beyond the hole we need to
237 * provide bogus data for memory that lies in the hole.
238 *
239 * Since this device does not support lseek(2), dd(1) will
240 * read(2) blocks of data to simulate the lseek(2).
241 */
242 hpa = vm_gpa_hold(sc->vm, lastcpu, gpa, c,
243 prot, &cookie);
244 if (hpa == NULL) {
245 if (uio->uio_rw == UIO_READ && gpa < maxaddr)
246 error = uiomove(__DECONST(void *, zero_region),
247 c, uio);
248 else
249 error = EFAULT;
250 } else {
251 error = uiomove(hpa, c, uio);
252 vm_gpa_release(cookie);
253 }
254 }
255 vcpu_unlock_one(sc, lastcpu);
256 return (error);
257 }
258
259 CTASSERT(sizeof(((struct vm_memseg *)0)->name) >= VM_MAX_SUFFIXLEN + 1);
260
261 static int
get_memseg(struct vmmdev_softc * sc,struct vm_memseg * mseg,size_t len)262 get_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len)
263 {
264 struct devmem_softc *dsc;
265 int error;
266 bool sysmem;
267
268 error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL);
269 if (error || mseg->len == 0)
270 return (error);
271
272 if (!sysmem) {
273 SLIST_FOREACH(dsc, &sc->devmem, link) {
274 if (dsc->segid == mseg->segid)
275 break;
276 }
277 KASSERT(dsc != NULL, ("%s: devmem segment %d not found",
278 __func__, mseg->segid));
279 error = copystr(dsc->name, mseg->name, len, NULL);
280 } else {
281 bzero(mseg->name, len);
282 }
283
284 return (error);
285 }
286
287 static int
alloc_memseg(struct vmmdev_softc * sc,struct vm_memseg * mseg,size_t len)288 alloc_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg, size_t len)
289 {
290 char *name;
291 int error;
292 bool sysmem;
293
294 error = 0;
295 name = NULL;
296 sysmem = true;
297
298 /*
299 * The allocation is lengthened by 1 to hold a terminating NUL. It'll
300 * by stripped off when devfs processes the full string.
301 */
302 if (VM_MEMSEG_NAME(mseg)) {
303 sysmem = false;
304 name = malloc(len, M_VMMDEV, M_WAITOK);
305 error = copystr(mseg->name, name, len, NULL);
306 if (error)
307 goto done;
308 }
309
310 error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem);
311 if (error)
312 goto done;
313
314 if (VM_MEMSEG_NAME(mseg)) {
315 error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name);
316 if (error)
317 vm_free_memseg(sc->vm, mseg->segid);
318 else
319 name = NULL; /* freed when 'cdev' is destroyed */
320 }
321 done:
322 free(name, M_VMMDEV);
323 return (error);
324 }
325
326 static int
vm_get_register_set(struct vm * vm,int vcpu,unsigned int count,int * regnum,uint64_t * regval)327 vm_get_register_set(struct vm *vm, int vcpu, unsigned int count, int *regnum,
328 uint64_t *regval)
329 {
330 int error, i;
331
332 error = 0;
333 for (i = 0; i < count; i++) {
334 error = vm_get_register(vm, vcpu, regnum[i], ®val[i]);
335 if (error)
336 break;
337 }
338 return (error);
339 }
340
341 static int
vm_set_register_set(struct vm * vm,int vcpu,unsigned int count,int * regnum,uint64_t * regval)342 vm_set_register_set(struct vm *vm, int vcpu, unsigned int count, int *regnum,
343 uint64_t *regval)
344 {
345 int error, i;
346
347 error = 0;
348 for (i = 0; i < count; i++) {
349 error = vm_set_register(vm, vcpu, regnum[i], regval[i]);
350 if (error)
351 break;
352 }
353 return (error);
354 }
355
356 static int
vmmdev_ioctl(struct cdev * cdev,u_long cmd,caddr_t data,int fflag,struct thread * td)357 vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
358 struct thread *td)
359 {
360 int error, vcpu, state_changed, size;
361 cpuset_t *cpuset;
362 struct vmmdev_softc *sc;
363 struct vm_register *vmreg;
364 struct vm_seg_desc *vmsegdesc;
365 struct vm_register_set *vmregset;
366 struct vm_run *vmrun;
367 struct vm_exception *vmexc;
368 struct vm_lapic_irq *vmirq;
369 struct vm_lapic_msi *vmmsi;
370 struct vm_ioapic_irq *ioapic_irq;
371 struct vm_isa_irq *isa_irq;
372 struct vm_isa_irq_trigger *isa_irq_trigger;
373 struct vm_capability *vmcap;
374 struct vm_pptdev *pptdev;
375 struct vm_pptdev_mmio *pptmmio;
376 struct vm_pptdev_msi *pptmsi;
377 struct vm_pptdev_msix *pptmsix;
378 struct vm_nmi *vmnmi;
379 struct vm_stats *vmstats;
380 struct vm_stat_desc *statdesc;
381 struct vm_x2apic *x2apic;
382 struct vm_gpa_pte *gpapte;
383 struct vm_suspend *vmsuspend;
384 struct vm_gla2gpa *gg;
385 struct vm_activate_cpu *vac;
386 struct vm_cpuset *vm_cpuset;
387 struct vm_intinfo *vmii;
388 struct vm_rtc_time *rtctime;
389 struct vm_rtc_data *rtcdata;
390 struct vm_memmap *mm;
391 struct vm_munmap *mu;
392 struct vm_cpu_topology *topology;
393 struct vm_readwrite_kernemu_device *kernemu;
394 uint64_t *regvals;
395 int *regnums;
396 #ifdef BHYVE_SNAPSHOT
397 struct vm_snapshot_meta *snapshot_meta;
398 #endif
399
400 error = vmm_priv_check(curthread->td_ucred);
401 if (error)
402 return (error);
403
404 sc = vmmdev_lookup2(cdev);
405 if (sc == NULL)
406 return (ENXIO);
407
408 vcpu = -1;
409 state_changed = 0;
410
411 /*
412 * Some VMM ioctls can operate only on vcpus that are not running.
413 */
414 switch (cmd) {
415 case VM_RUN:
416 case VM_GET_REGISTER:
417 case VM_SET_REGISTER:
418 case VM_GET_SEGMENT_DESCRIPTOR:
419 case VM_SET_SEGMENT_DESCRIPTOR:
420 case VM_GET_REGISTER_SET:
421 case VM_SET_REGISTER_SET:
422 case VM_INJECT_EXCEPTION:
423 case VM_GET_CAPABILITY:
424 case VM_SET_CAPABILITY:
425 case VM_PPTDEV_MSI:
426 case VM_PPTDEV_MSIX:
427 case VM_SET_X2APIC_STATE:
428 case VM_GLA2GPA:
429 case VM_GLA2GPA_NOFAULT:
430 case VM_ACTIVATE_CPU:
431 case VM_SET_INTINFO:
432 case VM_GET_INTINFO:
433 case VM_RESTART_INSTRUCTION:
434 /*
435 * XXX fragile, handle with care
436 * Assumes that the first field of the ioctl data is the vcpu.
437 */
438 vcpu = *(int *)data;
439 error = vcpu_lock_one(sc, vcpu);
440 if (error)
441 goto done;
442 state_changed = 1;
443 break;
444
445 case VM_MAP_PPTDEV_MMIO:
446 case VM_UNMAP_PPTDEV_MMIO:
447 case VM_BIND_PPTDEV:
448 case VM_UNBIND_PPTDEV:
449 #ifdef COMPAT_FREEBSD12
450 case VM_ALLOC_MEMSEG_FBSD12:
451 #endif
452 case VM_ALLOC_MEMSEG:
453 case VM_MMAP_MEMSEG:
454 case VM_MUNMAP_MEMSEG:
455 case VM_REINIT:
456 /*
457 * ioctls that operate on the entire virtual machine must
458 * prevent all vcpus from running.
459 */
460 error = vcpu_lock_all(sc);
461 if (error)
462 goto done;
463 state_changed = 2;
464 break;
465
466 #ifdef COMPAT_FREEBSD12
467 case VM_GET_MEMSEG_FBSD12:
468 #endif
469 case VM_GET_MEMSEG:
470 case VM_MMAP_GETNEXT:
471 /*
472 * Lock a vcpu to make sure that the memory map cannot be
473 * modified while it is being inspected.
474 */
475 vcpu = vm_get_maxcpus(sc->vm) - 1;
476 error = vcpu_lock_one(sc, vcpu);
477 if (error)
478 goto done;
479 state_changed = 1;
480 break;
481
482 default:
483 break;
484 }
485
486 switch(cmd) {
487 case VM_RUN:
488 vmrun = (struct vm_run *)data;
489 error = vm_run(sc->vm, vmrun);
490 break;
491 case VM_SUSPEND:
492 vmsuspend = (struct vm_suspend *)data;
493 error = vm_suspend(sc->vm, vmsuspend->how);
494 break;
495 case VM_REINIT:
496 error = vm_reinit(sc->vm);
497 break;
498 case VM_STAT_DESC: {
499 statdesc = (struct vm_stat_desc *)data;
500 error = vmm_stat_desc_copy(statdesc->index,
501 statdesc->desc, sizeof(statdesc->desc));
502 break;
503 }
504 case VM_STATS: {
505 CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_ELEMS);
506 vmstats = (struct vm_stats *)data;
507 getmicrotime(&vmstats->tv);
508 error = vmm_stat_copy(sc->vm, vmstats->cpuid,
509 &vmstats->num_entries, vmstats->statbuf);
510 break;
511 }
512 case VM_PPTDEV_MSI:
513 pptmsi = (struct vm_pptdev_msi *)data;
514 error = ppt_setup_msi(sc->vm, pptmsi->vcpu,
515 pptmsi->bus, pptmsi->slot, pptmsi->func,
516 pptmsi->addr, pptmsi->msg,
517 pptmsi->numvec);
518 break;
519 case VM_PPTDEV_MSIX:
520 pptmsix = (struct vm_pptdev_msix *)data;
521 error = ppt_setup_msix(sc->vm, pptmsix->vcpu,
522 pptmsix->bus, pptmsix->slot,
523 pptmsix->func, pptmsix->idx,
524 pptmsix->addr, pptmsix->msg,
525 pptmsix->vector_control);
526 break;
527 case VM_PPTDEV_DISABLE_MSIX:
528 pptdev = (struct vm_pptdev *)data;
529 error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot,
530 pptdev->func);
531 break;
532 case VM_MAP_PPTDEV_MMIO:
533 pptmmio = (struct vm_pptdev_mmio *)data;
534 error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
535 pptmmio->func, pptmmio->gpa, pptmmio->len,
536 pptmmio->hpa);
537 break;
538 case VM_UNMAP_PPTDEV_MMIO:
539 pptmmio = (struct vm_pptdev_mmio *)data;
540 error = ppt_unmap_mmio(sc->vm, pptmmio->bus, pptmmio->slot,
541 pptmmio->func, pptmmio->gpa, pptmmio->len);
542 break;
543 case VM_BIND_PPTDEV:
544 pptdev = (struct vm_pptdev *)data;
545 error = vm_assign_pptdev(sc->vm, pptdev->bus, pptdev->slot,
546 pptdev->func);
547 break;
548 case VM_UNBIND_PPTDEV:
549 pptdev = (struct vm_pptdev *)data;
550 error = vm_unassign_pptdev(sc->vm, pptdev->bus, pptdev->slot,
551 pptdev->func);
552 break;
553 case VM_INJECT_EXCEPTION:
554 vmexc = (struct vm_exception *)data;
555 error = vm_inject_exception(sc->vm, vmexc->cpuid,
556 vmexc->vector, vmexc->error_code_valid, vmexc->error_code,
557 vmexc->restart_instruction);
558 break;
559 case VM_INJECT_NMI:
560 vmnmi = (struct vm_nmi *)data;
561 error = vm_inject_nmi(sc->vm, vmnmi->cpuid);
562 break;
563 case VM_LAPIC_IRQ:
564 vmirq = (struct vm_lapic_irq *)data;
565 error = lapic_intr_edge(sc->vm, vmirq->cpuid, vmirq->vector);
566 break;
567 case VM_LAPIC_LOCAL_IRQ:
568 vmirq = (struct vm_lapic_irq *)data;
569 error = lapic_set_local_intr(sc->vm, vmirq->cpuid,
570 vmirq->vector);
571 break;
572 case VM_LAPIC_MSI:
573 vmmsi = (struct vm_lapic_msi *)data;
574 error = lapic_intr_msi(sc->vm, vmmsi->addr, vmmsi->msg);
575 break;
576 case VM_IOAPIC_ASSERT_IRQ:
577 ioapic_irq = (struct vm_ioapic_irq *)data;
578 error = vioapic_assert_irq(sc->vm, ioapic_irq->irq);
579 break;
580 case VM_IOAPIC_DEASSERT_IRQ:
581 ioapic_irq = (struct vm_ioapic_irq *)data;
582 error = vioapic_deassert_irq(sc->vm, ioapic_irq->irq);
583 break;
584 case VM_IOAPIC_PULSE_IRQ:
585 ioapic_irq = (struct vm_ioapic_irq *)data;
586 error = vioapic_pulse_irq(sc->vm, ioapic_irq->irq);
587 break;
588 case VM_IOAPIC_PINCOUNT:
589 *(int *)data = vioapic_pincount(sc->vm);
590 break;
591 case VM_SET_KERNEMU_DEV:
592 case VM_GET_KERNEMU_DEV: {
593 mem_region_write_t mwrite;
594 mem_region_read_t mread;
595 bool arg;
596
597 kernemu = (void *)data;
598
599 if (kernemu->access_width > 0)
600 size = (1u << kernemu->access_width);
601 else
602 size = 1;
603
604 if (kernemu->gpa >= DEFAULT_APIC_BASE && kernemu->gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
605 mread = lapic_mmio_read;
606 mwrite = lapic_mmio_write;
607 } else if (kernemu->gpa >= VIOAPIC_BASE && kernemu->gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
608 mread = vioapic_mmio_read;
609 mwrite = vioapic_mmio_write;
610 } else if (kernemu->gpa >= VHPET_BASE && kernemu->gpa < VHPET_BASE + VHPET_SIZE) {
611 mread = vhpet_mmio_read;
612 mwrite = vhpet_mmio_write;
613 } else {
614 error = EINVAL;
615 break;
616 }
617
618 if (cmd == VM_SET_KERNEMU_DEV)
619 error = mwrite(sc->vm, kernemu->vcpuid, kernemu->gpa,
620 kernemu->value, size, &arg);
621 else
622 error = mread(sc->vm, kernemu->vcpuid, kernemu->gpa,
623 &kernemu->value, size, &arg);
624 break;
625 }
626 case VM_ISA_ASSERT_IRQ:
627 isa_irq = (struct vm_isa_irq *)data;
628 error = vatpic_assert_irq(sc->vm, isa_irq->atpic_irq);
629 if (error == 0 && isa_irq->ioapic_irq != -1)
630 error = vioapic_assert_irq(sc->vm,
631 isa_irq->ioapic_irq);
632 break;
633 case VM_ISA_DEASSERT_IRQ:
634 isa_irq = (struct vm_isa_irq *)data;
635 error = vatpic_deassert_irq(sc->vm, isa_irq->atpic_irq);
636 if (error == 0 && isa_irq->ioapic_irq != -1)
637 error = vioapic_deassert_irq(sc->vm,
638 isa_irq->ioapic_irq);
639 break;
640 case VM_ISA_PULSE_IRQ:
641 isa_irq = (struct vm_isa_irq *)data;
642 error = vatpic_pulse_irq(sc->vm, isa_irq->atpic_irq);
643 if (error == 0 && isa_irq->ioapic_irq != -1)
644 error = vioapic_pulse_irq(sc->vm, isa_irq->ioapic_irq);
645 break;
646 case VM_ISA_SET_IRQ_TRIGGER:
647 isa_irq_trigger = (struct vm_isa_irq_trigger *)data;
648 error = vatpic_set_irq_trigger(sc->vm,
649 isa_irq_trigger->atpic_irq, isa_irq_trigger->trigger);
650 break;
651 case VM_MMAP_GETNEXT:
652 mm = (struct vm_memmap *)data;
653 error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid,
654 &mm->segoff, &mm->len, &mm->prot, &mm->flags);
655 break;
656 case VM_MMAP_MEMSEG:
657 mm = (struct vm_memmap *)data;
658 error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff,
659 mm->len, mm->prot, mm->flags);
660 break;
661 case VM_MUNMAP_MEMSEG:
662 mu = (struct vm_munmap *)data;
663 error = vm_munmap_memseg(sc->vm, mu->gpa, mu->len);
664 break;
665 #ifdef COMPAT_FREEBSD12
666 case VM_ALLOC_MEMSEG_FBSD12:
667 error = alloc_memseg(sc, (struct vm_memseg *)data,
668 sizeof(((struct vm_memseg_fbsd12 *)0)->name));
669 break;
670 #endif
671 case VM_ALLOC_MEMSEG:
672 error = alloc_memseg(sc, (struct vm_memseg *)data,
673 sizeof(((struct vm_memseg *)0)->name));
674 break;
675 #ifdef COMPAT_FREEBSD12
676 case VM_GET_MEMSEG_FBSD12:
677 error = get_memseg(sc, (struct vm_memseg *)data,
678 sizeof(((struct vm_memseg_fbsd12 *)0)->name));
679 break;
680 #endif
681 case VM_GET_MEMSEG:
682 error = get_memseg(sc, (struct vm_memseg *)data,
683 sizeof(((struct vm_memseg *)0)->name));
684 break;
685 case VM_GET_REGISTER:
686 vmreg = (struct vm_register *)data;
687 error = vm_get_register(sc->vm, vmreg->cpuid, vmreg->regnum,
688 &vmreg->regval);
689 break;
690 case VM_SET_REGISTER:
691 vmreg = (struct vm_register *)data;
692 error = vm_set_register(sc->vm, vmreg->cpuid, vmreg->regnum,
693 vmreg->regval);
694 break;
695 case VM_SET_SEGMENT_DESCRIPTOR:
696 vmsegdesc = (struct vm_seg_desc *)data;
697 error = vm_set_seg_desc(sc->vm, vmsegdesc->cpuid,
698 vmsegdesc->regnum,
699 &vmsegdesc->desc);
700 break;
701 case VM_GET_SEGMENT_DESCRIPTOR:
702 vmsegdesc = (struct vm_seg_desc *)data;
703 error = vm_get_seg_desc(sc->vm, vmsegdesc->cpuid,
704 vmsegdesc->regnum,
705 &vmsegdesc->desc);
706 break;
707 case VM_GET_REGISTER_SET:
708 vmregset = (struct vm_register_set *)data;
709 if (vmregset->count > VM_REG_LAST) {
710 error = EINVAL;
711 break;
712 }
713 regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
714 M_WAITOK);
715 regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
716 M_WAITOK);
717 error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
718 vmregset->count);
719 if (error == 0)
720 error = vm_get_register_set(sc->vm, vmregset->cpuid,
721 vmregset->count, regnums, regvals);
722 if (error == 0)
723 error = copyout(regvals, vmregset->regvals,
724 sizeof(regvals[0]) * vmregset->count);
725 free(regvals, M_VMMDEV);
726 free(regnums, M_VMMDEV);
727 break;
728 case VM_SET_REGISTER_SET:
729 vmregset = (struct vm_register_set *)data;
730 if (vmregset->count > VM_REG_LAST) {
731 error = EINVAL;
732 break;
733 }
734 regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV,
735 M_WAITOK);
736 regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV,
737 M_WAITOK);
738 error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) *
739 vmregset->count);
740 if (error == 0)
741 error = copyin(vmregset->regvals, regvals,
742 sizeof(regvals[0]) * vmregset->count);
743 if (error == 0)
744 error = vm_set_register_set(sc->vm, vmregset->cpuid,
745 vmregset->count, regnums, regvals);
746 free(regvals, M_VMMDEV);
747 free(regnums, M_VMMDEV);
748 break;
749 case VM_GET_CAPABILITY:
750 vmcap = (struct vm_capability *)data;
751 error = vm_get_capability(sc->vm, vmcap->cpuid,
752 vmcap->captype,
753 &vmcap->capval);
754 break;
755 case VM_SET_CAPABILITY:
756 vmcap = (struct vm_capability *)data;
757 error = vm_set_capability(sc->vm, vmcap->cpuid,
758 vmcap->captype,
759 vmcap->capval);
760 break;
761 case VM_SET_X2APIC_STATE:
762 x2apic = (struct vm_x2apic *)data;
763 error = vm_set_x2apic_state(sc->vm,
764 x2apic->cpuid, x2apic->state);
765 break;
766 case VM_GET_X2APIC_STATE:
767 x2apic = (struct vm_x2apic *)data;
768 error = vm_get_x2apic_state(sc->vm,
769 x2apic->cpuid, &x2apic->state);
770 break;
771 case VM_GET_GPA_PMAP:
772 gpapte = (struct vm_gpa_pte *)data;
773 pmap_get_mapping(vmspace_pmap(vm_get_vmspace(sc->vm)),
774 gpapte->gpa, gpapte->pte, &gpapte->ptenum);
775 error = 0;
776 break;
777 case VM_GET_HPET_CAPABILITIES:
778 error = vhpet_getcap((struct vm_hpet_cap *)data);
779 break;
780 case VM_GLA2GPA: {
781 CTASSERT(PROT_READ == VM_PROT_READ);
782 CTASSERT(PROT_WRITE == VM_PROT_WRITE);
783 CTASSERT(PROT_EXEC == VM_PROT_EXECUTE);
784 gg = (struct vm_gla2gpa *)data;
785 error = vm_gla2gpa(sc->vm, gg->vcpuid, &gg->paging, gg->gla,
786 gg->prot, &gg->gpa, &gg->fault);
787 KASSERT(error == 0 || error == EFAULT,
788 ("%s: vm_gla2gpa unknown error %d", __func__, error));
789 break;
790 }
791 case VM_GLA2GPA_NOFAULT:
792 gg = (struct vm_gla2gpa *)data;
793 error = vm_gla2gpa_nofault(sc->vm, gg->vcpuid, &gg->paging,
794 gg->gla, gg->prot, &gg->gpa, &gg->fault);
795 KASSERT(error == 0 || error == EFAULT,
796 ("%s: vm_gla2gpa unknown error %d", __func__, error));
797 break;
798 case VM_ACTIVATE_CPU:
799 vac = (struct vm_activate_cpu *)data;
800 error = vm_activate_cpu(sc->vm, vac->vcpuid);
801 break;
802 case VM_GET_CPUS:
803 error = 0;
804 vm_cpuset = (struct vm_cpuset *)data;
805 size = vm_cpuset->cpusetsize;
806 if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) {
807 error = ERANGE;
808 break;
809 }
810 cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO);
811 if (vm_cpuset->which == VM_ACTIVE_CPUS)
812 *cpuset = vm_active_cpus(sc->vm);
813 else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
814 *cpuset = vm_suspended_cpus(sc->vm);
815 else if (vm_cpuset->which == VM_DEBUG_CPUS)
816 *cpuset = vm_debug_cpus(sc->vm);
817 else
818 error = EINVAL;
819 if (error == 0)
820 error = copyout(cpuset, vm_cpuset->cpus, size);
821 free(cpuset, M_TEMP);
822 break;
823 case VM_SUSPEND_CPU:
824 vac = (struct vm_activate_cpu *)data;
825 error = vm_suspend_cpu(sc->vm, vac->vcpuid);
826 break;
827 case VM_RESUME_CPU:
828 vac = (struct vm_activate_cpu *)data;
829 error = vm_resume_cpu(sc->vm, vac->vcpuid);
830 break;
831 case VM_SET_INTINFO:
832 vmii = (struct vm_intinfo *)data;
833 error = vm_exit_intinfo(sc->vm, vmii->vcpuid, vmii->info1);
834 break;
835 case VM_GET_INTINFO:
836 vmii = (struct vm_intinfo *)data;
837 error = vm_get_intinfo(sc->vm, vmii->vcpuid, &vmii->info1,
838 &vmii->info2);
839 break;
840 case VM_RTC_WRITE:
841 rtcdata = (struct vm_rtc_data *)data;
842 error = vrtc_nvram_write(sc->vm, rtcdata->offset,
843 rtcdata->value);
844 break;
845 case VM_RTC_READ:
846 rtcdata = (struct vm_rtc_data *)data;
847 error = vrtc_nvram_read(sc->vm, rtcdata->offset,
848 &rtcdata->value);
849 break;
850 case VM_RTC_SETTIME:
851 rtctime = (struct vm_rtc_time *)data;
852 error = vrtc_set_time(sc->vm, rtctime->secs);
853 break;
854 case VM_RTC_GETTIME:
855 error = 0;
856 rtctime = (struct vm_rtc_time *)data;
857 rtctime->secs = vrtc_get_time(sc->vm);
858 break;
859 case VM_RESTART_INSTRUCTION:
860 error = vm_restart_instruction(sc->vm, vcpu);
861 break;
862 case VM_SET_TOPOLOGY:
863 topology = (struct vm_cpu_topology *)data;
864 error = vm_set_topology(sc->vm, topology->sockets,
865 topology->cores, topology->threads, topology->maxcpus);
866 break;
867 case VM_GET_TOPOLOGY:
868 topology = (struct vm_cpu_topology *)data;
869 vm_get_topology(sc->vm, &topology->sockets, &topology->cores,
870 &topology->threads, &topology->maxcpus);
871 error = 0;
872 break;
873 #ifdef BHYVE_SNAPSHOT
874 case VM_SNAPSHOT_REQ:
875 snapshot_meta = (struct vm_snapshot_meta *)data;
876 error = vm_snapshot_req(sc->vm, snapshot_meta);
877 break;
878 case VM_RESTORE_TIME:
879 error = vm_restore_time(sc->vm);
880 break;
881 #endif
882 default:
883 error = ENOTTY;
884 break;
885 }
886
887 if (state_changed == 1)
888 vcpu_unlock_one(sc, vcpu);
889 else if (state_changed == 2)
890 vcpu_unlock_all(sc);
891
892 done:
893 /*
894 * Make sure that no handler returns a kernel-internal
895 * error value to userspace.
896 */
897 KASSERT(error == ERESTART || error >= 0,
898 ("vmmdev_ioctl: invalid error return %d", error));
899 return (error);
900 }
901
902 static int
vmmdev_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t mapsize,struct vm_object ** objp,int nprot)903 vmmdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize,
904 struct vm_object **objp, int nprot)
905 {
906 struct vmmdev_softc *sc;
907 vm_paddr_t gpa;
908 size_t len;
909 vm_ooffset_t segoff, first, last;
910 int error, found, segid;
911 uint16_t lastcpu;
912 bool sysmem;
913
914 error = vmm_priv_check(curthread->td_ucred);
915 if (error)
916 return (error);
917
918 first = *offset;
919 last = first + mapsize;
920 if ((nprot & PROT_EXEC) || first < 0 || first >= last)
921 return (EINVAL);
922
923 sc = vmmdev_lookup2(cdev);
924 if (sc == NULL) {
925 /* virtual machine is in the process of being created */
926 return (EINVAL);
927 }
928
929 /*
930 * Get a read lock on the guest memory map by freezing any vcpu.
931 */
932 lastcpu = vm_get_maxcpus(sc->vm) - 1;
933 error = vcpu_lock_one(sc, lastcpu);
934 if (error)
935 return (error);
936
937 gpa = 0;
938 found = 0;
939 while (!found) {
940 error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len,
941 NULL, NULL);
942 if (error)
943 break;
944
945 if (first >= gpa && last <= gpa + len)
946 found = 1;
947 else
948 gpa += len;
949 }
950
951 if (found) {
952 error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp);
953 KASSERT(error == 0 && *objp != NULL,
954 ("%s: invalid memory segment %d", __func__, segid));
955 if (sysmem) {
956 vm_object_reference(*objp);
957 *offset = segoff + (first - gpa);
958 } else {
959 error = EINVAL;
960 }
961 }
962 vcpu_unlock_one(sc, lastcpu);
963 return (error);
964 }
965
966 static void
vmmdev_destroy(void * arg)967 vmmdev_destroy(void *arg)
968 {
969 struct vmmdev_softc *sc = arg;
970 struct devmem_softc *dsc;
971 int error;
972
973 error = vcpu_lock_all(sc);
974 KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error));
975
976 while ((dsc = SLIST_FIRST(&sc->devmem)) != NULL) {
977 KASSERT(dsc->cdev == NULL, ("%s: devmem not free", __func__));
978 SLIST_REMOVE_HEAD(&sc->devmem, link);
979 free(dsc->name, M_VMMDEV);
980 free(dsc, M_VMMDEV);
981 }
982
983 if (sc->cdev != NULL)
984 destroy_dev(sc->cdev);
985
986 if (sc->vm != NULL)
987 vm_destroy(sc->vm);
988
989 if (sc->ucred != NULL)
990 crfree(sc->ucred);
991
992 if ((sc->flags & VSC_LINKED) != 0) {
993 mtx_lock(&vmmdev_mtx);
994 SLIST_REMOVE(&head, sc, vmmdev_softc, link);
995 mtx_unlock(&vmmdev_mtx);
996 }
997
998 free(sc, M_VMMDEV);
999 }
1000
1001 static int
sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS)1002 sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS)
1003 {
1004 struct devmem_softc *dsc;
1005 struct vmmdev_softc *sc;
1006 struct cdev *cdev;
1007 char *buf;
1008 int error, buflen;
1009
1010 error = vmm_priv_check(req->td->td_ucred);
1011 if (error)
1012 return (error);
1013
1014 buflen = VM_MAX_NAMELEN + 1;
1015 buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
1016 strlcpy(buf, "beavis", buflen);
1017 error = sysctl_handle_string(oidp, buf, buflen, req);
1018 if (error != 0 || req->newptr == NULL)
1019 goto out;
1020
1021 mtx_lock(&vmmdev_mtx);
1022 sc = vmmdev_lookup(buf);
1023 if (sc == NULL || sc->cdev == NULL) {
1024 mtx_unlock(&vmmdev_mtx);
1025 error = EINVAL;
1026 goto out;
1027 }
1028
1029 /*
1030 * The 'cdev' will be destroyed asynchronously when 'si_threadcount'
1031 * goes down to 0 so we should not do it again in the callback.
1032 *
1033 * Setting 'sc->cdev' to NULL is also used to indicate that the VM
1034 * is scheduled for destruction.
1035 */
1036 cdev = sc->cdev;
1037 sc->cdev = NULL;
1038 mtx_unlock(&vmmdev_mtx);
1039
1040 /*
1041 * Schedule all cdevs to be destroyed:
1042 *
1043 * - any new operations on the 'cdev' will return an error (ENXIO).
1044 *
1045 * - when the 'si_threadcount' dwindles down to zero the 'cdev' will
1046 * be destroyed and the callback will be invoked in a taskqueue
1047 * context.
1048 *
1049 * - the 'devmem' cdevs are destroyed before the virtual machine 'cdev'
1050 */
1051 SLIST_FOREACH(dsc, &sc->devmem, link) {
1052 KASSERT(dsc->cdev != NULL, ("devmem cdev already destroyed"));
1053 destroy_dev_sched_cb(dsc->cdev, devmem_destroy, dsc);
1054 }
1055 destroy_dev_sched_cb(cdev, vmmdev_destroy, sc);
1056 error = 0;
1057
1058 out:
1059 free(buf, M_VMMDEV);
1060 return (error);
1061 }
1062 SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy,
1063 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
1064 NULL, 0, sysctl_vmm_destroy, "A",
1065 NULL);
1066
1067 static struct cdevsw vmmdevsw = {
1068 .d_name = "vmmdev",
1069 .d_version = D_VERSION,
1070 .d_ioctl = vmmdev_ioctl,
1071 .d_mmap_single = vmmdev_mmap_single,
1072 .d_read = vmmdev_rw,
1073 .d_write = vmmdev_rw,
1074 };
1075
1076 static int
sysctl_vmm_create(SYSCTL_HANDLER_ARGS)1077 sysctl_vmm_create(SYSCTL_HANDLER_ARGS)
1078 {
1079 struct vm *vm;
1080 struct cdev *cdev;
1081 struct vmmdev_softc *sc, *sc2;
1082 char *buf;
1083 int error, buflen;
1084
1085 error = vmm_priv_check(req->td->td_ucred);
1086 if (error)
1087 return (error);
1088
1089 buflen = VM_MAX_NAMELEN + 1;
1090 buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO);
1091 strlcpy(buf, "beavis", buflen);
1092 error = sysctl_handle_string(oidp, buf, buflen, req);
1093 if (error != 0 || req->newptr == NULL)
1094 goto out;
1095
1096 mtx_lock(&vmmdev_mtx);
1097 sc = vmmdev_lookup(buf);
1098 mtx_unlock(&vmmdev_mtx);
1099 if (sc != NULL) {
1100 error = EEXIST;
1101 goto out;
1102 }
1103
1104 error = vm_create(buf, &vm);
1105 if (error != 0)
1106 goto out;
1107
1108 sc = malloc(sizeof(struct vmmdev_softc), M_VMMDEV, M_WAITOK | M_ZERO);
1109 sc->ucred = crhold(curthread->td_ucred);
1110 sc->vm = vm;
1111 SLIST_INIT(&sc->devmem);
1112
1113 /*
1114 * Lookup the name again just in case somebody sneaked in when we
1115 * dropped the lock.
1116 */
1117 mtx_lock(&vmmdev_mtx);
1118 sc2 = vmmdev_lookup(buf);
1119 if (sc2 == NULL) {
1120 SLIST_INSERT_HEAD(&head, sc, link);
1121 sc->flags |= VSC_LINKED;
1122 }
1123 mtx_unlock(&vmmdev_mtx);
1124
1125 if (sc2 != NULL) {
1126 vmmdev_destroy(sc);
1127 error = EEXIST;
1128 goto out;
1129 }
1130
1131 error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, sc->ucred,
1132 UID_ROOT, GID_WHEEL, 0600, "vmm/%s", buf);
1133 if (error != 0) {
1134 vmmdev_destroy(sc);
1135 goto out;
1136 }
1137
1138 mtx_lock(&vmmdev_mtx);
1139 sc->cdev = cdev;
1140 sc->cdev->si_drv1 = sc;
1141 mtx_unlock(&vmmdev_mtx);
1142
1143 out:
1144 free(buf, M_VMMDEV);
1145 return (error);
1146 }
1147 SYSCTL_PROC(_hw_vmm, OID_AUTO, create,
1148 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE,
1149 NULL, 0, sysctl_vmm_create, "A",
1150 NULL);
1151
1152 void
vmmdev_init(void)1153 vmmdev_init(void)
1154 {
1155 mtx_init(&vmmdev_mtx, "vmm device mutex", NULL, MTX_DEF);
1156 pr_allow_flag = prison_add_allow(NULL, "vmm", NULL,
1157 "Allow use of vmm in a jail.");
1158 }
1159
1160 int
vmmdev_cleanup(void)1161 vmmdev_cleanup(void)
1162 {
1163 int error;
1164
1165 if (SLIST_EMPTY(&head))
1166 error = 0;
1167 else
1168 error = EBUSY;
1169
1170 return (error);
1171 }
1172
1173 static int
devmem_mmap_single(struct cdev * cdev,vm_ooffset_t * offset,vm_size_t len,struct vm_object ** objp,int nprot)1174 devmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t len,
1175 struct vm_object **objp, int nprot)
1176 {
1177 struct devmem_softc *dsc;
1178 vm_ooffset_t first, last;
1179 size_t seglen;
1180 int error;
1181 uint16_t lastcpu;
1182 bool sysmem;
1183
1184 dsc = cdev->si_drv1;
1185 if (dsc == NULL) {
1186 /* 'cdev' has been created but is not ready for use */
1187 return (ENXIO);
1188 }
1189
1190 first = *offset;
1191 last = *offset + len;
1192 if ((nprot & PROT_EXEC) || first < 0 || first >= last)
1193 return (EINVAL);
1194
1195 lastcpu = vm_get_maxcpus(dsc->sc->vm) - 1;
1196 error = vcpu_lock_one(dsc->sc, lastcpu);
1197 if (error)
1198 return (error);
1199
1200 error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp);
1201 KASSERT(error == 0 && !sysmem && *objp != NULL,
1202 ("%s: invalid devmem segment %d", __func__, dsc->segid));
1203
1204 vcpu_unlock_one(dsc->sc, lastcpu);
1205
1206 if (seglen >= last) {
1207 vm_object_reference(*objp);
1208 return (0);
1209 } else {
1210 return (EINVAL);
1211 }
1212 }
1213
1214 static struct cdevsw devmemsw = {
1215 .d_name = "devmem",
1216 .d_version = D_VERSION,
1217 .d_mmap_single = devmem_mmap_single,
1218 };
1219
1220 static int
devmem_create_cdev(const char * vmname,int segid,char * devname)1221 devmem_create_cdev(const char *vmname, int segid, char *devname)
1222 {
1223 struct devmem_softc *dsc;
1224 struct vmmdev_softc *sc;
1225 struct cdev *cdev;
1226 int error;
1227
1228 error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL,
1229 UID_ROOT, GID_WHEEL, 0600, "vmm.io/%s.%s", vmname, devname);
1230 if (error)
1231 return (error);
1232
1233 dsc = malloc(sizeof(struct devmem_softc), M_VMMDEV, M_WAITOK | M_ZERO);
1234
1235 mtx_lock(&vmmdev_mtx);
1236 sc = vmmdev_lookup(vmname);
1237 KASSERT(sc != NULL, ("%s: vm %s softc not found", __func__, vmname));
1238 if (sc->cdev == NULL) {
1239 /* virtual machine is being created or destroyed */
1240 mtx_unlock(&vmmdev_mtx);
1241 free(dsc, M_VMMDEV);
1242 destroy_dev_sched_cb(cdev, NULL, 0);
1243 return (ENODEV);
1244 }
1245
1246 dsc->segid = segid;
1247 dsc->name = devname;
1248 dsc->cdev = cdev;
1249 dsc->sc = sc;
1250 SLIST_INSERT_HEAD(&sc->devmem, dsc, link);
1251 mtx_unlock(&vmmdev_mtx);
1252
1253 /* The 'cdev' is ready for use after 'si_drv1' is initialized */
1254 cdev->si_drv1 = dsc;
1255 return (0);
1256 }
1257
1258 static void
devmem_destroy(void * arg)1259 devmem_destroy(void *arg)
1260 {
1261 struct devmem_softc *dsc = arg;
1262
1263 KASSERT(dsc->cdev, ("%s: devmem cdev already destroyed", __func__));
1264 dsc->cdev = NULL;
1265 dsc->sc = NULL;
1266 }
1267