1 /*-
2 * Copyright (c) 2018 VMware, Inc.
3 *
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5 */
6
7 /* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */
8
9 #include <sys/cdefs.h>
10 __FBSDID("$FreeBSD$");
11
12 #include <sys/param.h>
13 #include <sys/bus.h>
14 #include <sys/kernel.h>
15 #include <sys/malloc.h>
16 #include <sys/module.h>
17 #include <sys/rman.h>
18 #include <sys/systm.h>
19
20 #include <dev/pci/pcireg.h>
21 #include <dev/pci/pcivar.h>
22
23 #include <machine/bus.h>
24
25 #include "vmci.h"
26 #include "vmci_doorbell.h"
27 #include "vmci_driver.h"
28 #include "vmci_kernel_defs.h"
29 #include "vmci_queue_pair.h"
30
31 static int vmci_probe(device_t);
32 static int vmci_attach(device_t);
33 static int vmci_detach(device_t);
34 static int vmci_shutdown(device_t);
35
36 static int vmci_map_bars(struct vmci_softc *);
37 static void vmci_unmap_bars(struct vmci_softc *);
38
39 static int vmci_config_capabilities(struct vmci_softc *);
40
41 static int vmci_dma_malloc_int(struct vmci_softc *, bus_size_t,
42 bus_size_t, struct vmci_dma_alloc *);
43 static void vmci_dma_free_int(struct vmci_softc *,
44 struct vmci_dma_alloc *);
45
46 static int vmci_config_interrupts(struct vmci_softc *);
47 static int vmci_config_interrupt(struct vmci_softc *);
48 static int vmci_check_intr_cnt(struct vmci_softc *);
49 static int vmci_allocate_interrupt_resources(struct vmci_softc *);
50 static int vmci_setup_interrupts(struct vmci_softc *);
51 static void vmci_dismantle_interrupts(struct vmci_softc *);
52 static void vmci_interrupt(void *);
53 static void vmci_interrupt_bm(void *);
54 static void dispatch_datagrams(void *, int);
55 static void process_bitmap(void *, int);
56
57 static void vmci_delayed_work_fn_cb(void *context, int data);
58
59 static device_method_t vmci_methods[] = {
60 /* Device interface. */
61 DEVMETHOD(device_probe, vmci_probe),
62 DEVMETHOD(device_attach, vmci_attach),
63 DEVMETHOD(device_detach, vmci_detach),
64 DEVMETHOD(device_shutdown, vmci_shutdown),
65
66 DEVMETHOD_END
67 };
68
69 static driver_t vmci_driver = {
70 "vmci", vmci_methods, sizeof(struct vmci_softc)
71 };
72
73 static devclass_t vmci_devclass;
74 DRIVER_MODULE(vmci, pci, vmci_driver, vmci_devclass, 0, 0);
75 MODULE_VERSION(vmci, VMCI_VERSION);
76 const struct {
77 uint16_t vendor;
78 uint16_t device;
79 const char *desc;
80 } vmci_ids[] = {
81 { VMCI_VMWARE_VENDOR_ID, VMCI_VMWARE_DEVICE_ID,
82 "VMware Virtual Machine Communication Interface" },
83 };
84 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmci, vmci_ids,
85 nitems(vmci_ids));
86
87 MODULE_DEPEND(vmci, pci, 1, 1, 1);
88
89 static struct vmci_softc *vmci_sc;
90
91 #define LGPFX "vmci: "
92 /*
93 * Allocate a buffer for incoming datagrams globally to avoid repeated
94 * allocation in the interrupt handler's atomic context.
95 */
96 static uint8_t *data_buffer = NULL;
97 static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE;
98
99 struct vmci_delayed_work_info {
100 vmci_work_fn *work_fn;
101 void *data;
102 vmci_list_item(vmci_delayed_work_info) entry;
103 };
104
105 /*
106 *------------------------------------------------------------------------------
107 *
108 * vmci_probe --
109 *
110 * Probe to see if the VMCI device is present.
111 *
112 * Results:
113 * BUS_PROBE_DEFAULT if device exists, ENXIO otherwise.
114 *
115 * Side effects:
116 * None.
117 *
118 *------------------------------------------------------------------------------
119 */
120
121 static int
vmci_probe(device_t dev)122 vmci_probe(device_t dev)
123 {
124
125 if (pci_get_vendor(dev) == vmci_ids[0].vendor &&
126 pci_get_device(dev) == vmci_ids[0].device) {
127 device_set_desc(dev, vmci_ids[0].desc);
128
129 return (BUS_PROBE_DEFAULT);
130 }
131
132 return (ENXIO);
133 }
134
135 /*
136 *------------------------------------------------------------------------------
137 *
138 * vmci_attach --
139 *
140 * Attach VMCI device to the system after vmci_probe() has been called and
141 * the device has been detected.
142 *
143 * Results:
144 * 0 if success, ENXIO otherwise.
145 *
146 * Side effects:
147 * None.
148 *
149 *------------------------------------------------------------------------------
150 */
151
152 static int
vmci_attach(device_t dev)153 vmci_attach(device_t dev)
154 {
155 struct vmci_softc *sc;
156 int error, i;
157
158 sc = device_get_softc(dev);
159 sc->vmci_dev = dev;
160 vmci_sc = sc;
161
162 data_buffer = NULL;
163 sc->vmci_num_intr = 0;
164 for (i = 0; i < VMCI_MAX_INTRS; i++) {
165 sc->vmci_intrs[i].vmci_irq = NULL;
166 sc->vmci_intrs[i].vmci_handler = NULL;
167 }
168
169 TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc);
170 TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc);
171
172 TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc);
173
174 pci_enable_busmaster(dev);
175
176 mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN);
177 mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock",
178 NULL, MTX_DEF);
179
180 error = vmci_map_bars(sc);
181 if (error) {
182 VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n");
183 goto fail;
184 }
185
186 error = vmci_config_capabilities(sc);
187 if (error) {
188 VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n");
189 goto fail;
190 }
191
192 vmci_list_init(&sc->vmci_delayed_work_infos);
193
194 vmci_components_init();
195 vmci_util_init();
196 error = vmci_qp_guest_endpoints_init();
197 if (error) {
198 VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n");
199 goto fail;
200 }
201
202 error = vmci_config_interrupts(sc);
203 if (error)
204 VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n");
205
206 fail:
207 if (error) {
208 vmci_detach(dev);
209 return (ENXIO);
210 }
211
212 return (0);
213 }
214
215 /*
216 *------------------------------------------------------------------------------
217 *
218 * vmci_detach --
219 *
220 * Detach the VMCI device.
221 *
222 * Results:
223 * 0
224 *
225 * Side effects:
226 * None.
227 *
228 *------------------------------------------------------------------------------
229 */
230
231 static int
vmci_detach(device_t dev)232 vmci_detach(device_t dev)
233 {
234 struct vmci_softc *sc;
235
236 sc = device_get_softc(dev);
237
238 vmci_qp_guest_endpoints_exit();
239 vmci_util_exit();
240
241 vmci_dismantle_interrupts(sc);
242
243 vmci_components_cleanup();
244
245 if mtx_initialized(&sc->vmci_spinlock) {
246 taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task);
247 mtx_destroy(&sc->vmci_delayed_work_lock);
248 }
249
250 if (sc->vmci_res0 != NULL)
251 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
252 VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET);
253
254 if (sc->vmci_notifications_bitmap.dma_vaddr != NULL)
255 vmci_dma_free(&sc->vmci_notifications_bitmap);
256
257 vmci_unmap_bars(sc);
258
259 if mtx_initialized(&sc->vmci_spinlock)
260 mtx_destroy(&sc->vmci_spinlock);
261
262 pci_disable_busmaster(dev);
263
264 return (0);
265 }
266
267 /*
268 *------------------------------------------------------------------------------
269 *
270 * vmci_shutdown --
271 *
272 * This function is called during system shutdown. We don't do anything.
273 *
274 * Results:
275 * 0
276 *
277 * Side effects:
278 * None.
279 *
280 *------------------------------------------------------------------------------
281 */
282
283 static int
vmci_shutdown(device_t dev)284 vmci_shutdown(device_t dev)
285 {
286
287 return (0);
288 }
289
290 /*
291 *------------------------------------------------------------------------------
292 *
293 * vmci_map_bars --
294 *
295 * Maps the PCI I/O and MMIO BARs.
296 *
297 * Results:
298 * 0 on success, ENXIO otherwise.
299 *
300 * Side effects:
301 * None.
302 *
303 *------------------------------------------------------------------------------
304 */
305
306 static int
vmci_map_bars(struct vmci_softc * sc)307 vmci_map_bars(struct vmci_softc *sc)
308 {
309 int rid;
310
311 /* Map the PCI I/O BAR: BAR0 */
312 rid = PCIR_BAR(0);
313 sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT,
314 &rid, RF_ACTIVE);
315 if (sc->vmci_res0 == NULL) {
316 VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n");
317 return (ENXIO);
318 }
319
320 sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0);
321 sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0);
322 sc->vmci_ioaddr = rman_get_start(sc->vmci_res0);
323
324 /* Map the PCI MMIO BAR: BAR1 */
325 rid = PCIR_BAR(1);
326 sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY,
327 &rid, RF_ACTIVE);
328 if (sc->vmci_res1 == NULL) {
329 VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n");
330 return (ENXIO);
331 }
332
333 sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1);
334 sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1);
335
336 return (0);
337 }
338
339 /*
340 *------------------------------------------------------------------------------
341 *
342 * vmci_unmap_bars --
343 *
344 * Unmaps the VMCI PCI I/O and MMIO BARs.
345 *
346 * Results:
347 * None.
348 *
349 * Side effects:
350 * None.
351 *
352 *------------------------------------------------------------------------------
353 */
354
355 static void
vmci_unmap_bars(struct vmci_softc * sc)356 vmci_unmap_bars(struct vmci_softc *sc)
357 {
358 int rid;
359
360 if (sc->vmci_res0 != NULL) {
361 rid = PCIR_BAR(0);
362 bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid,
363 sc->vmci_res0);
364 sc->vmci_res0 = NULL;
365 }
366
367 if (sc->vmci_res1 != NULL) {
368 rid = PCIR_BAR(1);
369 bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid,
370 sc->vmci_res1);
371 sc->vmci_res1 = NULL;
372 }
373 }
374
375 /*
376 *------------------------------------------------------------------------------
377 *
378 * vmci_config_capabilities --
379 *
380 * Check the VMCI device capabilities and configure the device accordingly.
381 *
382 * Results:
383 * 0 if success, ENODEV otherwise.
384 *
385 * Side effects:
386 * Device capabilities are enabled.
387 *
388 *------------------------------------------------------------------------------
389 */
390
391 static int
vmci_config_capabilities(struct vmci_softc * sc)392 vmci_config_capabilities(struct vmci_softc *sc)
393 {
394 unsigned long bitmap_PPN;
395 int error;
396
397 /*
398 * Verify that the VMCI device supports the capabilities that we
399 * need. Datagrams are necessary and notifications will be used
400 * if the device supports it.
401 */
402 sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0,
403 VMCI_CAPS_ADDR);
404
405 if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) {
406 VMCI_LOG_ERROR(LGPFX"VMCI device does not support "
407 "datagrams.\n");
408 return (ENODEV);
409 }
410
411 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
412 sc->capabilities = VMCI_CAPS_DATAGRAM;
413 error = vmci_dma_malloc(PAGE_SIZE, 1,
414 &sc->vmci_notifications_bitmap);
415 if (error)
416 VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for "
417 "notification bitmap.\n");
418 else {
419 memset(sc->vmci_notifications_bitmap.dma_vaddr, 0,
420 PAGE_SIZE);
421 sc->capabilities |= VMCI_CAPS_NOTIFICATIONS;
422 }
423 } else
424 sc->capabilities = VMCI_CAPS_DATAGRAM;
425
426 /* Let the host know which capabilities we intend to use. */
427 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
428 VMCI_CAPS_ADDR, sc->capabilities);
429
430 /*
431 * Register notification bitmap with device if that capability is
432 * used.
433 */
434 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
435 bitmap_PPN =
436 sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT;
437 vmci_register_notification_bitmap(bitmap_PPN);
438 }
439
440 /* Check host capabilities. */
441 if (!vmci_check_host_capabilities())
442 return (ENODEV);
443
444 return (0);
445 }
446
447 /*
448 *------------------------------------------------------------------------------
449 *
450 * vmci_dmamap_cb --
451 *
452 * Callback to receive mapping information resulting from the load of a
453 * bus_dmamap_t via bus_dmamap_load()
454 *
455 * Results:
456 * None.
457 *
458 * Side effects:
459 * None.
460 *
461 *------------------------------------------------------------------------------
462 */
463
464 static void
vmci_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)465 vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
466 {
467 bus_addr_t *baddr = arg;
468
469 if (error == 0)
470 *baddr = segs->ds_addr;
471 }
472
473 /*
474 *------------------------------------------------------------------------------
475 *
476 * vmci_dma_malloc_int --
477 *
478 * Internal function that allocates DMA memory.
479 *
480 * Results:
481 * 0 if success.
482 * ENOMEM if insufficient memory.
483 * EINPROGRESS if mapping is deferred.
484 * EINVAL if the request was invalid.
485 *
486 * Side effects:
487 * DMA memory is allocated.
488 *
489 *------------------------------------------------------------------------------
490 */
491
492 static int
vmci_dma_malloc_int(struct vmci_softc * sc,bus_size_t size,bus_size_t align,struct vmci_dma_alloc * dma)493 vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align,
494 struct vmci_dma_alloc *dma)
495 {
496 int error;
497
498 bzero(dma, sizeof(struct vmci_dma_alloc));
499
500 error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev),
501 align, 0, /* alignment, bounds */
502 BUS_SPACE_MAXADDR, /* lowaddr */
503 BUS_SPACE_MAXADDR, /* highaddr */
504 NULL, NULL, /* filter, filterarg */
505 size, /* maxsize */
506 1, /* nsegments */
507 size, /* maxsegsize */
508 BUS_DMA_ALLOCNOW, /* flags */
509 NULL, /* lockfunc */
510 NULL, /* lockfuncarg */
511 &dma->dma_tag);
512 if (error) {
513 VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error);
514 goto fail;
515 }
516
517 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
518 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
519 if (error) {
520 VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error);
521 goto fail;
522 }
523
524 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
525 size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
526 if (error) {
527 VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error);
528 goto fail;
529 }
530
531 dma->dma_size = size;
532
533 fail:
534 if (error)
535 vmci_dma_free(dma);
536
537 return (error);
538 }
539
540 /*
541 *------------------------------------------------------------------------------
542 *
543 * vmci_dma_malloc --
544 *
545 * This function is a wrapper around vmci_dma_malloc_int for callers
546 * outside of this module. Since we only support a single VMCI device, this
547 * wrapper provides access to the device softc structure.
548 *
549 * Results:
550 * 0 if success.
551 * ENOMEM if insufficient memory.
552 * EINPROGRESS if mapping is deferred.
553 * EINVAL if the request was invalid.
554 *
555 * Side effects:
556 * DMA memory is allocated.
557 *
558 *------------------------------------------------------------------------------
559 */
560
561 int
vmci_dma_malloc(bus_size_t size,bus_size_t align,struct vmci_dma_alloc * dma)562 vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma)
563 {
564
565 return (vmci_dma_malloc_int(vmci_sc, size, align, dma));
566 }
567
568 /*
569 *------------------------------------------------------------------------------
570 *
571 * vmci_dma_free_int --
572 *
573 * Internal function that frees DMA memory.
574 *
575 * Results:
576 * None.
577 *
578 * Side effects:
579 * Frees DMA memory.
580 *
581 *------------------------------------------------------------------------------
582 */
583
584 static void
vmci_dma_free_int(struct vmci_softc * sc,struct vmci_dma_alloc * dma)585 vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma)
586 {
587
588 if (dma->dma_tag != NULL) {
589 if (dma->dma_paddr != 0) {
590 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
591 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
592 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
593 }
594
595 if (dma->dma_vaddr != NULL)
596 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
597 dma->dma_map);
598
599 bus_dma_tag_destroy(dma->dma_tag);
600 }
601 bzero(dma, sizeof(struct vmci_dma_alloc));
602 }
603
604 /*
605 *------------------------------------------------------------------------------
606 *
607 * vmci_dma_free --
608 *
609 * This function is a wrapper around vmci_dma_free_int for callers outside
610 * of this module. Since we only support a single VMCI device, this wrapper
611 * provides access to the device softc structure.
612 *
613 * Results:
614 * None.
615 *
616 * Side effects:
617 * Frees DMA memory.
618 *
619 *------------------------------------------------------------------------------
620 */
621
622 void
vmci_dma_free(struct vmci_dma_alloc * dma)623 vmci_dma_free(struct vmci_dma_alloc *dma)
624 {
625
626 vmci_dma_free_int(vmci_sc, dma);
627 }
628
629 /*
630 *------------------------------------------------------------------------------
631 *
632 * vmci_config_interrupts --
633 *
634 * Configures and enables interrupts. Try to configure MSI-X. If this fails,
635 * try to configure MSI. If even this fails, try legacy interrupts.
636 *
637 * Results:
638 * 0 if success.
639 * ENOMEM if insufficient memory.
640 * ENODEV if the device doesn't support interrupts.
641 * ENXIO if the device configuration failed.
642 *
643 * Side effects:
644 * Interrupts get enabled if successful.
645 *
646 *------------------------------------------------------------------------------
647 */
648
649 static int
vmci_config_interrupts(struct vmci_softc * sc)650 vmci_config_interrupts(struct vmci_softc *sc)
651 {
652 int error;
653
654 data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT);
655 if (data_buffer == NULL)
656 return (ENOMEM);
657
658 sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX;
659 error = vmci_config_interrupt(sc);
660 if (error) {
661 sc->vmci_intr_type = VMCI_INTR_TYPE_MSI;
662 error = vmci_config_interrupt(sc);
663 }
664 if (error) {
665 sc->vmci_intr_type = VMCI_INTR_TYPE_INTX;
666 error = vmci_config_interrupt(sc);
667 }
668 if (error)
669 return (error);
670
671 /* Enable specific interrupt bits. */
672 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS)
673 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
674 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION);
675 else
676 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
677 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM);
678
679 /* Enable interrupts. */
680 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
681 VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE);
682
683 return (0);
684 }
685
686 /*
687 *------------------------------------------------------------------------------
688 *
689 * vmci_config_interrupt --
690 *
691 * Check the number of interrupts supported, allocate resources and setup
692 * interrupts.
693 *
694 * Results:
695 * 0 if success.
696 * ENOMEM if insufficient memory.
697 * ENODEV if the device doesn't support interrupts.
698 * ENXIO if the device configuration failed.
699 *
700 * Side effects:
701 * Resources get allocated and interrupts get setup (but not enabled) if
702 * successful.
703 *
704 *------------------------------------------------------------------------------
705 */
706
707 static int
vmci_config_interrupt(struct vmci_softc * sc)708 vmci_config_interrupt(struct vmci_softc *sc)
709 {
710 int error;
711
712 error = vmci_check_intr_cnt(sc);
713 if (error)
714 return (error);
715
716 error = vmci_allocate_interrupt_resources(sc);
717 if (error)
718 return (error);
719
720 error = vmci_setup_interrupts(sc);
721 if (error)
722 return (error);
723
724 return (0);
725 }
726
727 /*
728 *------------------------------------------------------------------------------
729 *
730 * vmci_check_intr_cnt --
731 *
732 * Check the number of interrupts supported by the device and ask PCI bus
733 * to allocate appropriate number of interrupts.
734 *
735 * Results:
736 * 0 if success.
737 * ENODEV if the device doesn't support any interrupts.
738 * ENXIO if the device configuration failed.
739 *
740 * Side effects:
741 * Resources get allocated on success.
742 *
743 *------------------------------------------------------------------------------
744 */
745
746 static int
vmci_check_intr_cnt(struct vmci_softc * sc)747 vmci_check_intr_cnt(struct vmci_softc *sc)
748 {
749
750 if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) {
751 sc->vmci_num_intr = 1;
752 return (0);
753 }
754
755 /*
756 * Make sure that the device supports the required number of MSI/MSI-X
757 * messages. We try for 2 MSI-X messages but 1 is good too. We need at
758 * least 1 MSI message.
759 */
760 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
761 pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev);
762
763 if (!sc->vmci_num_intr) {
764 VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt"
765 " messages");
766 return (ENODEV);
767 }
768
769 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
770 VMCI_MAX_INTRS : 1;
771 if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) {
772 if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr))
773 return (ENXIO);
774 } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) {
775 if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr))
776 return (ENXIO);
777 }
778
779 return (0);
780 }
781
782 /*
783 *------------------------------------------------------------------------------
784 *
785 * vmci_allocate_interrupt_resources --
786 *
787 * Allocate resources necessary for interrupts.
788 *
789 * Results:
790 * 0 if success, ENXIO otherwise.
791 *
792 * Side effects:
793 * Resources get allocated on success.
794 *
795 *------------------------------------------------------------------------------
796 */
797
798 static int
vmci_allocate_interrupt_resources(struct vmci_softc * sc)799 vmci_allocate_interrupt_resources(struct vmci_softc *sc)
800 {
801 struct resource *irq;
802 int flags, i, rid;
803
804 flags = RF_ACTIVE;
805 flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0;
806 rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1;
807
808 for (i = 0; i < sc->vmci_num_intr; i++, rid++) {
809 irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid,
810 flags);
811 if (irq == NULL)
812 return (ENXIO);
813 sc->vmci_intrs[i].vmci_irq = irq;
814 sc->vmci_intrs[i].vmci_rid = rid;
815 }
816
817 return (0);
818 }
819
820 /*
821 *------------------------------------------------------------------------------
822 *
823 * vmci_setup_interrupts --
824 *
825 * Sets up the interrupts.
826 *
827 * Results:
828 * 0 if success, appropriate error code from bus_setup_intr otherwise.
829 *
830 * Side effects:
831 * Interrupt handler gets attached.
832 *
833 *------------------------------------------------------------------------------
834 */
835
836 static int
vmci_setup_interrupts(struct vmci_softc * sc)837 vmci_setup_interrupts(struct vmci_softc *sc)
838 {
839 struct vmci_interrupt *intr;
840 int error, flags;
841
842 flags = INTR_TYPE_NET | INTR_MPSAFE;
843 if (sc->vmci_num_intr > 1)
844 flags |= INTR_EXCL;
845
846 intr = &sc->vmci_intrs[0];
847 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL,
848 vmci_interrupt, NULL, &intr->vmci_handler);
849 if (error)
850 return (error);
851
852 if (sc->vmci_num_intr == 2) {
853 bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
854 intr->vmci_handler, "dg");
855 intr = &sc->vmci_intrs[1];
856 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags,
857 NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler);
858 if (error)
859 return (error);
860 bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
861 intr->vmci_handler, "bm");
862 }
863
864 return (0);
865 }
866
867 /*
868 *------------------------------------------------------------------------------
869 *
870 * vmci_interrupt --
871 *
872 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X
873 * interrupt (vector VMCI_INTR_DATAGRAM).
874 *
875 * Results:
876 * None.
877 *
878 * Side effects:
879 * None.
880 *
881 *------------------------------------------------------------------------------
882 */
883
884 static void
vmci_interrupt(void * arg)885 vmci_interrupt(void *arg)
886 {
887
888 if (vmci_sc->vmci_num_intr == 2)
889 taskqueue_enqueue(taskqueue_swi,
890 &vmci_sc->vmci_interrupt_dq_task);
891 else {
892 unsigned int icr;
893
894 icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR);
895 if (icr == 0 || icr == 0xffffffff)
896 return;
897 if (icr & VMCI_ICR_DATAGRAM) {
898 taskqueue_enqueue(taskqueue_swi,
899 &vmci_sc->vmci_interrupt_dq_task);
900 icr &= ~VMCI_ICR_DATAGRAM;
901 }
902 if (icr & VMCI_ICR_NOTIFICATION) {
903 taskqueue_enqueue(taskqueue_swi,
904 &vmci_sc->vmci_interrupt_bm_task);
905 icr &= ~VMCI_ICR_NOTIFICATION;
906 }
907 if (icr != 0)
908 VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt "
909 "cause");
910 }
911 }
912
913 /*
914 *------------------------------------------------------------------------------
915 *
916 * vmci_interrupt_bm --
917 *
918 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
919 * which is for the notification bitmap. Will only get called if we are
920 * using MSI-X with exclusive vectors.
921 *
922 * Results:
923 * None.
924 *
925 * Side effects:
926 * None.
927 *
928 *------------------------------------------------------------------------------
929 */
930
931 static void
vmci_interrupt_bm(void * arg)932 vmci_interrupt_bm(void *arg)
933 {
934
935 ASSERT(vmci_sc->vmci_num_intr == 2);
936 taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task);
937 }
938
939 /*
940 *------------------------------------------------------------------------------
941 *
942 * dispatch_datagrams --
943 *
944 * Reads and dispatches incoming datagrams.
945 *
946 * Results:
947 * None.
948 *
949 * Side effects:
950 * Reads data from the device.
951 *
952 *------------------------------------------------------------------------------
953 */
954
955 static void
dispatch_datagrams(void * context,int data)956 dispatch_datagrams(void *context, int data)
957 {
958
959 if (data_buffer == NULL)
960 VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer "
961 "present");
962
963 vmci_read_datagrams_from_port((vmci_io_handle) 0,
964 vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR,
965 data_buffer, data_buffer_size);
966 }
967
968 /*
969 *------------------------------------------------------------------------------
970 *
971 * process_bitmap --
972 *
973 * Scans the notification bitmap for raised flags, clears them and handles
974 * the notifications.
975 *
976 * Results:
977 * None.
978 *
979 * Side effects:
980 * None.
981 *
982 *------------------------------------------------------------------------------
983 */
984
985 static void
process_bitmap(void * context,int data)986 process_bitmap(void *context, int data)
987 {
988
989 if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL)
990 VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present");
991
992 vmci_scan_notification_bitmap(
993 vmci_sc->vmci_notifications_bitmap.dma_vaddr);
994 }
995
996 /*
997 *------------------------------------------------------------------------------
998 *
999 * vmci_dismantle_interrupts --
1000 *
1001 * Releases resources, detaches the interrupt handler and drains the task
1002 * queue.
1003 *
1004 * Results:
1005 * None.
1006 *
1007 * Side effects:
1008 * No more interrupts.
1009 *
1010 *------------------------------------------------------------------------------
1011 */
1012
1013 static void
vmci_dismantle_interrupts(struct vmci_softc * sc)1014 vmci_dismantle_interrupts(struct vmci_softc *sc)
1015 {
1016 struct vmci_interrupt *intr;
1017 int i;
1018
1019 for (i = 0; i < sc->vmci_num_intr; i++) {
1020 intr = &sc->vmci_intrs[i];
1021 if (intr->vmci_handler != NULL) {
1022 bus_teardown_intr(sc->vmci_dev, intr->vmci_irq,
1023 intr->vmci_handler);
1024 intr->vmci_handler = NULL;
1025 }
1026 if (intr->vmci_irq != NULL) {
1027 bus_release_resource(sc->vmci_dev, SYS_RES_IRQ,
1028 intr->vmci_rid, intr->vmci_irq);
1029 intr->vmci_irq = NULL;
1030 intr->vmci_rid = -1;
1031 }
1032 }
1033
1034 if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) &&
1035 (sc->vmci_num_intr))
1036 pci_release_msi(sc->vmci_dev);
1037
1038 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task);
1039 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task);
1040
1041 if (data_buffer != NULL)
1042 free(data_buffer, M_DEVBUF);
1043 }
1044
1045 /*
1046 *------------------------------------------------------------------------------
1047 *
1048 * vmci_delayed_work_fn_cb --
1049 *
1050 * Callback function that executes the queued up delayed work functions.
1051 *
1052 * Results:
1053 * None.
1054 *
1055 * Side effects:
1056 * None.
1057 *
1058 *------------------------------------------------------------------------------
1059 */
1060
1061 static void
vmci_delayed_work_fn_cb(void * context,int data)1062 vmci_delayed_work_fn_cb(void *context, int data)
1063 {
1064 vmci_list(vmci_delayed_work_info) temp_list;
1065
1066 vmci_list_init(&temp_list);
1067
1068 /*
1069 * Swap vmci_delayed_work_infos list with the empty temp_list while
1070 * holding a lock. vmci_delayed_work_infos would then be an empty list
1071 * and temp_list would contain the elements from the original
1072 * vmci_delayed_work_infos. Finally, iterate through temp_list
1073 * executing the delayed callbacks.
1074 */
1075
1076 mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1077 vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos,
1078 vmci_delayed_work_info, entry);
1079 mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1080
1081 while (!vmci_list_empty(&temp_list)) {
1082 struct vmci_delayed_work_info *delayed_work_info =
1083 vmci_list_first(&temp_list);
1084
1085 delayed_work_info->work_fn(delayed_work_info->data);
1086
1087 vmci_list_remove(delayed_work_info, entry);
1088 vmci_free_kernel_mem(delayed_work_info,
1089 sizeof(*delayed_work_info));
1090 }
1091 }
1092
1093 /*
1094 *------------------------------------------------------------------------------
1095 *
1096 * vmci_schedule_delayed_work_fn --
1097 *
1098 * Schedule the specified callback.
1099 *
1100 * Results:
1101 * 0 if success, error code otherwise.
1102 *
1103 * Side effects:
1104 * None.
1105 *
1106 *------------------------------------------------------------------------------
1107 */
1108
1109 int
vmci_schedule_delayed_work_fn(vmci_work_fn * work_fn,void * data)1110 vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data)
1111 {
1112 struct vmci_delayed_work_info *delayed_work_info;
1113
1114 delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info),
1115 VMCI_MEMORY_ATOMIC);
1116
1117 if (!delayed_work_info)
1118 return (VMCI_ERROR_NO_MEM);
1119
1120 delayed_work_info->work_fn = work_fn;
1121 delayed_work_info->data = data;
1122 mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1123 vmci_list_insert(&vmci_sc->vmci_delayed_work_infos,
1124 delayed_work_info, entry);
1125 mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1126
1127 taskqueue_enqueue(taskqueue_thread,
1128 &vmci_sc->vmci_delayed_work_task);
1129
1130 return (VMCI_SUCCESS);
1131 }
1132
1133 /*
1134 *------------------------------------------------------------------------------
1135 *
1136 * vmci_send_datagram --
1137 *
1138 * VM to hypervisor call mechanism.
1139 *
1140 * Results:
1141 * The result of the hypercall.
1142 *
1143 * Side effects:
1144 * None.
1145 *
1146 *------------------------------------------------------------------------------
1147 */
1148
1149 int
vmci_send_datagram(struct vmci_datagram * dg)1150 vmci_send_datagram(struct vmci_datagram *dg)
1151 {
1152 int result;
1153
1154 if (dg == NULL)
1155 return (VMCI_ERROR_INVALID_ARGS);
1156
1157 /*
1158 * Need to acquire spinlock on the device because
1159 * the datagram data may be spread over multiple pages and the monitor
1160 * may interleave device user rpc calls from multiple VCPUs. Acquiring
1161 * the spinlock precludes that possibility. Disabling interrupts to
1162 * avoid incoming datagrams during a "rep out" and possibly landing up
1163 * in this function.
1164 */
1165 mtx_lock_spin(&vmci_sc->vmci_spinlock);
1166
1167 /*
1168 * Send the datagram and retrieve the return value from the result
1169 * register.
1170 */
1171 __asm__ __volatile__(
1172 "cld\n\t"
1173 "rep outsb\n\t"
1174 : /* No output. */
1175 : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR),
1176 "c"(VMCI_DG_SIZE(dg)), "S"(dg)
1177 );
1178
1179 /*
1180 * XXX: Should read result high port as well when updating handlers to
1181 * return 64bit.
1182 */
1183
1184 result = bus_space_read_4(vmci_sc->vmci_iot0,
1185 vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR);
1186 mtx_unlock_spin(&vmci_sc->vmci_spinlock);
1187
1188 return (result);
1189 }
1190