1 /*-
2 * Copyright (c) 2018 VMware, Inc.
3 *
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5 */
6
7 /* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */
8
9 #include <sys/cdefs.h>
10 __FBSDID("$FreeBSD$");
11
12 #include <sys/types.h>
13 #include <sys/bus.h>
14 #include <sys/kernel.h>
15 #include <sys/malloc.h>
16 #include <sys/module.h>
17 #include <sys/rman.h>
18 #include <sys/systm.h>
19
20 #include <dev/pci/pcireg.h>
21 #include <dev/pci/pcivar.h>
22
23 #include <machine/bus.h>
24
25 #include "vmci.h"
26 #include "vmci_doorbell.h"
27 #include "vmci_driver.h"
28 #include "vmci_kernel_defs.h"
29 #include "vmci_queue_pair.h"
30
31 static int vmci_probe(device_t);
32 static int vmci_attach(device_t);
33 static int vmci_detach(device_t);
34 static int vmci_shutdown(device_t);
35
36 static int vmci_map_bars(struct vmci_softc *);
37 static void vmci_unmap_bars(struct vmci_softc *);
38
39 static int vmci_config_capabilities(struct vmci_softc *);
40
41 static int vmci_dma_malloc_int(struct vmci_softc *, bus_size_t,
42 bus_size_t, struct vmci_dma_alloc *);
43 static void vmci_dma_free_int(struct vmci_softc *,
44 struct vmci_dma_alloc *);
45
46 static int vmci_config_interrupts(struct vmci_softc *);
47 static int vmci_config_interrupt(struct vmci_softc *);
48 static int vmci_check_intr_cnt(struct vmci_softc *);
49 static int vmci_allocate_interrupt_resources(struct vmci_softc *);
50 static int vmci_setup_interrupts(struct vmci_softc *);
51 static void vmci_dismantle_interrupts(struct vmci_softc *);
52 static void vmci_interrupt(void *);
53 static void vmci_interrupt_bm(void *);
54 static void dispatch_datagrams(void *, int);
55 static void process_bitmap(void *, int);
56
57 static void vmci_delayed_work_fn_cb(void *context, int data);
58
59 static device_method_t vmci_methods[] = {
60 /* Device interface. */
61 DEVMETHOD(device_probe, vmci_probe),
62 DEVMETHOD(device_attach, vmci_attach),
63 DEVMETHOD(device_detach, vmci_detach),
64 DEVMETHOD(device_shutdown, vmci_shutdown),
65
66 DEVMETHOD_END
67 };
68
69 static driver_t vmci_driver = {
70 "vmci", vmci_methods, sizeof(struct vmci_softc)
71 };
72
73 static devclass_t vmci_devclass;
74 DRIVER_MODULE(vmci, pci, vmci_driver, vmci_devclass, 0, 0);
75 MODULE_VERSION(vmci, VMCI_VERSION);
76
77 MODULE_DEPEND(vmci, pci, 1, 1, 1);
78
79 static struct vmci_softc *vmci_sc;
80
81 #define LGPFX "vmci: "
82 /*
83 * Allocate a buffer for incoming datagrams globally to avoid repeated
84 * allocation in the interrupt handler's atomic context.
85 */
86 static uint8_t *data_buffer = NULL;
87 static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE;
88
89 struct vmci_delayed_work_info {
90 vmci_work_fn *work_fn;
91 void *data;
92 vmci_list_item(vmci_delayed_work_info) entry;
93 };
94
95 /*
96 *------------------------------------------------------------------------------
97 *
98 * vmci_probe --
99 *
100 * Probe to see if the VMCI device is present.
101 *
102 * Results:
103 * BUS_PROBE_DEFAULT if device exists, ENXIO otherwise.
104 *
105 * Side effects:
106 * None.
107 *
108 *------------------------------------------------------------------------------
109 */
110
111 static int
vmci_probe(device_t dev)112 vmci_probe(device_t dev)
113 {
114
115 if (pci_get_vendor(dev) == VMCI_VMWARE_VENDOR_ID &&
116 pci_get_device(dev) == VMCI_VMWARE_DEVICE_ID) {
117 device_set_desc(dev,
118 "VMware Virtual Machine Communication Interface");
119
120 return (BUS_PROBE_DEFAULT);
121 }
122
123 return (ENXIO);
124 }
125
126 /*
127 *------------------------------------------------------------------------------
128 *
129 * vmci_attach --
130 *
131 * Attach VMCI device to the system after vmci_probe() has been called and
132 * the device has been detected.
133 *
134 * Results:
135 * 0 if success, ENXIO otherwise.
136 *
137 * Side effects:
138 * None.
139 *
140 *------------------------------------------------------------------------------
141 */
142
143 static int
vmci_attach(device_t dev)144 vmci_attach(device_t dev)
145 {
146 struct vmci_softc *sc;
147 int error, i;
148
149 sc = device_get_softc(dev);
150 sc->vmci_dev = dev;
151 vmci_sc = sc;
152
153 data_buffer = NULL;
154 sc->vmci_num_intr = 0;
155 for (i = 0; i < VMCI_MAX_INTRS; i++) {
156 sc->vmci_intrs[i].vmci_irq = NULL;
157 sc->vmci_intrs[i].vmci_handler = NULL;
158 }
159
160 TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc);
161 TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc);
162
163 TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc);
164
165 pci_enable_busmaster(dev);
166
167 mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN);
168 mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock",
169 NULL, MTX_DEF);
170
171 error = vmci_map_bars(sc);
172 if (error) {
173 VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n");
174 goto fail;
175 }
176
177 error = vmci_config_capabilities(sc);
178 if (error) {
179 VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n");
180 goto fail;
181 }
182
183 vmci_list_init(&sc->vmci_delayed_work_infos);
184
185 vmci_components_init();
186 vmci_util_init();
187 error = vmci_qp_guest_endpoints_init();
188 if (error) {
189 VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n");
190 goto fail;
191 }
192
193 error = vmci_config_interrupts(sc);
194 if (error)
195 VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n");
196
197 fail:
198 if (error) {
199 vmci_detach(dev);
200 return (ENXIO);
201 }
202
203 return (0);
204 }
205
206 /*
207 *------------------------------------------------------------------------------
208 *
209 * vmci_detach --
210 *
211 * Detach the VMCI device.
212 *
213 * Results:
214 * 0
215 *
216 * Side effects:
217 * None.
218 *
219 *------------------------------------------------------------------------------
220 */
221
222 static int
vmci_detach(device_t dev)223 vmci_detach(device_t dev)
224 {
225 struct vmci_softc *sc;
226
227 sc = device_get_softc(dev);
228
229 vmci_qp_guest_endpoints_exit();
230 vmci_util_exit();
231
232 vmci_dismantle_interrupts(sc);
233
234 vmci_components_cleanup();
235
236 taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task);
237 mtx_destroy(&sc->vmci_delayed_work_lock);
238
239 if (sc->vmci_res0 != NULL)
240 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
241 VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET);
242
243 if (sc->vmci_notifications_bitmap.dma_vaddr != NULL)
244 vmci_dma_free(&sc->vmci_notifications_bitmap);
245
246 vmci_unmap_bars(sc);
247
248 mtx_destroy(&sc->vmci_spinlock);
249
250 pci_disable_busmaster(dev);
251
252 return (0);
253 }
254
255 /*
256 *------------------------------------------------------------------------------
257 *
258 * vmci_shutdown --
259 *
260 * This function is called during system shutdown. We don't do anything.
261 *
262 * Results:
263 * 0
264 *
265 * Side effects:
266 * None.
267 *
268 *------------------------------------------------------------------------------
269 */
270
271 static int
vmci_shutdown(device_t dev)272 vmci_shutdown(device_t dev)
273 {
274
275 return (0);
276 }
277
278 /*
279 *------------------------------------------------------------------------------
280 *
281 * vmci_map_bars --
282 *
283 * Maps the PCI I/O and MMIO BARs.
284 *
285 * Results:
286 * 0 on success, ENXIO otherwise.
287 *
288 * Side effects:
289 * None.
290 *
291 *------------------------------------------------------------------------------
292 */
293
294 static int
vmci_map_bars(struct vmci_softc * sc)295 vmci_map_bars(struct vmci_softc *sc)
296 {
297 int rid;
298
299 /* Map the PCI I/O BAR: BAR0 */
300 rid = PCIR_BAR(0);
301 sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT,
302 &rid, RF_ACTIVE);
303 if (sc->vmci_res0 == NULL) {
304 VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n");
305 return (ENXIO);
306 }
307
308 sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0);
309 sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0);
310 sc->vmci_ioaddr = rman_get_start(sc->vmci_res0);
311
312 /* Map the PCI MMIO BAR: BAR1 */
313 rid = PCIR_BAR(1);
314 sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY,
315 &rid, RF_ACTIVE);
316 if (sc->vmci_res1 == NULL) {
317 VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n");
318 return (ENXIO);
319 }
320
321 sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1);
322 sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1);
323
324 return (0);
325 }
326
327 /*
328 *------------------------------------------------------------------------------
329 *
330 * vmci_unmap_bars --
331 *
332 * Unmaps the VMCI PCI I/O and MMIO BARs.
333 *
334 * Results:
335 * None.
336 *
337 * Side effects:
338 * None.
339 *
340 *------------------------------------------------------------------------------
341 */
342
343 static void
vmci_unmap_bars(struct vmci_softc * sc)344 vmci_unmap_bars(struct vmci_softc *sc)
345 {
346 int rid;
347
348 if (sc->vmci_res0 != NULL) {
349 rid = PCIR_BAR(0);
350 bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid,
351 sc->vmci_res0);
352 sc->vmci_res0 = NULL;
353 }
354
355 if (sc->vmci_res1 != NULL) {
356 rid = PCIR_BAR(1);
357 bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid,
358 sc->vmci_res1);
359 sc->vmci_res1 = NULL;
360 }
361 }
362
363 /*
364 *------------------------------------------------------------------------------
365 *
366 * vmci_config_capabilities --
367 *
368 * Check the VMCI device capabilities and configure the device accordingly.
369 *
370 * Results:
371 * 0 if success, ENODEV otherwise.
372 *
373 * Side effects:
374 * Device capabilities are enabled.
375 *
376 *------------------------------------------------------------------------------
377 */
378
379 static int
vmci_config_capabilities(struct vmci_softc * sc)380 vmci_config_capabilities(struct vmci_softc *sc)
381 {
382 unsigned long bitmap_PPN;
383 int error;
384
385 /*
386 * Verify that the VMCI device supports the capabilities that we
387 * need. Datagrams are necessary and notifications will be used
388 * if the device supports it.
389 */
390 sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0,
391 VMCI_CAPS_ADDR);
392
393 if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) {
394 VMCI_LOG_ERROR(LGPFX"VMCI device does not support "
395 "datagrams.\n");
396 return (ENODEV);
397 }
398
399 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
400 sc->capabilities = VMCI_CAPS_DATAGRAM;
401 error = vmci_dma_malloc(PAGE_SIZE, 1,
402 &sc->vmci_notifications_bitmap);
403 if (error)
404 VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for "
405 "notification bitmap.\n");
406 else {
407 memset(sc->vmci_notifications_bitmap.dma_vaddr, 0,
408 PAGE_SIZE);
409 sc->capabilities |= VMCI_CAPS_NOTIFICATIONS;
410 }
411 } else
412 sc->capabilities = VMCI_CAPS_DATAGRAM;
413
414 /* Let the host know which capabilities we intend to use. */
415 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
416 VMCI_CAPS_ADDR, sc->capabilities);
417
418 /*
419 * Register notification bitmap with device if that capability is
420 * used.
421 */
422 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
423 bitmap_PPN =
424 sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT;
425 vmci_register_notification_bitmap(bitmap_PPN);
426 }
427
428 /* Check host capabilities. */
429 if (!vmci_check_host_capabilities())
430 return (ENODEV);
431
432 return (0);
433 }
434
435 /*
436 *------------------------------------------------------------------------------
437 *
438 * vmci_dmamap_cb --
439 *
440 * Callback to receive mapping information resulting from the load of a
441 * bus_dmamap_t via bus_dmamap_load()
442 *
443 * Results:
444 * None.
445 *
446 * Side effects:
447 * None.
448 *
449 *------------------------------------------------------------------------------
450 */
451
452 static void
vmci_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)453 vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
454 {
455 bus_addr_t *baddr = arg;
456
457 if (error == 0)
458 *baddr = segs->ds_addr;
459 }
460
461 /*
462 *------------------------------------------------------------------------------
463 *
464 * vmci_dma_malloc_int --
465 *
466 * Internal function that allocates DMA memory.
467 *
468 * Results:
469 * 0 if success.
470 * ENOMEM if insufficient memory.
471 * EINPROGRESS if mapping is deferred.
472 * EINVAL if the request was invalid.
473 *
474 * Side effects:
475 * DMA memory is allocated.
476 *
477 *------------------------------------------------------------------------------
478 */
479
480 static int
vmci_dma_malloc_int(struct vmci_softc * sc,bus_size_t size,bus_size_t align,struct vmci_dma_alloc * dma)481 vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align,
482 struct vmci_dma_alloc *dma)
483 {
484 int error;
485
486 bzero(dma, sizeof(struct vmci_dma_alloc));
487
488 error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev),
489 align, 0, /* alignment, bounds */
490 BUS_SPACE_MAXADDR, /* lowaddr */
491 BUS_SPACE_MAXADDR, /* highaddr */
492 NULL, NULL, /* filter, filterarg */
493 size, /* maxsize */
494 1, /* nsegments */
495 size, /* maxsegsize */
496 BUS_DMA_ALLOCNOW, /* flags */
497 NULL, /* lockfunc */
498 NULL, /* lockfuncarg */
499 &dma->dma_tag);
500 if (error) {
501 VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error);
502 goto fail;
503 }
504
505 error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
506 BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
507 if (error) {
508 VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error);
509 goto fail;
510 }
511
512 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
513 size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
514 if (error) {
515 VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error);
516 goto fail;
517 }
518
519 dma->dma_size = size;
520
521 fail:
522 if (error)
523 vmci_dma_free(dma);
524
525 return (error);
526 }
527
528 /*
529 *------------------------------------------------------------------------------
530 *
531 * vmci_dma_malloc --
532 *
533 * This function is a wrapper around vmci_dma_malloc_int for callers
534 * outside of this module. Since we only support a single VMCI device, this
535 * wrapper provides access to the device softc structure.
536 *
537 * Results:
538 * 0 if success.
539 * ENOMEM if insufficient memory.
540 * EINPROGRESS if mapping is deferred.
541 * EINVAL if the request was invalid.
542 *
543 * Side effects:
544 * DMA memory is allocated.
545 *
546 *------------------------------------------------------------------------------
547 */
548
549 int
vmci_dma_malloc(bus_size_t size,bus_size_t align,struct vmci_dma_alloc * dma)550 vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma)
551 {
552
553 return (vmci_dma_malloc_int(vmci_sc, size, align, dma));
554 }
555
556 /*
557 *------------------------------------------------------------------------------
558 *
559 * vmci_dma_free_int --
560 *
561 * Internal function that frees DMA memory.
562 *
563 * Results:
564 * None.
565 *
566 * Side effects:
567 * Frees DMA memory.
568 *
569 *------------------------------------------------------------------------------
570 */
571
572 static void
vmci_dma_free_int(struct vmci_softc * sc,struct vmci_dma_alloc * dma)573 vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma)
574 {
575
576 if (dma->dma_tag != NULL) {
577 if (dma->dma_paddr != 0) {
578 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
579 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
580 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
581 }
582
583 if (dma->dma_vaddr != NULL)
584 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
585 dma->dma_map);
586
587 bus_dma_tag_destroy(dma->dma_tag);
588 }
589 bzero(dma, sizeof(struct vmci_dma_alloc));
590 }
591
592 /*
593 *------------------------------------------------------------------------------
594 *
595 * vmci_dma_free --
596 *
597 * This function is a wrapper around vmci_dma_free_int for callers outside
598 * of this module. Since we only support a single VMCI device, this wrapper
599 * provides access to the device softc structure.
600 *
601 * Results:
602 * None.
603 *
604 * Side effects:
605 * Frees DMA memory.
606 *
607 *------------------------------------------------------------------------------
608 */
609
610 void
vmci_dma_free(struct vmci_dma_alloc * dma)611 vmci_dma_free(struct vmci_dma_alloc *dma)
612 {
613
614 vmci_dma_free_int(vmci_sc, dma);
615 }
616
617 /*
618 *------------------------------------------------------------------------------
619 *
620 * vmci_config_interrupts --
621 *
622 * Configures and enables interrupts. Try to configure MSI-X. If this fails,
623 * try to configure MSI. If even this fails, try legacy interrupts.
624 *
625 * Results:
626 * 0 if success.
627 * ENOMEM if insufficient memory.
628 * ENODEV if the device doesn't support interrupts.
629 * ENXIO if the device configuration failed.
630 *
631 * Side effects:
632 * Interrupts get enabled if successful.
633 *
634 *------------------------------------------------------------------------------
635 */
636
637 static int
vmci_config_interrupts(struct vmci_softc * sc)638 vmci_config_interrupts(struct vmci_softc *sc)
639 {
640 int error;
641
642 data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT);
643 if (data_buffer == NULL)
644 return (ENOMEM);
645
646 sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX;
647 error = vmci_config_interrupt(sc);
648 if (error) {
649 sc->vmci_intr_type = VMCI_INTR_TYPE_MSI;
650 error = vmci_config_interrupt(sc);
651 }
652 if (error) {
653 sc->vmci_intr_type = VMCI_INTR_TYPE_INTX;
654 error = vmci_config_interrupt(sc);
655 }
656 if (error)
657 return (error);
658
659 /* Enable specific interrupt bits. */
660 if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS)
661 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
662 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION);
663 else
664 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
665 VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM);
666
667 /* Enable interrupts. */
668 bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
669 VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE);
670
671 return (0);
672 }
673
674 /*
675 *------------------------------------------------------------------------------
676 *
677 * vmci_config_interrupt --
678 *
679 * Check the number of interrupts supported, allocate resources and setup
680 * interrupts.
681 *
682 * Results:
683 * 0 if success.
684 * ENOMEM if insufficient memory.
685 * ENODEV if the device doesn't support interrupts.
686 * ENXIO if the device configuration failed.
687 *
688 * Side effects:
689 * Resources get allocated and interrupts get setup (but not enabled) if
690 * successful.
691 *
692 *------------------------------------------------------------------------------
693 */
694
695 static int
vmci_config_interrupt(struct vmci_softc * sc)696 vmci_config_interrupt(struct vmci_softc *sc)
697 {
698 int error;
699
700 error = vmci_check_intr_cnt(sc);
701 if (error)
702 return (error);
703
704 error = vmci_allocate_interrupt_resources(sc);
705 if (error)
706 return (error);
707
708 error = vmci_setup_interrupts(sc);
709 if (error)
710 return (error);
711
712 return (0);
713 }
714
715 /*
716 *------------------------------------------------------------------------------
717 *
718 * vmci_check_intr_cnt --
719 *
720 * Check the number of interrupts supported by the device and ask PCI bus
721 * to allocate appropriate number of interrupts.
722 *
723 * Results:
724 * 0 if success.
725 * ENODEV if the device doesn't support any interrupts.
726 * ENXIO if the device configuration failed.
727 *
728 * Side effects:
729 * Resources get allocated on success.
730 *
731 *------------------------------------------------------------------------------
732 */
733
734 static int
vmci_check_intr_cnt(struct vmci_softc * sc)735 vmci_check_intr_cnt(struct vmci_softc *sc)
736 {
737
738 if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) {
739 sc->vmci_num_intr = 1;
740 return (0);
741 }
742
743 /*
744 * Make sure that the device supports the required number of MSI/MSI-X
745 * messages. We try for 2 MSI-X messages but 1 is good too. We need at
746 * least 1 MSI message.
747 */
748 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
749 pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev);
750
751 if (!sc->vmci_num_intr) {
752 VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt"
753 " messages");
754 return (ENODEV);
755 }
756
757 sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
758 VMCI_MAX_INTRS : 1;
759 if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) {
760 if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr))
761 return (ENXIO);
762 } else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) {
763 if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr))
764 return (ENXIO);
765 }
766
767 return (0);
768 }
769
770 /*
771 *------------------------------------------------------------------------------
772 *
773 * vmci_allocate_interrupt_resources --
774 *
775 * Allocate resources necessary for interrupts.
776 *
777 * Results:
778 * 0 if success, ENXIO otherwise.
779 *
780 * Side effects:
781 * Resources get allocated on success.
782 *
783 *------------------------------------------------------------------------------
784 */
785
786 static int
vmci_allocate_interrupt_resources(struct vmci_softc * sc)787 vmci_allocate_interrupt_resources(struct vmci_softc *sc)
788 {
789 struct resource *irq;
790 int flags, i, rid;
791
792 flags = RF_ACTIVE;
793 flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0;
794 rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1;
795
796 for (i = 0; i < sc->vmci_num_intr; i++, rid++) {
797 irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid,
798 flags);
799 if (irq == NULL)
800 return (ENXIO);
801 sc->vmci_intrs[i].vmci_irq = irq;
802 sc->vmci_intrs[i].vmci_rid = rid;
803 }
804
805 return (0);
806 }
807
808 /*
809 *------------------------------------------------------------------------------
810 *
811 * vmci_setup_interrupts --
812 *
813 * Sets up the interrupts.
814 *
815 * Results:
816 * 0 if success, appropriate error code from bus_setup_intr otherwise.
817 *
818 * Side effects:
819 * Interrupt handler gets attached.
820 *
821 *------------------------------------------------------------------------------
822 */
823
824 static int
vmci_setup_interrupts(struct vmci_softc * sc)825 vmci_setup_interrupts(struct vmci_softc *sc)
826 {
827 struct vmci_interrupt *intr;
828 int error, flags;
829
830 flags = INTR_TYPE_NET | INTR_MPSAFE;
831 if (sc->vmci_num_intr > 1)
832 flags |= INTR_EXCL;
833
834 intr = &sc->vmci_intrs[0];
835 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL,
836 vmci_interrupt, NULL, &intr->vmci_handler);
837 if (error)
838 return (error);
839 bus_describe_intr(sc->vmci_dev, intr->vmci_irq, intr->vmci_handler,
840 "vmci_interrupt");
841
842 if (sc->vmci_num_intr == 2) {
843 intr = &sc->vmci_intrs[1];
844 error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags,
845 NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler);
846 if (error)
847 return (error);
848 bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
849 intr->vmci_handler, "vmci_interrupt_bm");
850 }
851
852 return (0);
853 }
854
855 /*
856 *------------------------------------------------------------------------------
857 *
858 * vmci_interrupt --
859 *
860 * Interrupt handler for legacy or MSI interrupt, or for first MSI-X
861 * interrupt (vector VMCI_INTR_DATAGRAM).
862 *
863 * Results:
864 * None.
865 *
866 * Side effects:
867 * None.
868 *
869 *------------------------------------------------------------------------------
870 */
871
872 static void
vmci_interrupt(void * arg)873 vmci_interrupt(void *arg)
874 {
875
876 if (vmci_sc->vmci_num_intr == 2)
877 taskqueue_enqueue(taskqueue_swi,
878 &vmci_sc->vmci_interrupt_dq_task);
879 else {
880 unsigned int icr;
881
882 icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR);
883 if (icr == 0 || icr == 0xffffffff)
884 return;
885 if (icr & VMCI_ICR_DATAGRAM) {
886 taskqueue_enqueue(taskqueue_swi,
887 &vmci_sc->vmci_interrupt_dq_task);
888 icr &= ~VMCI_ICR_DATAGRAM;
889 }
890 if (icr & VMCI_ICR_NOTIFICATION) {
891 taskqueue_enqueue(taskqueue_swi,
892 &vmci_sc->vmci_interrupt_bm_task);
893 icr &= ~VMCI_ICR_NOTIFICATION;
894 }
895 if (icr != 0)
896 VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt "
897 "cause");
898 }
899 }
900
901 /*
902 *------------------------------------------------------------------------------
903 *
904 * vmci_interrupt_bm --
905 *
906 * Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
907 * which is for the notification bitmap. Will only get called if we are
908 * using MSI-X with exclusive vectors.
909 *
910 * Results:
911 * None.
912 *
913 * Side effects:
914 * None.
915 *
916 *------------------------------------------------------------------------------
917 */
918
919 static void
vmci_interrupt_bm(void * arg)920 vmci_interrupt_bm(void *arg)
921 {
922
923 ASSERT(vmci_sc->vmci_num_intr == 2);
924 taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task);
925 }
926
927 /*
928 *------------------------------------------------------------------------------
929 *
930 * dispatch_datagrams --
931 *
932 * Reads and dispatches incoming datagrams.
933 *
934 * Results:
935 * None.
936 *
937 * Side effects:
938 * Reads data from the device.
939 *
940 *------------------------------------------------------------------------------
941 */
942
943 static void
dispatch_datagrams(void * context,int data)944 dispatch_datagrams(void *context, int data)
945 {
946
947 if (data_buffer == NULL)
948 VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer "
949 "present");
950
951 vmci_read_datagrams_from_port((vmci_io_handle) 0,
952 vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR,
953 data_buffer, data_buffer_size);
954 }
955
956 /*
957 *------------------------------------------------------------------------------
958 *
959 * process_bitmap --
960 *
961 * Scans the notification bitmap for raised flags, clears them and handles
962 * the notifications.
963 *
964 * Results:
965 * None.
966 *
967 * Side effects:
968 * None.
969 *
970 *------------------------------------------------------------------------------
971 */
972
973 static void
process_bitmap(void * context,int data)974 process_bitmap(void *context, int data)
975 {
976
977 if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL)
978 VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present");
979
980 vmci_scan_notification_bitmap(
981 vmci_sc->vmci_notifications_bitmap.dma_vaddr);
982 }
983
984 /*
985 *------------------------------------------------------------------------------
986 *
987 * vmci_dismantle_interrupts --
988 *
989 * Releases resources, detaches the interrupt handler and drains the task
990 * queue.
991 *
992 * Results:
993 * None.
994 *
995 * Side effects:
996 * No more interrupts.
997 *
998 *------------------------------------------------------------------------------
999 */
1000
1001 static void
vmci_dismantle_interrupts(struct vmci_softc * sc)1002 vmci_dismantle_interrupts(struct vmci_softc *sc)
1003 {
1004 struct vmci_interrupt *intr;
1005 int i;
1006
1007 for (i = 0; i < sc->vmci_num_intr; i++) {
1008 intr = &sc->vmci_intrs[i];
1009 if (intr->vmci_handler != NULL) {
1010 bus_teardown_intr(sc->vmci_dev, intr->vmci_irq,
1011 intr->vmci_handler);
1012 intr->vmci_handler = NULL;
1013 }
1014 if (intr->vmci_irq != NULL) {
1015 bus_release_resource(sc->vmci_dev, SYS_RES_IRQ,
1016 intr->vmci_rid, intr->vmci_irq);
1017 intr->vmci_irq = NULL;
1018 intr->vmci_rid = -1;
1019 }
1020 }
1021
1022 if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) &&
1023 (sc->vmci_num_intr))
1024 pci_release_msi(sc->vmci_dev);
1025
1026 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task);
1027 taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task);
1028
1029 if (data_buffer != NULL)
1030 free(data_buffer, M_DEVBUF);
1031 }
1032
1033 /*
1034 *------------------------------------------------------------------------------
1035 *
1036 * vmci_delayed_work_fn_cb --
1037 *
1038 * Callback function that executes the queued up delayed work functions.
1039 *
1040 * Results:
1041 * None.
1042 *
1043 * Side effects:
1044 * None.
1045 *
1046 *------------------------------------------------------------------------------
1047 */
1048
1049 static void
vmci_delayed_work_fn_cb(void * context,int data)1050 vmci_delayed_work_fn_cb(void *context, int data)
1051 {
1052 vmci_list(vmci_delayed_work_info) temp_list;
1053
1054 vmci_list_init(&temp_list);
1055
1056 /*
1057 * Swap vmci_delayed_work_infos list with the empty temp_list while
1058 * holding a lock. vmci_delayed_work_infos would then be an empty list
1059 * and temp_list would contain the elements from the original
1060 * vmci_delayed_work_infos. Finally, iterate through temp_list
1061 * executing the delayed callbacks.
1062 */
1063
1064 mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1065 vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos,
1066 vmci_delayed_work_info, entry);
1067 mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1068
1069 while (!vmci_list_empty(&temp_list)) {
1070 struct vmci_delayed_work_info *delayed_work_info =
1071 vmci_list_first(&temp_list);
1072
1073 delayed_work_info->work_fn(delayed_work_info->data);
1074
1075 vmci_list_remove(delayed_work_info, entry);
1076 vmci_free_kernel_mem(delayed_work_info,
1077 sizeof(*delayed_work_info));
1078 }
1079 }
1080
1081 /*
1082 *------------------------------------------------------------------------------
1083 *
1084 * vmci_schedule_delayed_work_fn --
1085 *
1086 * Schedule the specified callback.
1087 *
1088 * Results:
1089 * 0 if success, error code otherwise.
1090 *
1091 * Side effects:
1092 * None.
1093 *
1094 *------------------------------------------------------------------------------
1095 */
1096
1097 int
vmci_schedule_delayed_work_fn(vmci_work_fn * work_fn,void * data)1098 vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data)
1099 {
1100 struct vmci_delayed_work_info *delayed_work_info;
1101
1102 delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info),
1103 VMCI_MEMORY_ATOMIC);
1104
1105 if (!delayed_work_info)
1106 return (VMCI_ERROR_NO_MEM);
1107
1108 delayed_work_info->work_fn = work_fn;
1109 delayed_work_info->data = data;
1110 mtx_lock(&vmci_sc->vmci_delayed_work_lock);
1111 vmci_list_insert(&vmci_sc->vmci_delayed_work_infos,
1112 delayed_work_info, entry);
1113 mtx_unlock(&vmci_sc->vmci_delayed_work_lock);
1114
1115 taskqueue_enqueue(taskqueue_thread,
1116 &vmci_sc->vmci_delayed_work_task);
1117
1118 return (VMCI_SUCCESS);
1119 }
1120
1121 /*
1122 *------------------------------------------------------------------------------
1123 *
1124 * vmci_send_datagram --
1125 *
1126 * VM to hypervisor call mechanism.
1127 *
1128 * Results:
1129 * The result of the hypercall.
1130 *
1131 * Side effects:
1132 * None.
1133 *
1134 *------------------------------------------------------------------------------
1135 */
1136
1137 int
vmci_send_datagram(struct vmci_datagram * dg)1138 vmci_send_datagram(struct vmci_datagram *dg)
1139 {
1140 int result;
1141
1142 if (dg == NULL)
1143 return (VMCI_ERROR_INVALID_ARGS);
1144
1145 /*
1146 * Need to acquire spinlock on the device because
1147 * the datagram data may be spread over multiple pages and the monitor
1148 * may interleave device user rpc calls from multiple VCPUs. Acquiring
1149 * the spinlock precludes that possibility. Disabling interrupts to
1150 * avoid incoming datagrams during a "rep out" and possibly landing up
1151 * in this function.
1152 */
1153 mtx_lock_spin(&vmci_sc->vmci_spinlock);
1154
1155 /*
1156 * Send the datagram and retrieve the return value from the result
1157 * register.
1158 */
1159 __asm__ __volatile__(
1160 "cld\n\t"
1161 "rep outsb\n\t"
1162 : /* No output. */
1163 : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR),
1164 "c"(VMCI_DG_SIZE(dg)), "S"(dg)
1165 );
1166
1167 /*
1168 * XXX: Should read result high port as well when updating handlers to
1169 * return 64bit.
1170 */
1171
1172 result = bus_space_read_4(vmci_sc->vmci_iot0,
1173 vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR);
1174 mtx_unlock_spin(&vmci_sc->vmci_spinlock);
1175
1176 return (result);
1177 }
1178