1 /******************************************************************************
2 * xen_intr.c
3 *
4 * Xen event and interrupt services for x86 HVM guests.
5 *
6 * Copyright (c) 2002-2005, K A Fraser
7 * Copyright (c) 2005, Intel Corporation <[email protected]>
8 * Copyright (c) 2012, Spectra Logic Corporation
9 *
10 * This file may be distributed separately from the Linux kernel, or
11 * incorporated into other software packages, subject to the following license:
12 *
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
16 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
17 * and to permit persons to whom the Software is furnished to do so, subject to
18 * the following conditions:
19 *
20 * The above copyright notice and this permission notice shall be included in
21 * all copies or substantial portions of the Software.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 * IN THE SOFTWARE.
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include "opt_ddb.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/bus.h>
40 #include <sys/malloc.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/interrupt.h>
46 #include <sys/pcpu.h>
47 #include <sys/smp.h>
48 #include <sys/refcount.h>
49
50 #include <vm/vm.h>
51 #include <vm/pmap.h>
52
53 #include <machine/intr_machdep.h>
54 #include <x86/apicvar.h>
55 #include <x86/apicreg.h>
56 #include <machine/smp.h>
57 #include <machine/stdarg.h>
58
59 #include <machine/xen/synch_bitops.h>
60 #include <machine/xen/xen-os.h>
61
62 #include <xen/xen-os.h>
63 #include <xen/hvm.h>
64 #include <xen/hypervisor.h>
65 #include <xen/xen_intr.h>
66 #include <xen/evtchn/evtchnvar.h>
67
68 #include <dev/xen/xenpci/xenpcivar.h>
69 #include <dev/pci/pcivar.h>
70
71 #ifdef DDB
72 #include <ddb/ddb.h>
73 #endif
74
75 static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services");
76
77 static u_int first_evtchn_irq;
78
79 /**
80 * Per-cpu event channel processing state.
81 */
82 struct xen_intr_pcpu_data {
83 /**
84 * The last event channel bitmap section (level one bit) processed.
85 * This is used to ensure we scan all ports before
86 * servicing an already servied port again.
87 */
88 u_int last_processed_l1i;
89
90 /**
91 * The last event channel processed within the event channel
92 * bitmap being scanned.
93 */
94 u_int last_processed_l2i;
95
96 /** Pointer to this CPU's interrupt statistic counter. */
97 u_long *evtchn_intrcnt;
98
99 /**
100 * A bitmap of ports that can be serviced from this CPU.
101 * A set bit means interrupt handling is enabled.
102 */
103 u_long evtchn_enabled[sizeof(u_long) * 8];
104 };
105
106 /*
107 * Start the scan at port 0 by initializing the last scanned
108 * location as the highest numbered event channel port.
109 */
110 DPCPU_DEFINE_STATIC(struct xen_intr_pcpu_data, xen_intr_pcpu) = {
111 .last_processed_l1i = LONG_BIT - 1,
112 .last_processed_l2i = LONG_BIT - 1
113 };
114
115 DPCPU_DECLARE(struct vcpu_info *, vcpu_info);
116
117 #define XEN_EEXIST 17 /* Xen "already exists" error */
118 #define XEN_ALLOCATE_VECTOR 0 /* Allocate a vector for this event channel */
119 #define XEN_INVALID_EVTCHN 0 /* Invalid event channel */
120
121 #define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN)
122
123 struct xenisrc {
124 struct intsrc xi_intsrc;
125 enum evtchn_type xi_type;
126 int xi_cpu; /* VCPU for delivery. */
127 int xi_vector; /* Global isrc vector number. */
128 evtchn_port_t xi_port;
129 int xi_pirq;
130 int xi_virq;
131 void *xi_cookie;
132 u_int xi_close:1; /* close on unbind? */
133 u_int xi_activehi:1;
134 u_int xi_edgetrigger:1;
135 u_int xi_masked:1;
136 volatile u_int xi_refcount;
137 };
138
139 static void xen_intr_suspend(struct pic *);
140 static void xen_intr_resume(struct pic *, bool suspend_cancelled);
141 static void xen_intr_enable_source(struct intsrc *isrc);
142 static void xen_intr_disable_source(struct intsrc *isrc, int eoi);
143 static void xen_intr_eoi_source(struct intsrc *isrc);
144 static void xen_intr_enable_intr(struct intsrc *isrc);
145 static void xen_intr_disable_intr(struct intsrc *isrc);
146 static int xen_intr_vector(struct intsrc *isrc);
147 static int xen_intr_source_pending(struct intsrc *isrc);
148 static int xen_intr_config_intr(struct intsrc *isrc,
149 enum intr_trigger trig, enum intr_polarity pol);
150 static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id);
151
152 static void xen_intr_pirq_enable_source(struct intsrc *isrc);
153 static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi);
154 static void xen_intr_pirq_eoi_source(struct intsrc *isrc);
155 static void xen_intr_pirq_enable_intr(struct intsrc *isrc);
156 static void xen_intr_pirq_disable_intr(struct intsrc *isrc);
157 static int xen_intr_pirq_config_intr(struct intsrc *isrc,
158 enum intr_trigger trig, enum intr_polarity pol);
159
160 /**
161 * PIC interface for all event channel port types except physical IRQs.
162 */
163 struct pic xen_intr_pic = {
164 .pic_enable_source = xen_intr_enable_source,
165 .pic_disable_source = xen_intr_disable_source,
166 .pic_eoi_source = xen_intr_eoi_source,
167 .pic_enable_intr = xen_intr_enable_intr,
168 .pic_disable_intr = xen_intr_disable_intr,
169 .pic_vector = xen_intr_vector,
170 .pic_source_pending = xen_intr_source_pending,
171 .pic_suspend = xen_intr_suspend,
172 .pic_resume = xen_intr_resume,
173 .pic_config_intr = xen_intr_config_intr,
174 .pic_assign_cpu = xen_intr_assign_cpu
175 };
176
177 /**
178 * PIC interface for all event channel representing
179 * physical interrupt sources.
180 */
181 struct pic xen_intr_pirq_pic = {
182 #ifdef __amd64__
183 .pic_register_sources = xenpv_register_pirqs,
184 #endif
185 .pic_enable_source = xen_intr_pirq_enable_source,
186 .pic_disable_source = xen_intr_pirq_disable_source,
187 .pic_eoi_source = xen_intr_pirq_eoi_source,
188 .pic_enable_intr = xen_intr_pirq_enable_intr,
189 .pic_disable_intr = xen_intr_pirq_disable_intr,
190 .pic_vector = xen_intr_vector,
191 .pic_source_pending = xen_intr_source_pending,
192 .pic_config_intr = xen_intr_pirq_config_intr,
193 .pic_assign_cpu = xen_intr_assign_cpu
194 };
195
196 static struct mtx xen_intr_isrc_lock;
197 static u_int xen_intr_auto_vector_count;
198 static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS];
199 static u_long *xen_intr_pirq_eoi_map;
200 static boolean_t xen_intr_pirq_eoi_map_enabled;
201
202 /*------------------------- Private Functions --------------------------------*/
203 /**
204 * Disable signal delivery for an event channel port on the
205 * specified CPU.
206 *
207 * \param port The event channel port to mask.
208 *
209 * This API is used to manage the port<=>CPU binding of event
210 * channel handlers.
211 *
212 * \note This operation does not preclude reception of an event
213 * for this event channel on another CPU. To mask the
214 * event channel globally, use evtchn_mask().
215 */
216 static inline void
evtchn_cpu_mask_port(u_int cpu,evtchn_port_t port)217 evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port)
218 {
219 struct xen_intr_pcpu_data *pcpu;
220
221 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
222 xen_clear_bit(port, pcpu->evtchn_enabled);
223 }
224
225 /**
226 * Enable signal delivery for an event channel port on the
227 * specified CPU.
228 *
229 * \param port The event channel port to unmask.
230 *
231 * This API is used to manage the port<=>CPU binding of event
232 * channel handlers.
233 *
234 * \note This operation does not guarantee that event delivery
235 * is enabled for this event channel port. The port must
236 * also be globally enabled. See evtchn_unmask().
237 */
238 static inline void
evtchn_cpu_unmask_port(u_int cpu,evtchn_port_t port)239 evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port)
240 {
241 struct xen_intr_pcpu_data *pcpu;
242
243 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
244 xen_set_bit(port, pcpu->evtchn_enabled);
245 }
246
247 /**
248 * Allocate and register a per-cpu Xen upcall interrupt counter.
249 *
250 * \param cpu The cpu for which to register this interrupt count.
251 */
252 static void
xen_intr_intrcnt_add(u_int cpu)253 xen_intr_intrcnt_add(u_int cpu)
254 {
255 char buf[MAXCOMLEN + 1];
256 struct xen_intr_pcpu_data *pcpu;
257
258 pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu);
259 if (pcpu->evtchn_intrcnt != NULL)
260 return;
261
262 snprintf(buf, sizeof(buf), "cpu%d:xen", cpu);
263 intrcnt_add(buf, &pcpu->evtchn_intrcnt);
264 }
265
266 /**
267 * Search for an already allocated but currently unused Xen interrupt
268 * source object.
269 *
270 * \param type Restrict the search to interrupt sources of the given
271 * type.
272 *
273 * \return A pointer to a free Xen interrupt source object or NULL.
274 */
275 static struct xenisrc *
xen_intr_find_unused_isrc(enum evtchn_type type)276 xen_intr_find_unused_isrc(enum evtchn_type type)
277 {
278 int isrc_idx;
279
280 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held"));
281
282 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) {
283 struct xenisrc *isrc;
284 u_int vector;
285
286 vector = first_evtchn_irq + isrc_idx;
287 isrc = (struct xenisrc *)intr_lookup_source(vector);
288 if (isrc != NULL
289 && isrc->xi_type == EVTCHN_TYPE_UNBOUND) {
290 KASSERT(isrc->xi_intsrc.is_handlers == 0,
291 ("Free evtchn still has handlers"));
292 isrc->xi_type = type;
293 return (isrc);
294 }
295 }
296 return (NULL);
297 }
298
299 /**
300 * Allocate a Xen interrupt source object.
301 *
302 * \param type The type of interrupt source to create.
303 *
304 * \return A pointer to a newly allocated Xen interrupt source
305 * object or NULL.
306 */
307 static struct xenisrc *
xen_intr_alloc_isrc(enum evtchn_type type,int vector)308 xen_intr_alloc_isrc(enum evtchn_type type, int vector)
309 {
310 static int warned;
311 struct xenisrc *isrc;
312
313 KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held"));
314
315 if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) {
316 if (!warned) {
317 warned = 1;
318 printf("xen_intr_alloc: Event channels exhausted.\n");
319 }
320 return (NULL);
321 }
322
323 if (type != EVTCHN_TYPE_PIRQ) {
324 vector = first_evtchn_irq + xen_intr_auto_vector_count;
325 xen_intr_auto_vector_count++;
326 }
327
328 KASSERT((intr_lookup_source(vector) == NULL),
329 ("Trying to use an already allocated vector"));
330
331 mtx_unlock(&xen_intr_isrc_lock);
332 isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO);
333 isrc->xi_intsrc.is_pic =
334 (type == EVTCHN_TYPE_PIRQ) ? &xen_intr_pirq_pic : &xen_intr_pic;
335 isrc->xi_vector = vector;
336 isrc->xi_type = type;
337 intr_register_source(&isrc->xi_intsrc);
338 mtx_lock(&xen_intr_isrc_lock);
339
340 return (isrc);
341 }
342
343 /**
344 * Attempt to free an active Xen interrupt source object.
345 *
346 * \param isrc The interrupt source object to release.
347 *
348 * \returns EBUSY if the source is still in use, otherwise 0.
349 */
350 static int
xen_intr_release_isrc(struct xenisrc * isrc)351 xen_intr_release_isrc(struct xenisrc *isrc)
352 {
353
354 mtx_lock(&xen_intr_isrc_lock);
355 KASSERT(isrc->xi_intsrc.is_handlers == 0,
356 ("Release called, but xenisrc still in use"));
357 evtchn_mask_port(isrc->xi_port);
358 evtchn_clear_port(isrc->xi_port);
359
360 /* Rebind port to CPU 0. */
361 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
362 evtchn_cpu_unmask_port(0, isrc->xi_port);
363
364 if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) {
365 struct evtchn_close close = { .port = isrc->xi_port };
366 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
367 panic("EVTCHNOP_close failed");
368 }
369
370 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
371 isrc->xi_cpu = 0;
372 isrc->xi_type = EVTCHN_TYPE_UNBOUND;
373 isrc->xi_port = 0;
374 isrc->xi_cookie = NULL;
375 mtx_unlock(&xen_intr_isrc_lock);
376 return (0);
377 }
378
379 /**
380 * Associate an interrupt handler with an already allocated local Xen
381 * event channel port.
382 *
383 * \param isrcp The returned Xen interrupt object associated with
384 * the specified local port.
385 * \param local_port The event channel to bind.
386 * \param type The event channel type of local_port.
387 * \param intr_owner The device making this bind request.
388 * \param filter An interrupt filter handler. Specify NULL
389 * to always dispatch to the ithread handler.
390 * \param handler An interrupt ithread handler. Optional (can
391 * specify NULL) if all necessary event actions
392 * are performed by filter.
393 * \param arg Argument to present to both filter and handler.
394 * \param irqflags Interrupt handler flags. See sys/bus.h.
395 * \param handlep Pointer to an opaque handle used to manage this
396 * registration.
397 *
398 * \returns 0 on success, otherwise an errno.
399 */
400 static int
xen_intr_bind_isrc(struct xenisrc ** isrcp,evtchn_port_t local_port,enum evtchn_type type,const char * intr_owner,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)401 xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port,
402 enum evtchn_type type, const char *intr_owner, driver_filter_t filter,
403 driver_intr_t handler, void *arg, enum intr_type flags,
404 xen_intr_handle_t *port_handlep)
405 {
406 struct xenisrc *isrc;
407 int error;
408
409 *isrcp = NULL;
410 if (port_handlep == NULL) {
411 printf("%s: xen_intr_bind_isrc: Bad event handle\n",
412 intr_owner);
413 return (EINVAL);
414 }
415
416 mtx_lock(&xen_intr_isrc_lock);
417 isrc = xen_intr_find_unused_isrc(type);
418 if (isrc == NULL) {
419 isrc = xen_intr_alloc_isrc(type, XEN_ALLOCATE_VECTOR);
420 if (isrc == NULL) {
421 mtx_unlock(&xen_intr_isrc_lock);
422 return (ENOSPC);
423 }
424 }
425 isrc->xi_port = local_port;
426 xen_intr_port_to_isrc[local_port] = isrc;
427 refcount_init(&isrc->xi_refcount, 1);
428 mtx_unlock(&xen_intr_isrc_lock);
429
430 /* Assign the opaque handler (the event channel port) */
431 *port_handlep = &isrc->xi_vector;
432
433 #ifdef SMP
434 if (type == EVTCHN_TYPE_PORT) {
435 /*
436 * By default all interrupts are assigned to vCPU#0
437 * unless specified otherwise, so shuffle them to balance
438 * the interrupt load.
439 */
440 xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu(0));
441 }
442 #endif
443
444 if (filter == NULL && handler == NULL) {
445 /*
446 * No filter/handler provided, leave the event channel
447 * masked and without a valid handler, the caller is
448 * in charge of setting that up.
449 */
450 *isrcp = isrc;
451 return (0);
452 }
453
454 error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags,
455 *port_handlep);
456 if (error != 0) {
457 xen_intr_release_isrc(isrc);
458 return (error);
459 }
460 *isrcp = isrc;
461 return (0);
462 }
463
464 /**
465 * Lookup a Xen interrupt source object given an interrupt binding handle.
466 *
467 * \param handle A handle initialized by a previous call to
468 * xen_intr_bind_isrc().
469 *
470 * \returns A pointer to the Xen interrupt source object associated
471 * with the given interrupt handle. NULL if no association
472 * currently exists.
473 */
474 static struct xenisrc *
xen_intr_isrc(xen_intr_handle_t handle)475 xen_intr_isrc(xen_intr_handle_t handle)
476 {
477 int vector;
478
479 if (handle == NULL)
480 return (NULL);
481
482 vector = *(int *)handle;
483 KASSERT(vector >= first_evtchn_irq &&
484 vector < (first_evtchn_irq + xen_intr_auto_vector_count),
485 ("Xen interrupt vector is out of range"));
486
487 return ((struct xenisrc *)intr_lookup_source(vector));
488 }
489
490 /**
491 * Determine the event channel ports at the given section of the
492 * event port bitmap which have pending events for the given cpu.
493 *
494 * \param pcpu The Xen interrupt pcpu data for the cpu being querried.
495 * \param sh The Xen shared info area.
496 * \param idx The index of the section of the event channel bitmap to
497 * inspect.
498 *
499 * \returns A u_long with bits set for every event channel with pending
500 * events.
501 */
502 static inline u_long
xen_intr_active_ports(struct xen_intr_pcpu_data * pcpu,shared_info_t * sh,u_int idx)503 xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh,
504 u_int idx)
505 {
506
507 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(sh->evtchn_pending[0]));
508 CTASSERT(sizeof(sh->evtchn_mask[0]) == sizeof(pcpu->evtchn_enabled[0]));
509 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(sh->evtchn_pending));
510 CTASSERT(sizeof(sh->evtchn_mask) == sizeof(pcpu->evtchn_enabled));
511 return (sh->evtchn_pending[idx]
512 & ~sh->evtchn_mask[idx]
513 & pcpu->evtchn_enabled[idx]);
514 }
515
516 /**
517 * Interrupt handler for processing all Xen event channel events.
518 *
519 * \param trap_frame The trap frame context for the current interrupt.
520 */
521 void
xen_intr_handle_upcall(struct trapframe * trap_frame)522 xen_intr_handle_upcall(struct trapframe *trap_frame)
523 {
524 u_int l1i, l2i, port, cpu;
525 u_long masked_l1, masked_l2;
526 struct xenisrc *isrc;
527 shared_info_t *s;
528 vcpu_info_t *v;
529 struct xen_intr_pcpu_data *pc;
530 u_long l1, l2;
531
532 /*
533 * Disable preemption in order to always check and fire events
534 * on the right vCPU
535 */
536 critical_enter();
537
538 cpu = PCPU_GET(cpuid);
539 pc = DPCPU_PTR(xen_intr_pcpu);
540 s = HYPERVISOR_shared_info;
541 v = DPCPU_GET(vcpu_info);
542
543 if (xen_hvm_domain() && !xen_vector_callback_enabled) {
544 KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU"));
545 }
546
547 v->evtchn_upcall_pending = 0;
548
549 #if 0
550 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
551 /* Clear master flag /before/ clearing selector flag. */
552 wmb();
553 #endif
554 #endif
555
556 l1 = atomic_readandclear_long(&v->evtchn_pending_sel);
557
558 l1i = pc->last_processed_l1i;
559 l2i = pc->last_processed_l2i;
560 (*pc->evtchn_intrcnt)++;
561
562 while (l1 != 0) {
563 l1i = (l1i + 1) % LONG_BIT;
564 masked_l1 = l1 & ((~0UL) << l1i);
565
566 if (masked_l1 == 0) {
567 /*
568 * if we masked out all events, wrap around
569 * to the beginning.
570 */
571 l1i = LONG_BIT - 1;
572 l2i = LONG_BIT - 1;
573 continue;
574 }
575 l1i = ffsl(masked_l1) - 1;
576
577 do {
578 l2 = xen_intr_active_ports(pc, s, l1i);
579
580 l2i = (l2i + 1) % LONG_BIT;
581 masked_l2 = l2 & ((~0UL) << l2i);
582
583 if (masked_l2 == 0) {
584 /* if we masked out all events, move on */
585 l2i = LONG_BIT - 1;
586 break;
587 }
588 l2i = ffsl(masked_l2) - 1;
589
590 /* process port */
591 port = (l1i * LONG_BIT) + l2i;
592 synch_clear_bit(port, &s->evtchn_pending[0]);
593
594 isrc = xen_intr_port_to_isrc[port];
595 if (__predict_false(isrc == NULL))
596 continue;
597
598 /* Make sure we are firing on the right vCPU */
599 KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)),
600 ("Received unexpected event on vCPU#%d, event bound to vCPU#%d",
601 PCPU_GET(cpuid), isrc->xi_cpu));
602
603 intr_execute_handlers(&isrc->xi_intsrc, trap_frame);
604
605 /*
606 * If this is the final port processed,
607 * we'll pick up here+1 next time.
608 */
609 pc->last_processed_l1i = l1i;
610 pc->last_processed_l2i = l2i;
611
612 } while (l2i != LONG_BIT - 1);
613
614 l2 = xen_intr_active_ports(pc, s, l1i);
615 if (l2 == 0) {
616 /*
617 * We handled all ports, so we can clear the
618 * selector bit.
619 */
620 l1 &= ~(1UL << l1i);
621 }
622 }
623
624 if (xen_evtchn_needs_ack)
625 lapic_eoi();
626
627 critical_exit();
628 }
629
630 static int
xen_intr_init(void * dummy __unused)631 xen_intr_init(void *dummy __unused)
632 {
633 shared_info_t *s = HYPERVISOR_shared_info;
634 struct xen_intr_pcpu_data *pcpu;
635 struct physdev_pirq_eoi_gmfn eoi_gmfn;
636 int i, rc;
637
638 if (!xen_domain())
639 return (0);
640
641 mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF);
642
643 /*
644 * Set the per-cpu mask of CPU#0 to enable all, since by default all
645 * event channels are bound to CPU#0.
646 */
647 CPU_FOREACH(i) {
648 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
649 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
650 sizeof(pcpu->evtchn_enabled));
651 }
652
653 for (i = 0; i < nitems(s->evtchn_mask); i++)
654 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
655
656 /* Try to register PIRQ EOI map */
657 xen_intr_pirq_eoi_map = malloc(PAGE_SIZE, M_XENINTR, M_WAITOK | M_ZERO);
658 eoi_gmfn.gmfn = atop(vtophys(xen_intr_pirq_eoi_map));
659 rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn);
660 if (rc != 0 && bootverbose)
661 printf("Xen interrupts: unable to register PIRQ EOI map\n");
662 else
663 xen_intr_pirq_eoi_map_enabled = true;
664
665 intr_register_pic(&xen_intr_pic);
666 if (xen_pv_domain() && xen_initial_domain())
667 intr_register_pic(&xen_intr_pirq_pic);
668
669 if (bootverbose)
670 printf("Xen interrupt system initialized\n");
671
672 return (0);
673 }
674 SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL);
675
676 static void
xen_intrcnt_init(void * dummy __unused)677 xen_intrcnt_init(void *dummy __unused)
678 {
679 unsigned int i;
680
681 if (!xen_domain())
682 return;
683
684 /*
685 * Register interrupt count manually as we aren't guaranteed to see a
686 * call to xen_intr_assign_cpu() before our first interrupt.
687 */
688 CPU_FOREACH(i)
689 xen_intr_intrcnt_add(i);
690 }
691 SYSINIT(xen_intrcnt_init, SI_SUB_INTR, SI_ORDER_MIDDLE, xen_intrcnt_init, NULL);
692
693 void
xen_intr_alloc_irqs(void)694 xen_intr_alloc_irqs(void)
695 {
696
697 if (num_io_irqs > UINT_MAX - NR_EVENT_CHANNELS)
698 panic("IRQ allocation overflow (num_msi_irqs too high?)");
699 first_evtchn_irq = num_io_irqs;
700 num_io_irqs += NR_EVENT_CHANNELS;
701 }
702
703 /*--------------------------- Common PIC Functions ---------------------------*/
704 /**
705 * Prepare this PIC for system suspension.
706 */
707 static void
xen_intr_suspend(struct pic * unused)708 xen_intr_suspend(struct pic *unused)
709 {
710 }
711
712 static void
xen_rebind_ipi(struct xenisrc * isrc)713 xen_rebind_ipi(struct xenisrc *isrc)
714 {
715 #ifdef SMP
716 int cpu = isrc->xi_cpu;
717 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
718 int error;
719 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
720
721 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi,
722 &bind_ipi);
723 if (error != 0)
724 panic("unable to rebind xen IPI: %d", error);
725
726 isrc->xi_port = bind_ipi.port;
727 isrc->xi_cpu = 0;
728 xen_intr_port_to_isrc[bind_ipi.port] = isrc;
729
730 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
731 cpu_apic_ids[cpu]);
732 if (error)
733 panic("unable to bind xen IPI to CPU#%d: %d",
734 cpu, error);
735
736 evtchn_unmask_port(bind_ipi.port);
737 #else
738 panic("Resume IPI event channel on UP");
739 #endif
740 }
741
742 static void
xen_rebind_virq(struct xenisrc * isrc)743 xen_rebind_virq(struct xenisrc *isrc)
744 {
745 int cpu = isrc->xi_cpu;
746 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
747 int error;
748 struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq,
749 .vcpu = vcpu_id };
750
751 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
752 &bind_virq);
753 if (error != 0)
754 panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error);
755
756 isrc->xi_port = bind_virq.port;
757 isrc->xi_cpu = 0;
758 xen_intr_port_to_isrc[bind_virq.port] = isrc;
759
760 #ifdef SMP
761 error = xen_intr_assign_cpu(&isrc->xi_intsrc,
762 cpu_apic_ids[cpu]);
763 if (error)
764 panic("unable to bind xen VIRQ#%d to CPU#%d: %d",
765 isrc->xi_virq, cpu, error);
766 #endif
767
768 evtchn_unmask_port(bind_virq.port);
769 }
770
771 /**
772 * Return this PIC to service after being suspended.
773 */
774 static void
xen_intr_resume(struct pic * unused,bool suspend_cancelled)775 xen_intr_resume(struct pic *unused, bool suspend_cancelled)
776 {
777 shared_info_t *s = HYPERVISOR_shared_info;
778 struct xenisrc *isrc;
779 u_int isrc_idx;
780 int i;
781
782 if (suspend_cancelled)
783 return;
784
785 /* Reset the per-CPU masks */
786 CPU_FOREACH(i) {
787 struct xen_intr_pcpu_data *pcpu;
788
789 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
790 memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0,
791 sizeof(pcpu->evtchn_enabled));
792 }
793
794 /* Mask all event channels. */
795 for (i = 0; i < nitems(s->evtchn_mask); i++)
796 atomic_store_rel_long(&s->evtchn_mask[i], ~0);
797
798 /* Remove port -> isrc mappings */
799 memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc));
800
801 /* Free unused isrcs and rebind VIRQs and IPIs */
802 for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) {
803 u_int vector;
804
805 vector = first_evtchn_irq + isrc_idx;
806 isrc = (struct xenisrc *)intr_lookup_source(vector);
807 if (isrc != NULL) {
808 isrc->xi_port = 0;
809 switch (isrc->xi_type) {
810 case EVTCHN_TYPE_IPI:
811 xen_rebind_ipi(isrc);
812 break;
813 case EVTCHN_TYPE_VIRQ:
814 xen_rebind_virq(isrc);
815 break;
816 default:
817 break;
818 }
819 }
820 }
821 }
822
823 /**
824 * Disable a Xen interrupt source.
825 *
826 * \param isrc The interrupt source to disable.
827 */
828 static void
xen_intr_disable_intr(struct intsrc * base_isrc)829 xen_intr_disable_intr(struct intsrc *base_isrc)
830 {
831 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
832
833 evtchn_mask_port(isrc->xi_port);
834 }
835
836 /**
837 * Determine the global interrupt vector number for
838 * a Xen interrupt source.
839 *
840 * \param isrc The interrupt source to query.
841 *
842 * \return The vector number corresponding to the given interrupt source.
843 */
844 static int
xen_intr_vector(struct intsrc * base_isrc)845 xen_intr_vector(struct intsrc *base_isrc)
846 {
847 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
848
849 return (isrc->xi_vector);
850 }
851
852 /**
853 * Determine whether or not interrupt events are pending on the
854 * the given interrupt source.
855 *
856 * \param isrc The interrupt source to query.
857 *
858 * \returns 0 if no events are pending, otherwise non-zero.
859 */
860 static int
xen_intr_source_pending(struct intsrc * isrc)861 xen_intr_source_pending(struct intsrc *isrc)
862 {
863 /*
864 * EventChannels are edge triggered and never masked.
865 * There can be no pending events.
866 */
867 return (0);
868 }
869
870 /**
871 * Perform configuration of an interrupt source.
872 *
873 * \param isrc The interrupt source to configure.
874 * \param trig Edge or level.
875 * \param pol Active high or low.
876 *
877 * \returns 0 if no events are pending, otherwise non-zero.
878 */
879 static int
xen_intr_config_intr(struct intsrc * isrc,enum intr_trigger trig,enum intr_polarity pol)880 xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig,
881 enum intr_polarity pol)
882 {
883 /* Configuration is only possible via the evtchn apis. */
884 return (ENODEV);
885 }
886
887 /**
888 * Configure CPU affinity for interrupt source event delivery.
889 *
890 * \param isrc The interrupt source to configure.
891 * \param apic_id The apic id of the CPU for handling future events.
892 *
893 * \returns 0 if successful, otherwise an errno.
894 */
895 static int
xen_intr_assign_cpu(struct intsrc * base_isrc,u_int apic_id)896 xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id)
897 {
898 #ifdef SMP
899 struct evtchn_bind_vcpu bind_vcpu;
900 struct xenisrc *isrc;
901 u_int to_cpu, vcpu_id;
902 int error, masked;
903
904 if (xen_vector_callback_enabled == 0)
905 return (EOPNOTSUPP);
906
907 to_cpu = apic_cpuid(apic_id);
908 vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id;
909
910 mtx_lock(&xen_intr_isrc_lock);
911 isrc = (struct xenisrc *)base_isrc;
912 if (!is_valid_evtchn(isrc->xi_port)) {
913 mtx_unlock(&xen_intr_isrc_lock);
914 return (EINVAL);
915 }
916
917 /*
918 * Mask the event channel while binding it to prevent interrupt
919 * delivery with an inconsistent state in isrc->xi_cpu.
920 */
921 masked = evtchn_test_and_set_mask(isrc->xi_port);
922 if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) ||
923 (isrc->xi_type == EVTCHN_TYPE_IPI)) {
924 /*
925 * Virtual IRQs are associated with a cpu by
926 * the Hypervisor at evtchn_bind_virq time, so
927 * all we need to do is update the per-CPU masks.
928 */
929 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
930 isrc->xi_cpu = to_cpu;
931 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
932 goto out;
933 }
934
935 bind_vcpu.port = isrc->xi_port;
936 bind_vcpu.vcpu = vcpu_id;
937
938 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu);
939 if (isrc->xi_cpu != to_cpu) {
940 if (error == 0) {
941 /* Commit to new binding by removing the old one. */
942 evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port);
943 isrc->xi_cpu = to_cpu;
944 evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port);
945 }
946 }
947
948 out:
949 if (masked == 0)
950 evtchn_unmask_port(isrc->xi_port);
951 mtx_unlock(&xen_intr_isrc_lock);
952 return (0);
953 #else
954 return (EOPNOTSUPP);
955 #endif
956 }
957
958 /*------------------- Virtual Interrupt Source PIC Functions -----------------*/
959 /*
960 * Mask a level triggered interrupt source.
961 *
962 * \param isrc The interrupt source to mask (if necessary).
963 * \param eoi If non-zero, perform any necessary end-of-interrupt
964 * acknowledgements.
965 */
966 static void
xen_intr_disable_source(struct intsrc * base_isrc,int eoi)967 xen_intr_disable_source(struct intsrc *base_isrc, int eoi)
968 {
969 struct xenisrc *isrc;
970
971 isrc = (struct xenisrc *)base_isrc;
972
973 /*
974 * NB: checking if the event channel is already masked is
975 * needed because the event channel user-space device
976 * masks event channels on its filter as part of its
977 * normal operation, and those shouldn't be automatically
978 * unmasked by the generic interrupt code. The event channel
979 * device will unmask them when needed.
980 */
981 isrc->xi_masked = !!evtchn_test_and_set_mask(isrc->xi_port);
982 }
983
984 /*
985 * Unmask a level triggered interrupt source.
986 *
987 * \param isrc The interrupt source to unmask (if necessary).
988 */
989 static void
xen_intr_enable_source(struct intsrc * base_isrc)990 xen_intr_enable_source(struct intsrc *base_isrc)
991 {
992 struct xenisrc *isrc;
993
994 isrc = (struct xenisrc *)base_isrc;
995
996 if (isrc->xi_masked == 0)
997 evtchn_unmask_port(isrc->xi_port);
998 }
999
1000 /*
1001 * Perform any necessary end-of-interrupt acknowledgements.
1002 *
1003 * \param isrc The interrupt source to EOI.
1004 */
1005 static void
xen_intr_eoi_source(struct intsrc * base_isrc)1006 xen_intr_eoi_source(struct intsrc *base_isrc)
1007 {
1008 }
1009
1010 /*
1011 * Enable and unmask the interrupt source.
1012 *
1013 * \param isrc The interrupt source to enable.
1014 */
1015 static void
xen_intr_enable_intr(struct intsrc * base_isrc)1016 xen_intr_enable_intr(struct intsrc *base_isrc)
1017 {
1018 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1019
1020 evtchn_unmask_port(isrc->xi_port);
1021 }
1022
1023 /*------------------ Physical Interrupt Source PIC Functions -----------------*/
1024 /*
1025 * Mask a level triggered interrupt source.
1026 *
1027 * \param isrc The interrupt source to mask (if necessary).
1028 * \param eoi If non-zero, perform any necessary end-of-interrupt
1029 * acknowledgements.
1030 */
1031 static void
xen_intr_pirq_disable_source(struct intsrc * base_isrc,int eoi)1032 xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi)
1033 {
1034 struct xenisrc *isrc;
1035
1036 isrc = (struct xenisrc *)base_isrc;
1037
1038 if (isrc->xi_edgetrigger == 0)
1039 evtchn_mask_port(isrc->xi_port);
1040 if (eoi == PIC_EOI)
1041 xen_intr_pirq_eoi_source(base_isrc);
1042 }
1043
1044 /*
1045 * Unmask a level triggered interrupt source.
1046 *
1047 * \param isrc The interrupt source to unmask (if necessary).
1048 */
1049 static void
xen_intr_pirq_enable_source(struct intsrc * base_isrc)1050 xen_intr_pirq_enable_source(struct intsrc *base_isrc)
1051 {
1052 struct xenisrc *isrc;
1053
1054 isrc = (struct xenisrc *)base_isrc;
1055
1056 if (isrc->xi_edgetrigger == 0)
1057 evtchn_unmask_port(isrc->xi_port);
1058 }
1059
1060 /*
1061 * Perform any necessary end-of-interrupt acknowledgements.
1062 *
1063 * \param isrc The interrupt source to EOI.
1064 */
1065 static void
xen_intr_pirq_eoi_source(struct intsrc * base_isrc)1066 xen_intr_pirq_eoi_source(struct intsrc *base_isrc)
1067 {
1068 struct xenisrc *isrc;
1069 int error;
1070
1071 isrc = (struct xenisrc *)base_isrc;
1072
1073 if (xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map)) {
1074 struct physdev_eoi eoi = { .irq = isrc->xi_pirq };
1075
1076 error = HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
1077 if (error != 0)
1078 panic("Unable to EOI PIRQ#%d: %d\n",
1079 isrc->xi_pirq, error);
1080 }
1081 }
1082
1083 /*
1084 * Enable and unmask the interrupt source.
1085 *
1086 * \param isrc The interrupt source to enable.
1087 */
1088 static void
xen_intr_pirq_enable_intr(struct intsrc * base_isrc)1089 xen_intr_pirq_enable_intr(struct intsrc *base_isrc)
1090 {
1091 struct xenisrc *isrc;
1092 struct evtchn_bind_pirq bind_pirq;
1093 struct physdev_irq_status_query irq_status;
1094 int error;
1095
1096 isrc = (struct xenisrc *)base_isrc;
1097
1098 if (!xen_intr_pirq_eoi_map_enabled) {
1099 irq_status.irq = isrc->xi_pirq;
1100 error = HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query,
1101 &irq_status);
1102 if (error)
1103 panic("unable to get status of IRQ#%d", isrc->xi_pirq);
1104
1105 if (irq_status.flags & XENIRQSTAT_needs_eoi) {
1106 /*
1107 * Since the dynamic PIRQ EOI map is not available
1108 * mark the PIRQ as needing EOI unconditionally.
1109 */
1110 xen_set_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map);
1111 }
1112 }
1113
1114 bind_pirq.pirq = isrc->xi_pirq;
1115 bind_pirq.flags = isrc->xi_edgetrigger ? 0 : BIND_PIRQ__WILL_SHARE;
1116 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq);
1117 if (error)
1118 panic("unable to bind IRQ#%d", isrc->xi_pirq);
1119
1120 isrc->xi_port = bind_pirq.port;
1121
1122 mtx_lock(&xen_intr_isrc_lock);
1123 KASSERT((xen_intr_port_to_isrc[bind_pirq.port] == NULL),
1124 ("trying to override an already setup event channel port"));
1125 xen_intr_port_to_isrc[bind_pirq.port] = isrc;
1126 mtx_unlock(&xen_intr_isrc_lock);
1127
1128 evtchn_unmask_port(isrc->xi_port);
1129 }
1130
1131 /*
1132 * Disable an interrupt source.
1133 *
1134 * \param isrc The interrupt source to disable.
1135 */
1136 static void
xen_intr_pirq_disable_intr(struct intsrc * base_isrc)1137 xen_intr_pirq_disable_intr(struct intsrc *base_isrc)
1138 {
1139 struct xenisrc *isrc;
1140 struct evtchn_close close;
1141 int error;
1142
1143 isrc = (struct xenisrc *)base_isrc;
1144
1145 evtchn_mask_port(isrc->xi_port);
1146
1147 close.port = isrc->xi_port;
1148 error = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
1149 if (error)
1150 panic("unable to close event channel %d IRQ#%d",
1151 isrc->xi_port, isrc->xi_pirq);
1152
1153 mtx_lock(&xen_intr_isrc_lock);
1154 xen_intr_port_to_isrc[isrc->xi_port] = NULL;
1155 mtx_unlock(&xen_intr_isrc_lock);
1156
1157 isrc->xi_port = 0;
1158 }
1159
1160 /**
1161 * Perform configuration of an interrupt source.
1162 *
1163 * \param isrc The interrupt source to configure.
1164 * \param trig Edge or level.
1165 * \param pol Active high or low.
1166 *
1167 * \returns 0 if no events are pending, otherwise non-zero.
1168 */
1169 static int
xen_intr_pirq_config_intr(struct intsrc * base_isrc,enum intr_trigger trig,enum intr_polarity pol)1170 xen_intr_pirq_config_intr(struct intsrc *base_isrc, enum intr_trigger trig,
1171 enum intr_polarity pol)
1172 {
1173 struct xenisrc *isrc = (struct xenisrc *)base_isrc;
1174 struct physdev_setup_gsi setup_gsi;
1175 int error;
1176
1177 KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM),
1178 ("%s: Conforming trigger or polarity\n", __func__));
1179
1180 setup_gsi.gsi = isrc->xi_pirq;
1181 setup_gsi.triggering = trig == INTR_TRIGGER_EDGE ? 0 : 1;
1182 setup_gsi.polarity = pol == INTR_POLARITY_HIGH ? 0 : 1;
1183
1184 error = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi);
1185 if (error == -XEN_EEXIST) {
1186 if ((isrc->xi_edgetrigger && (trig != INTR_TRIGGER_EDGE)) ||
1187 (isrc->xi_activehi && (pol != INTR_POLARITY_HIGH)))
1188 panic("unable to reconfigure interrupt IRQ#%d",
1189 isrc->xi_pirq);
1190 error = 0;
1191 }
1192 if (error)
1193 panic("unable to configure IRQ#%d\n", isrc->xi_pirq);
1194
1195 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1196 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1197
1198 return (0);
1199 }
1200
1201 /*--------------------------- Public Functions -------------------------------*/
1202 /*------- API comments for these methods can be found in xen/xenintr.h -------*/
1203 int
xen_intr_bind_local_port(device_t dev,evtchn_port_t local_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1204 xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port,
1205 driver_filter_t filter, driver_intr_t handler, void *arg,
1206 enum intr_type flags, xen_intr_handle_t *port_handlep)
1207 {
1208 struct xenisrc *isrc;
1209 int error;
1210
1211 error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT,
1212 device_get_nameunit(dev), filter, handler, arg, flags,
1213 port_handlep);
1214 if (error != 0)
1215 return (error);
1216
1217 /*
1218 * The Event Channel API didn't open this port, so it is not
1219 * responsible for closing it automatically on unbind.
1220 */
1221 isrc->xi_close = 0;
1222 return (0);
1223 }
1224
1225 int
xen_intr_alloc_and_bind_local_port(device_t dev,u_int remote_domain,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1226 xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain,
1227 driver_filter_t filter, driver_intr_t handler, void *arg,
1228 enum intr_type flags, xen_intr_handle_t *port_handlep)
1229 {
1230 struct xenisrc *isrc;
1231 struct evtchn_alloc_unbound alloc_unbound;
1232 int error;
1233
1234 alloc_unbound.dom = DOMID_SELF;
1235 alloc_unbound.remote_dom = remote_domain;
1236 error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
1237 &alloc_unbound);
1238 if (error != 0) {
1239 /*
1240 * XXX Trap Hypercall error code Linuxisms in
1241 * the HYPERCALL layer.
1242 */
1243 return (-error);
1244 }
1245
1246 error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT,
1247 device_get_nameunit(dev), filter, handler, arg, flags,
1248 port_handlep);
1249 if (error != 0) {
1250 evtchn_close_t close = { .port = alloc_unbound.port };
1251 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1252 panic("EVTCHNOP_close failed");
1253 return (error);
1254 }
1255
1256 isrc->xi_close = 1;
1257 return (0);
1258 }
1259
1260 int
xen_intr_bind_remote_port(device_t dev,u_int remote_domain,u_int remote_port,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1261 xen_intr_bind_remote_port(device_t dev, u_int remote_domain,
1262 u_int remote_port, driver_filter_t filter, driver_intr_t handler,
1263 void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep)
1264 {
1265 struct xenisrc *isrc;
1266 struct evtchn_bind_interdomain bind_interdomain;
1267 int error;
1268
1269 bind_interdomain.remote_dom = remote_domain;
1270 bind_interdomain.remote_port = remote_port;
1271 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
1272 &bind_interdomain);
1273 if (error != 0) {
1274 /*
1275 * XXX Trap Hypercall error code Linuxisms in
1276 * the HYPERCALL layer.
1277 */
1278 return (-error);
1279 }
1280
1281 error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port,
1282 EVTCHN_TYPE_PORT, device_get_nameunit(dev), filter, handler, arg,
1283 flags, port_handlep);
1284 if (error) {
1285 evtchn_close_t close = { .port = bind_interdomain.local_port };
1286 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1287 panic("EVTCHNOP_close failed");
1288 return (error);
1289 }
1290
1291 /*
1292 * The Event Channel API opened this port, so it is
1293 * responsible for closing it automatically on unbind.
1294 */
1295 isrc->xi_close = 1;
1296 return (0);
1297 }
1298
1299 int
xen_intr_bind_virq(device_t dev,u_int virq,u_int cpu,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t * port_handlep)1300 xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu,
1301 driver_filter_t filter, driver_intr_t handler, void *arg,
1302 enum intr_type flags, xen_intr_handle_t *port_handlep)
1303 {
1304 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1305 struct xenisrc *isrc;
1306 struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id };
1307 int error;
1308
1309 isrc = NULL;
1310 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq);
1311 if (error != 0) {
1312 /*
1313 * XXX Trap Hypercall error code Linuxisms in
1314 * the HYPERCALL layer.
1315 */
1316 return (-error);
1317 }
1318
1319 error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ,
1320 device_get_nameunit(dev), filter, handler, arg, flags,
1321 port_handlep);
1322
1323 #ifdef SMP
1324 if (error == 0)
1325 error = intr_event_bind(isrc->xi_intsrc.is_event, cpu);
1326 #endif
1327
1328 if (error != 0) {
1329 evtchn_close_t close = { .port = bind_virq.port };
1330
1331 xen_intr_unbind(*port_handlep);
1332 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1333 panic("EVTCHNOP_close failed");
1334 return (error);
1335 }
1336
1337 #ifdef SMP
1338 if (isrc->xi_cpu != cpu) {
1339 /*
1340 * Too early in the boot process for the generic interrupt
1341 * code to perform the binding. Update our event channel
1342 * masks manually so events can't fire on the wrong cpu
1343 * during AP startup.
1344 */
1345 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1346 }
1347 #endif
1348
1349 /*
1350 * The Event Channel API opened this port, so it is
1351 * responsible for closing it automatically on unbind.
1352 */
1353 isrc->xi_close = 1;
1354 isrc->xi_virq = virq;
1355
1356 return (0);
1357 }
1358
1359 int
xen_intr_alloc_and_bind_ipi(u_int cpu,driver_filter_t filter,enum intr_type flags,xen_intr_handle_t * port_handlep)1360 xen_intr_alloc_and_bind_ipi(u_int cpu, driver_filter_t filter,
1361 enum intr_type flags, xen_intr_handle_t *port_handlep)
1362 {
1363 #ifdef SMP
1364 int vcpu_id = pcpu_find(cpu)->pc_vcpu_id;
1365 struct xenisrc *isrc;
1366 struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id };
1367 /* Same size as the one used by intr_handler->ih_name. */
1368 char name[MAXCOMLEN + 1];
1369 int error;
1370
1371 isrc = NULL;
1372 error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi);
1373 if (error != 0) {
1374 /*
1375 * XXX Trap Hypercall error code Linuxisms in
1376 * the HYPERCALL layer.
1377 */
1378 return (-error);
1379 }
1380
1381 snprintf(name, sizeof(name), "cpu%u", cpu);
1382
1383 error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI,
1384 name, filter, NULL, NULL, flags, port_handlep);
1385 if (error != 0) {
1386 evtchn_close_t close = { .port = bind_ipi.port };
1387
1388 xen_intr_unbind(*port_handlep);
1389 if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close))
1390 panic("EVTCHNOP_close failed");
1391 return (error);
1392 }
1393
1394 if (isrc->xi_cpu != cpu) {
1395 /*
1396 * Too early in the boot process for the generic interrupt
1397 * code to perform the binding. Update our event channel
1398 * masks manually so events can't fire on the wrong cpu
1399 * during AP startup.
1400 */
1401 xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]);
1402 }
1403
1404 /*
1405 * The Event Channel API opened this port, so it is
1406 * responsible for closing it automatically on unbind.
1407 */
1408 isrc->xi_close = 1;
1409 return (0);
1410 #else
1411 return (EOPNOTSUPP);
1412 #endif
1413 }
1414
1415 int
xen_register_pirq(int vector,enum intr_trigger trig,enum intr_polarity pol)1416 xen_register_pirq(int vector, enum intr_trigger trig, enum intr_polarity pol)
1417 {
1418 struct physdev_map_pirq map_pirq;
1419 struct xenisrc *isrc;
1420 int error;
1421
1422 if (vector == 0)
1423 return (EINVAL);
1424
1425 if (bootverbose)
1426 printf("xen: register IRQ#%d\n", vector);
1427
1428 map_pirq.domid = DOMID_SELF;
1429 map_pirq.type = MAP_PIRQ_TYPE_GSI;
1430 map_pirq.index = vector;
1431 map_pirq.pirq = vector;
1432
1433 error = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_pirq);
1434 if (error) {
1435 printf("xen: unable to map IRQ#%d\n", vector);
1436 return (error);
1437 }
1438
1439 mtx_lock(&xen_intr_isrc_lock);
1440 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector);
1441 mtx_unlock(&xen_intr_isrc_lock);
1442 KASSERT((isrc != NULL), ("xen: unable to allocate isrc for interrupt"));
1443 isrc->xi_pirq = vector;
1444 isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0;
1445 isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0;
1446
1447 return (0);
1448 }
1449
1450 int
xen_register_msi(device_t dev,int vector,int count)1451 xen_register_msi(device_t dev, int vector, int count)
1452 {
1453 struct physdev_map_pirq msi_irq;
1454 struct xenisrc *isrc;
1455 int ret;
1456
1457 memset(&msi_irq, 0, sizeof(msi_irq));
1458 msi_irq.domid = DOMID_SELF;
1459 msi_irq.type = count == 1 ?
1460 MAP_PIRQ_TYPE_MSI_SEG : MAP_PIRQ_TYPE_MULTI_MSI;
1461 msi_irq.index = -1;
1462 msi_irq.pirq = -1;
1463 msi_irq.bus = pci_get_bus(dev) | (pci_get_domain(dev) << 16);
1464 msi_irq.devfn = (pci_get_slot(dev) << 3) | pci_get_function(dev);
1465 msi_irq.entry_nr = count;
1466
1467 ret = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &msi_irq);
1468 if (ret != 0)
1469 return (ret);
1470 if (count != msi_irq.entry_nr) {
1471 panic("unable to setup all requested MSI vectors "
1472 "(expected %d got %d)", count, msi_irq.entry_nr);
1473 }
1474
1475 mtx_lock(&xen_intr_isrc_lock);
1476 for (int i = 0; i < count; i++) {
1477 isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector + i);
1478 KASSERT(isrc != NULL,
1479 ("xen: unable to allocate isrc for interrupt"));
1480 isrc->xi_pirq = msi_irq.pirq + i;
1481 /* MSI interrupts are always edge triggered */
1482 isrc->xi_edgetrigger = 1;
1483 }
1484 mtx_unlock(&xen_intr_isrc_lock);
1485
1486 return (0);
1487 }
1488
1489 int
xen_release_msi(int vector)1490 xen_release_msi(int vector)
1491 {
1492 struct physdev_unmap_pirq unmap;
1493 struct xenisrc *isrc;
1494 int ret;
1495
1496 isrc = (struct xenisrc *)intr_lookup_source(vector);
1497 if (isrc == NULL)
1498 return (ENXIO);
1499
1500 unmap.pirq = isrc->xi_pirq;
1501 ret = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap);
1502 if (ret != 0)
1503 return (ret);
1504
1505 xen_intr_release_isrc(isrc);
1506
1507 return (0);
1508 }
1509
1510 int
xen_intr_describe(xen_intr_handle_t port_handle,const char * fmt,...)1511 xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...)
1512 {
1513 char descr[MAXCOMLEN + 1];
1514 struct xenisrc *isrc;
1515 va_list ap;
1516
1517 isrc = xen_intr_isrc(port_handle);
1518 if (isrc == NULL)
1519 return (EINVAL);
1520
1521 va_start(ap, fmt);
1522 vsnprintf(descr, sizeof(descr), fmt, ap);
1523 va_end(ap);
1524 return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr));
1525 }
1526
1527 void
xen_intr_unbind(xen_intr_handle_t * port_handlep)1528 xen_intr_unbind(xen_intr_handle_t *port_handlep)
1529 {
1530 struct xenisrc *isrc;
1531
1532 KASSERT(port_handlep != NULL,
1533 ("NULL xen_intr_handle_t passed to xen_intr_unbind"));
1534
1535 isrc = xen_intr_isrc(*port_handlep);
1536 *port_handlep = NULL;
1537 if (isrc == NULL)
1538 return;
1539
1540 mtx_lock(&xen_intr_isrc_lock);
1541 if (refcount_release(&isrc->xi_refcount) == 0) {
1542 mtx_unlock(&xen_intr_isrc_lock);
1543 return;
1544 }
1545 mtx_unlock(&xen_intr_isrc_lock);
1546
1547 if (isrc->xi_cookie != NULL)
1548 intr_remove_handler(isrc->xi_cookie);
1549 xen_intr_release_isrc(isrc);
1550 }
1551
1552 void
xen_intr_signal(xen_intr_handle_t handle)1553 xen_intr_signal(xen_intr_handle_t handle)
1554 {
1555 struct xenisrc *isrc;
1556
1557 isrc = xen_intr_isrc(handle);
1558 if (isrc != NULL) {
1559 KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT ||
1560 isrc->xi_type == EVTCHN_TYPE_IPI,
1561 ("evtchn_signal on something other than a local port"));
1562 struct evtchn_send send = { .port = isrc->xi_port };
1563 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
1564 }
1565 }
1566
1567 evtchn_port_t
xen_intr_port(xen_intr_handle_t handle)1568 xen_intr_port(xen_intr_handle_t handle)
1569 {
1570 struct xenisrc *isrc;
1571
1572 isrc = xen_intr_isrc(handle);
1573 if (isrc == NULL)
1574 return (0);
1575
1576 return (isrc->xi_port);
1577 }
1578
1579 int
xen_intr_add_handler(const char * name,driver_filter_t filter,driver_intr_t handler,void * arg,enum intr_type flags,xen_intr_handle_t handle)1580 xen_intr_add_handler(const char *name, driver_filter_t filter,
1581 driver_intr_t handler, void *arg, enum intr_type flags,
1582 xen_intr_handle_t handle)
1583 {
1584 struct xenisrc *isrc;
1585 int error;
1586
1587 isrc = xen_intr_isrc(handle);
1588 if (isrc == NULL || isrc->xi_cookie != NULL)
1589 return (EINVAL);
1590
1591 error = intr_add_handler(name, isrc->xi_vector,filter, handler, arg,
1592 flags|INTR_EXCL, &isrc->xi_cookie, 0);
1593 if (error != 0) {
1594 printf(
1595 "%s: xen_intr_add_handler: intr_add_handler failed: %d\n",
1596 name, error);
1597 }
1598
1599 return (error);
1600 }
1601
1602 int
xen_intr_get_evtchn_from_port(evtchn_port_t port,xen_intr_handle_t * handlep)1603 xen_intr_get_evtchn_from_port(evtchn_port_t port, xen_intr_handle_t *handlep)
1604 {
1605
1606 if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS)
1607 return (EINVAL);
1608
1609 if (handlep == NULL) {
1610 return (EINVAL);
1611 }
1612
1613 mtx_lock(&xen_intr_isrc_lock);
1614 if (xen_intr_port_to_isrc[port] == NULL) {
1615 mtx_unlock(&xen_intr_isrc_lock);
1616 return (EINVAL);
1617 }
1618 refcount_acquire(&xen_intr_port_to_isrc[port]->xi_refcount);
1619 mtx_unlock(&xen_intr_isrc_lock);
1620
1621 /* Assign the opaque handler (the event channel port) */
1622 *handlep = &xen_intr_port_to_isrc[port]->xi_vector;
1623
1624 return (0);
1625 }
1626
1627 #ifdef DDB
1628 static const char *
xen_intr_print_type(enum evtchn_type type)1629 xen_intr_print_type(enum evtchn_type type)
1630 {
1631 static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = {
1632 [EVTCHN_TYPE_UNBOUND] = "UNBOUND",
1633 [EVTCHN_TYPE_PIRQ] = "PIRQ",
1634 [EVTCHN_TYPE_VIRQ] = "VIRQ",
1635 [EVTCHN_TYPE_IPI] = "IPI",
1636 [EVTCHN_TYPE_PORT] = "PORT",
1637 };
1638
1639 if (type >= EVTCHN_TYPE_COUNT)
1640 return ("UNKNOWN");
1641
1642 return (evtchn_type_to_string[type]);
1643 }
1644
1645 static void
xen_intr_dump_port(struct xenisrc * isrc)1646 xen_intr_dump_port(struct xenisrc *isrc)
1647 {
1648 struct xen_intr_pcpu_data *pcpu;
1649 shared_info_t *s = HYPERVISOR_shared_info;
1650 int i;
1651
1652 db_printf("Port %d Type: %s\n",
1653 isrc->xi_port, xen_intr_print_type(isrc->xi_type));
1654 if (isrc->xi_type == EVTCHN_TYPE_PIRQ) {
1655 db_printf("\tPirq: %d ActiveHi: %d EdgeTrigger: %d "
1656 "NeedsEOI: %d\n",
1657 isrc->xi_pirq, isrc->xi_activehi, isrc->xi_edgetrigger,
1658 !!xen_test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map));
1659 }
1660 if (isrc->xi_type == EVTCHN_TYPE_VIRQ)
1661 db_printf("\tVirq: %d\n", isrc->xi_virq);
1662
1663 db_printf("\tMasked: %d Pending: %d\n",
1664 !!xen_test_bit(isrc->xi_port, &s->evtchn_mask[0]),
1665 !!xen_test_bit(isrc->xi_port, &s->evtchn_pending[0]));
1666
1667 db_printf("\tPer-CPU Masks: ");
1668 CPU_FOREACH(i) {
1669 pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu);
1670 db_printf("cpu#%d: %d ", i,
1671 !!xen_test_bit(isrc->xi_port, pcpu->evtchn_enabled));
1672 }
1673 db_printf("\n");
1674 }
1675
DB_SHOW_COMMAND(xen_evtchn,db_show_xen_evtchn)1676 DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn)
1677 {
1678 int i;
1679
1680 if (!xen_domain()) {
1681 db_printf("Only available on Xen guests\n");
1682 return;
1683 }
1684
1685 for (i = 0; i < NR_EVENT_CHANNELS; i++) {
1686 struct xenisrc *isrc;
1687
1688 isrc = xen_intr_port_to_isrc[i];
1689 if (isrc == NULL)
1690 continue;
1691
1692 xen_intr_dump_port(isrc);
1693 }
1694 }
1695 #endif /* DDB */
1696