1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2013 Tycho Nightingale <[email protected]>
5 * Copyright (c) 2013 Neel Natu <[email protected]>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include "opt_bhyve_snapshot.h"
36
37 #include <sys/param.h>
38 #include <sys/queue.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/systm.h>
42 #include <sys/kernel.h>
43 #include <sys/malloc.h>
44
45 #include <x86/apicreg.h>
46 #include <machine/vmm.h>
47 #include <machine/vmm_snapshot.h>
48
49 #include "vmm_ktr.h"
50 #include "vmm_lapic.h"
51 #include "vlapic.h"
52 #include "vioapic.h"
53
54 #define IOREGSEL 0x00
55 #define IOWIN 0x10
56
57 #define REDIR_ENTRIES 32
58 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
59
60 struct vioapic {
61 struct vm *vm;
62 struct mtx mtx;
63 uint32_t id;
64 uint32_t ioregsel;
65 struct {
66 uint64_t reg;
67 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
68 } rtbl[REDIR_ENTRIES];
69 };
70
71 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
72 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
73 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
74
75 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
76
77 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
78 VM_CTR1((vioapic)->vm, fmt, a1)
79
80 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
81 VM_CTR2((vioapic)->vm, fmt, a1, a2)
82
83 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
84 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
85
86 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
87 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
88
89 #ifdef KTR
90 static const char *
pinstate_str(bool asserted)91 pinstate_str(bool asserted)
92 {
93
94 if (asserted)
95 return ("asserted");
96 else
97 return ("deasserted");
98 }
99 #endif
100
101 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)102 vioapic_send_intr(struct vioapic *vioapic, int pin)
103 {
104 int vector, delmode;
105 uint32_t low, high, dest;
106 bool level, phys;
107
108 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
109 ("vioapic_set_pinstate: invalid pin number %d", pin));
110
111 KASSERT(VIOAPIC_LOCKED(vioapic),
112 ("vioapic_set_pinstate: vioapic is not locked"));
113
114 low = vioapic->rtbl[pin].reg;
115 high = vioapic->rtbl[pin].reg >> 32;
116
117 if ((low & IOART_INTMASK) == IOART_INTMSET) {
118 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
119 return;
120 }
121
122 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
123 delmode = low & IOART_DELMOD;
124 level = low & IOART_TRGRLVL ? true : false;
125 if (level)
126 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
127
128 vector = low & IOART_INTVEC;
129 dest = high >> APIC_ID_SHIFT;
130 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
131 }
132
133 static void
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)134 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
135 {
136 int oldcnt, newcnt;
137 bool needintr;
138
139 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
140 ("vioapic_set_pinstate: invalid pin number %d", pin));
141
142 KASSERT(VIOAPIC_LOCKED(vioapic),
143 ("vioapic_set_pinstate: vioapic is not locked"));
144
145 oldcnt = vioapic->rtbl[pin].acnt;
146 if (newstate)
147 vioapic->rtbl[pin].acnt++;
148 else
149 vioapic->rtbl[pin].acnt--;
150 newcnt = vioapic->rtbl[pin].acnt;
151
152 if (newcnt < 0) {
153 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
154 pin, newcnt);
155 }
156
157 needintr = false;
158 if (oldcnt == 0 && newcnt == 1) {
159 needintr = true;
160 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
161 } else if (oldcnt == 1 && newcnt == 0) {
162 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
163 } else {
164 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
165 pin, pinstate_str(newstate), newcnt);
166 }
167
168 if (needintr)
169 vioapic_send_intr(vioapic, pin);
170 }
171
172 enum irqstate {
173 IRQSTATE_ASSERT,
174 IRQSTATE_DEASSERT,
175 IRQSTATE_PULSE
176 };
177
178 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)179 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
180 {
181 struct vioapic *vioapic;
182
183 if (irq < 0 || irq >= REDIR_ENTRIES)
184 return (EINVAL);
185
186 vioapic = vm_ioapic(vm);
187
188 VIOAPIC_LOCK(vioapic);
189 switch (irqstate) {
190 case IRQSTATE_ASSERT:
191 vioapic_set_pinstate(vioapic, irq, true);
192 break;
193 case IRQSTATE_DEASSERT:
194 vioapic_set_pinstate(vioapic, irq, false);
195 break;
196 case IRQSTATE_PULSE:
197 vioapic_set_pinstate(vioapic, irq, true);
198 vioapic_set_pinstate(vioapic, irq, false);
199 break;
200 default:
201 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
202 }
203 VIOAPIC_UNLOCK(vioapic);
204
205 return (0);
206 }
207
208 int
vioapic_assert_irq(struct vm * vm,int irq)209 vioapic_assert_irq(struct vm *vm, int irq)
210 {
211
212 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
213 }
214
215 int
vioapic_deassert_irq(struct vm * vm,int irq)216 vioapic_deassert_irq(struct vm *vm, int irq)
217 {
218
219 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
220 }
221
222 int
vioapic_pulse_irq(struct vm * vm,int irq)223 vioapic_pulse_irq(struct vm *vm, int irq)
224 {
225
226 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
227 }
228
229 /*
230 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
231 * configuration.
232 */
233 static void
vioapic_update_tmr(struct vm * vm,int vcpuid,void * arg)234 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg)
235 {
236 struct vioapic *vioapic;
237 struct vlapic *vlapic;
238 uint32_t low, high, dest;
239 int delmode, pin, vector;
240 bool level, phys;
241
242 vlapic = vm_lapic(vm, vcpuid);
243 vioapic = vm_ioapic(vm);
244
245 VIOAPIC_LOCK(vioapic);
246 /*
247 * Reset all vectors to be edge-triggered.
248 */
249 vlapic_reset_tmr(vlapic);
250 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
251 low = vioapic->rtbl[pin].reg;
252 high = vioapic->rtbl[pin].reg >> 32;
253
254 level = low & IOART_TRGRLVL ? true : false;
255 if (!level)
256 continue;
257
258 /*
259 * For a level-triggered 'pin' let the vlapic figure out if
260 * an assertion on this 'pin' would result in an interrupt
261 * being delivered to it. If yes, then it will modify the
262 * TMR bit associated with this vector to level-triggered.
263 */
264 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
265 delmode = low & IOART_DELMOD;
266 vector = low & IOART_INTVEC;
267 dest = high >> APIC_ID_SHIFT;
268 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
269 }
270 VIOAPIC_UNLOCK(vioapic);
271 }
272
273 static uint32_t
vioapic_read(struct vioapic * vioapic,int vcpuid,uint32_t addr)274 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
275 {
276 int regnum, pin, rshift;
277
278 regnum = addr & 0xff;
279 switch (regnum) {
280 case IOAPIC_ID:
281 return (vioapic->id);
282 break;
283 case IOAPIC_VER:
284 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
285 break;
286 case IOAPIC_ARB:
287 return (vioapic->id);
288 break;
289 default:
290 break;
291 }
292
293 /* redirection table entries */
294 if (regnum >= IOAPIC_REDTBL &&
295 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
296 pin = (regnum - IOAPIC_REDTBL) / 2;
297 if ((regnum - IOAPIC_REDTBL) % 2)
298 rshift = 32;
299 else
300 rshift = 0;
301
302 return (vioapic->rtbl[pin].reg >> rshift);
303 }
304
305 return (0);
306 }
307
308 static void
vioapic_write(struct vioapic * vioapic,int vcpuid,uint32_t addr,uint32_t data)309 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
310 {
311 uint64_t data64, mask64;
312 uint64_t last, changed;
313 int regnum, pin, lshift;
314 cpuset_t allvcpus;
315
316 regnum = addr & 0xff;
317 switch (regnum) {
318 case IOAPIC_ID:
319 vioapic->id = data & APIC_ID_MASK;
320 break;
321 case IOAPIC_VER:
322 case IOAPIC_ARB:
323 /* readonly */
324 break;
325 default:
326 break;
327 }
328
329 /* redirection table entries */
330 if (regnum >= IOAPIC_REDTBL &&
331 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
332 pin = (regnum - IOAPIC_REDTBL) / 2;
333 if ((regnum - IOAPIC_REDTBL) % 2)
334 lshift = 32;
335 else
336 lshift = 0;
337
338 last = vioapic->rtbl[pin].reg;
339
340 data64 = (uint64_t)data << lshift;
341 mask64 = (uint64_t)0xffffffff << lshift;
342 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
343 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
344
345 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
346 pin, vioapic->rtbl[pin].reg);
347
348 /*
349 * If any fields in the redirection table entry (except mask
350 * or polarity) have changed then rendezvous all the vcpus
351 * to update their vlapic trigger-mode registers.
352 */
353 changed = last ^ vioapic->rtbl[pin].reg;
354 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
355 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
356 "vlapic trigger-mode register", pin);
357 VIOAPIC_UNLOCK(vioapic);
358 allvcpus = vm_active_cpus(vioapic->vm);
359 (void)vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
360 vioapic_update_tmr, NULL);
361 VIOAPIC_LOCK(vioapic);
362 }
363
364 /*
365 * Generate an interrupt if the following conditions are met:
366 * - pin is not masked
367 * - previous interrupt has been EOIed
368 * - pin level is asserted
369 */
370 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR &&
371 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 &&
372 (vioapic->rtbl[pin].acnt > 0)) {
373 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
374 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
375 vioapic_send_intr(vioapic, pin);
376 }
377 }
378 }
379
380 static int
vioapic_mmio_rw(struct vioapic * vioapic,int vcpuid,uint64_t gpa,uint64_t * data,int size,bool doread)381 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
382 uint64_t *data, int size, bool doread)
383 {
384 uint64_t offset;
385
386 offset = gpa - VIOAPIC_BASE;
387
388 /*
389 * The IOAPIC specification allows 32-bit wide accesses to the
390 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
391 */
392 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
393 if (doread)
394 *data = 0;
395 return (0);
396 }
397
398 VIOAPIC_LOCK(vioapic);
399 if (offset == IOREGSEL) {
400 if (doread)
401 *data = vioapic->ioregsel;
402 else
403 vioapic->ioregsel = *data;
404 } else {
405 if (doread) {
406 *data = vioapic_read(vioapic, vcpuid,
407 vioapic->ioregsel);
408 } else {
409 vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
410 *data);
411 }
412 }
413 VIOAPIC_UNLOCK(vioapic);
414
415 return (0);
416 }
417
418 int
vioapic_mmio_read(void * vm,int vcpuid,uint64_t gpa,uint64_t * rval,int size,void * arg)419 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
420 int size, void *arg)
421 {
422 int error;
423 struct vioapic *vioapic;
424
425 vioapic = vm_ioapic(vm);
426 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
427 return (error);
428 }
429
430 int
vioapic_mmio_write(void * vm,int vcpuid,uint64_t gpa,uint64_t wval,int size,void * arg)431 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval,
432 int size, void *arg)
433 {
434 int error;
435 struct vioapic *vioapic;
436
437 vioapic = vm_ioapic(vm);
438 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
439 return (error);
440 }
441
442 void
vioapic_process_eoi(struct vm * vm,int vcpuid,int vector)443 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
444 {
445 struct vioapic *vioapic;
446 int pin;
447
448 KASSERT(vector >= 0 && vector < 256,
449 ("vioapic_process_eoi: invalid vector %d", vector));
450
451 vioapic = vm_ioapic(vm);
452 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
453
454 /*
455 * XXX keep track of the pins associated with this vector instead
456 * of iterating on every single pin each time.
457 */
458 VIOAPIC_LOCK(vioapic);
459 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
460 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
461 continue;
462 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
463 continue;
464 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
465 if (vioapic->rtbl[pin].acnt > 0) {
466 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
467 "acnt %d", pin, vioapic->rtbl[pin].acnt);
468 vioapic_send_intr(vioapic, pin);
469 }
470 }
471 VIOAPIC_UNLOCK(vioapic);
472 }
473
474 struct vioapic *
vioapic_init(struct vm * vm)475 vioapic_init(struct vm *vm)
476 {
477 int i;
478 struct vioapic *vioapic;
479
480 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
481
482 vioapic->vm = vm;
483 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
484
485 /* Initialize all redirection entries to mask all interrupts */
486 for (i = 0; i < REDIR_ENTRIES; i++)
487 vioapic->rtbl[i].reg = 0x0001000000010000UL;
488
489 return (vioapic);
490 }
491
492 void
vioapic_cleanup(struct vioapic * vioapic)493 vioapic_cleanup(struct vioapic *vioapic)
494 {
495
496 free(vioapic, M_VIOAPIC);
497 }
498
499 int
vioapic_pincount(struct vm * vm)500 vioapic_pincount(struct vm *vm)
501 {
502
503 return (REDIR_ENTRIES);
504 }
505
506 #ifdef BHYVE_SNAPSHOT
507 int
vioapic_snapshot(struct vioapic * vioapic,struct vm_snapshot_meta * meta)508 vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
509 {
510 int ret;
511 int i;
512
513 SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
514
515 for (i = 0; i < nitems(vioapic->rtbl); i++) {
516 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
517 SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
518 }
519
520 done:
521 return (ret);
522 }
523 #endif
524