1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2016, Anish Gupta ([email protected])
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/malloc.h>
38 #include <sys/pcpu.h>
39 #include <sys/rman.h>
40 #include <sys/smp.h>
41 #include <sys/sysctl.h>
42
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45
46 #include <dev/pci/pcivar.h>
47 #include <dev/pci/pcireg.h>
48
49 #include <machine/resource.h>
50 #include <machine/vmm.h>
51 #include <machine/pmap.h>
52 #include <machine/vmparam.h>
53 #include <machine/pci_cfgreg.h>
54
55 #include "ivhd_if.h"
56 #include "pcib_if.h"
57
58 #include "io/iommu.h"
59 #include "amdvi_priv.h"
60
61 SYSCTL_DECL(_hw_vmm);
62 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
63 NULL);
64
65 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
66 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
67
68 /* Print RID or device ID in PCI string format. */
69 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
70
71 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count);
72 static void amdvi_print_dev_cap(struct amdvi_softc *softc);
73
74 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
75
76 extern device_t *ivhd_devs;
77
78 extern int ivhd_count;
79 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
80 0, NULL);
81
82 static int amdvi_enable_user = 0;
83 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
84 &amdvi_enable_user, 0, NULL);
85 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
86
87 #ifdef AMDVI_ATS_ENABLE
88 /* XXX: ATS is not tested. */
89 static int amdvi_enable_iotlb = 1;
90 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
91 &amdvi_enable_iotlb, 0, NULL);
92 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
93 #endif
94
95 static int amdvi_host_ptp = 1; /* Use page tables for host. */
96 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
97 &amdvi_host_ptp, 0, NULL);
98 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
99
100 /* Page table level used <= supported by h/w[v1=7]. */
101 int amdvi_ptp_level = 4;
102 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
103 &amdvi_ptp_level, 0, NULL);
104 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
105
106 /* Disable fault event reporting. */
107 static int amdvi_disable_io_fault = 0;
108 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
109 &amdvi_disable_io_fault, 0, NULL);
110 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
111
112 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */
113 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
114 &amdvi_dom_id, 0, NULL);
115 /*
116 * Device table entry.
117 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
118 * = 256 * 2 * PAGE_SIZE.
119 */
120 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
121 CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
122 CTASSERT(sizeof(amdvi_dte) == 0x200000);
123
124 static SLIST_HEAD (, amdvi_domain) dom_head;
125
126 static inline uint32_t
amdvi_pci_read(struct amdvi_softc * softc,int off)127 amdvi_pci_read(struct amdvi_softc *softc, int off)
128 {
129
130 return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid),
131 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
132 off, 4));
133 }
134
135 #ifdef AMDVI_ATS_ENABLE
136 /* XXX: Should be in pci.c */
137 /*
138 * Check if device has ATS capability and its enabled.
139 * If ATS is absent or disabled, return (-1), otherwise ATS
140 * queue length.
141 */
142 static int
amdvi_find_ats_qlen(uint16_t devid)143 amdvi_find_ats_qlen(uint16_t devid)
144 {
145 device_t dev;
146 uint32_t off, cap;
147 int qlen = -1;
148
149 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
150 PCI_RID2FUNC(devid));
151
152 if (!dev) {
153 return (-1);
154 }
155 #define PCIM_ATS_EN BIT(31)
156
157 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
158 cap = pci_read_config(dev, off + 4, 4);
159 qlen = (cap & 0x1F);
160 qlen = qlen ? qlen : 32;
161 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
162 RID2PCI_STR(devid),
163 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
164 qlen);
165 qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
166 }
167
168 return (qlen);
169 }
170
171 /*
172 * Check if an endpoint device support device IOTLB or ATS.
173 */
174 static inline bool
amdvi_dev_support_iotlb(struct amdvi_softc * softc,uint16_t devid)175 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
176 {
177 struct ivhd_dev_cfg *cfg;
178 int qlen, i;
179 bool pci_ats, ivhd_ats;
180
181 qlen = amdvi_find_ats_qlen(devid);
182 if (qlen < 0)
183 return (false);
184
185 KASSERT(softc, ("softc is NULL"));
186 cfg = softc->dev_cfg;
187
188 ivhd_ats = false;
189 for (i = 0; i < softc->dev_cfg_cnt; i++) {
190 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
191 ivhd_ats = cfg->enable_ats;
192 break;
193 }
194 cfg++;
195 }
196
197 pci_ats = (qlen < 0) ? false : true;
198 if (pci_ats != ivhd_ats)
199 device_printf(softc->dev,
200 "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
201 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
202
203 /* Ignore IVRS setting and respect PCI setting. */
204 return (pci_ats);
205 }
206 #endif
207
208 /* Enable IOTLB support for IOMMU if its supported. */
209 static inline void
amdvi_hw_enable_iotlb(struct amdvi_softc * softc)210 amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
211 {
212 #ifndef AMDVI_ATS_ENABLE
213 softc->iotlb = false;
214 #else
215 bool supported;
216
217 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
218
219 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
220 if (!supported)
221 device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
222
223 if (supported && !amdvi_enable_iotlb) {
224 device_printf(softc->dev, "IOTLB disabled by user.\n");
225 supported = false;
226 }
227 } else
228 supported = false;
229
230 softc->iotlb = supported;
231
232 #endif
233 }
234
235 static int
amdvi_init_cmd(struct amdvi_softc * softc)236 amdvi_init_cmd(struct amdvi_softc *softc)
237 {
238 struct amdvi_ctrl *ctrl = softc->ctrl;
239
240 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */
241 softc->cmd_max = 1 << ctrl->cmd.len;
242
243 softc->cmd = malloc(sizeof(struct amdvi_cmd) *
244 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
245
246 if ((uintptr_t)softc->cmd & PAGE_MASK)
247 panic("AMDVi: Command buffer not aligned on page boundary.");
248
249 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
250 /*
251 * XXX: Reset the h/w pointers in case IOMMU is restarting,
252 * h/w doesn't clear these pointers based on empirical data.
253 */
254 ctrl->cmd_tail = 0;
255 ctrl->cmd_head = 0;
256
257 return (0);
258 }
259
260 /*
261 * Note: Update tail pointer after we have written the command since tail
262 * pointer update cause h/w to execute new commands, see section 3.3
263 * of AMD IOMMU spec ver 2.0.
264 */
265 /* Get the command tail pointer w/o updating it. */
266 static struct amdvi_cmd *
amdvi_get_cmd_tail(struct amdvi_softc * softc)267 amdvi_get_cmd_tail(struct amdvi_softc *softc)
268 {
269 struct amdvi_ctrl *ctrl;
270 struct amdvi_cmd *tail;
271
272 KASSERT(softc, ("softc is NULL"));
273 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
274
275 ctrl = softc->ctrl;
276 KASSERT(ctrl != NULL, ("ctrl is NULL"));
277
278 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
279 ctrl->cmd_tail);
280
281 return (tail);
282 }
283
284 /*
285 * Update the command tail pointer which will start command execution.
286 */
287 static void
amdvi_update_cmd_tail(struct amdvi_softc * softc)288 amdvi_update_cmd_tail(struct amdvi_softc *softc)
289 {
290 struct amdvi_ctrl *ctrl;
291 int size;
292
293 size = sizeof(struct amdvi_cmd);
294 KASSERT(softc->cmd != NULL, ("cmd is NULL"));
295
296 ctrl = softc->ctrl;
297 KASSERT(ctrl != NULL, ("ctrl is NULL"));
298
299 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
300 softc->total_cmd++;
301
302 #ifdef AMDVI_DEBUG_CMD
303 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
304 ctrl->cmd_tail,
305 ctrl->cmd_head);
306 #endif
307
308 }
309
310 /*
311 * Various commands supported by IOMMU.
312 */
313
314 /* Completion wait command. */
315 static void
amdvi_cmd_cmp(struct amdvi_softc * softc,const uint64_t data)316 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
317 {
318 struct amdvi_cmd *cmd;
319 uint64_t pa;
320
321 cmd = amdvi_get_cmd_tail(softc);
322 KASSERT(cmd != NULL, ("Cmd is NULL"));
323
324 pa = vtophys(&softc->cmp_data);
325 cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
326 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
327 cmd->word1 = (pa >> 32) & 0xFFFFF;
328 cmd->addr = data;
329
330 amdvi_update_cmd_tail(softc);
331 }
332
333 /* Invalidate device table entry. */
334 static void
amdvi_cmd_inv_dte(struct amdvi_softc * softc,uint16_t devid)335 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
336 {
337 struct amdvi_cmd *cmd;
338
339 cmd = amdvi_get_cmd_tail(softc);
340 KASSERT(cmd != NULL, ("Cmd is NULL"));
341 cmd->opcode = AMDVI_INVD_DTE_OPCODE;
342 cmd->word0 = devid;
343 amdvi_update_cmd_tail(softc);
344 #ifdef AMDVI_DEBUG_CMD
345 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
346 #endif
347 }
348
349 /* Invalidate IOMMU page, use for invalidation of domain. */
350 static void
amdvi_cmd_inv_iommu_pages(struct amdvi_softc * softc,uint16_t domain_id,uint64_t addr,bool guest_nested,bool pde,bool page)351 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
352 uint64_t addr, bool guest_nested,
353 bool pde, bool page)
354 {
355 struct amdvi_cmd *cmd;
356
357 cmd = amdvi_get_cmd_tail(softc);
358 KASSERT(cmd != NULL, ("Cmd is NULL"));
359
360 cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
361 cmd->word1 = domain_id;
362 /*
363 * Invalidate all addresses for this domain.
364 */
365 cmd->addr = addr;
366 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
367 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
368
369 amdvi_update_cmd_tail(softc);
370 }
371
372 #ifdef AMDVI_ATS_ENABLE
373 /* Invalidate device IOTLB. */
374 static void
amdvi_cmd_inv_iotlb(struct amdvi_softc * softc,uint16_t devid)375 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
376 {
377 struct amdvi_cmd *cmd;
378 int qlen;
379
380 if (!softc->iotlb)
381 return;
382
383 qlen = amdvi_find_ats_qlen(devid);
384 if (qlen < 0) {
385 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
386 qlen, RID2PCI_STR(devid));
387 }
388 cmd = amdvi_get_cmd_tail(softc);
389 KASSERT(cmd != NULL, ("Cmd is NULL"));
390
391 #ifdef AMDVI_DEBUG_CMD
392 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
393 " Qlen:%d\n", devid, qlen);
394 #endif
395 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
396 cmd->word0 = devid;
397 cmd->word1 = qlen;
398 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
399 AMDVI_INVD_IOTLB_S;
400 amdvi_update_cmd_tail(softc);
401 }
402 #endif
403
404 #ifdef notyet /* For Interrupt Remap. */
405 static void
amdvi_cmd_inv_intr_map(struct amdvi_softc * softc,uint16_t devid)406 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
407 uint16_t devid)
408 {
409 struct amdvi_cmd *cmd;
410
411 cmd = amdvi_get_cmd_tail(softc);
412 KASSERT(cmd != NULL, ("Cmd is NULL"));
413 cmd->opcode = AMDVI_INVD_INTR_OPCODE;
414 cmd->word0 = devid;
415 amdvi_update_cmd_tail(softc);
416 #ifdef AMDVI_DEBUG_CMD
417 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
418 #endif
419 }
420 #endif
421
422 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
423 static void
amdvi_inv_domain(struct amdvi_softc * softc,uint16_t domain_id)424 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
425 {
426 struct amdvi_cmd *cmd;
427
428 cmd = amdvi_get_cmd_tail(softc);
429 KASSERT(cmd != NULL, ("Cmd is NULL"));
430
431 /*
432 * See section 3.3.3 of IOMMU spec rev 2.0, software note
433 * for invalidating domain.
434 */
435 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
436 false, true, true);
437
438 #ifdef AMDVI_DEBUG_CMD
439 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
440
441 #endif
442 }
443
444 static bool
amdvi_cmp_wait(struct amdvi_softc * softc)445 amdvi_cmp_wait(struct amdvi_softc *softc)
446 {
447 struct amdvi_ctrl *ctrl;
448 const uint64_t VERIFY = 0xA5A5;
449 volatile uint64_t *read;
450 int i;
451 bool status;
452
453 ctrl = softc->ctrl;
454 read = &softc->cmp_data;
455 *read = 0;
456 amdvi_cmd_cmp(softc, VERIFY);
457 /* Wait for h/w to update completion data. */
458 for (i = 0; i < 100 && (*read != VERIFY); i++) {
459 DELAY(1000); /* 1 ms */
460 }
461 status = (VERIFY == softc->cmp_data) ? true : false;
462
463 #ifdef AMDVI_DEBUG_CMD
464 if (status)
465 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
466 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
467 ctrl->cmd_head, loop);
468 #endif
469 return (status);
470 }
471
472 static void
amdvi_wait(struct amdvi_softc * softc)473 amdvi_wait(struct amdvi_softc *softc)
474 {
475 struct amdvi_ctrl *ctrl;
476 int i;
477
478 KASSERT(softc, ("softc is NULL"));
479
480 ctrl = softc->ctrl;
481 KASSERT(ctrl != NULL, ("ctrl is NULL"));
482 /* Don't wait if h/w is not enabled. */
483 if ((ctrl->control & AMDVI_CTRL_EN) == 0)
484 return;
485
486 for (i = 0; i < 10; i++) {
487 if (amdvi_cmp_wait(softc))
488 return;
489 }
490
491 device_printf(softc->dev, "Error: completion failed"
492 " tail:0x%x, head:0x%x.\n",
493 ctrl->cmd_tail, ctrl->cmd_head);
494 /* Dump the last command. */
495 amdvi_dump_cmds(softc, 1);
496 }
497
498 static void
amdvi_dump_cmds(struct amdvi_softc * softc,int count)499 amdvi_dump_cmds(struct amdvi_softc *softc, int count)
500 {
501 struct amdvi_ctrl *ctrl;
502 struct amdvi_cmd *cmd;
503 int off, i;
504
505 ctrl = softc->ctrl;
506 device_printf(softc->dev, "Dump last %d command(s):\n", count);
507 /*
508 * If h/w is stuck in completion, it is the previous command,
509 * start dumping from previous command onward.
510 */
511 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
512 softc->cmd_max);
513 for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
514 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
515 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
516 " 0x%x 0x%lx\n", i, off, cmd->opcode,
517 cmd->word0, cmd->word1, cmd->addr);
518 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
519 }
520 }
521
522 static int
amdvi_init_event(struct amdvi_softc * softc)523 amdvi_init_event(struct amdvi_softc *softc)
524 {
525 struct amdvi_ctrl *ctrl;
526
527 ctrl = softc->ctrl;
528 ctrl->event.len = 8;
529 softc->event_max = 1 << ctrl->event.len;
530 softc->event = malloc(sizeof(struct amdvi_event) *
531 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
532 if ((uintptr_t)softc->event & PAGE_MASK) {
533 device_printf(softc->dev, "Event buffer not aligned on page.");
534 return (false);
535 }
536 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
537
538 /* Reset the pointers. */
539 ctrl->evt_head = 0;
540 ctrl->evt_tail = 0;
541
542 return (0);
543 }
544
545 static inline void
amdvi_decode_evt_flag(uint16_t flag)546 amdvi_decode_evt_flag(uint16_t flag)
547 {
548
549 flag &= AMDVI_EVENT_FLAG_MASK;
550 printf(" 0x%b]\n", flag,
551 "\020"
552 "\001GN"
553 "\002NX"
554 "\003US"
555 "\004I"
556 "\005PR"
557 "\006RW"
558 "\007PE"
559 "\010RZ"
560 "\011TR"
561 );
562 }
563
564 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
565 static inline void
amdvi_decode_evt_flag_type(uint8_t type)566 amdvi_decode_evt_flag_type(uint8_t type)
567 {
568
569 switch (AMDVI_EVENT_FLAG_TYPE(type)) {
570 case 0:
571 printf("RSVD\n");
572 break;
573 case 1:
574 printf("Master Abort\n");
575 break;
576 case 2:
577 printf("Target Abort\n");
578 break;
579 case 3:
580 printf("Data Err\n");
581 break;
582 default:
583 break;
584 }
585 }
586
587 static void
amdvi_decode_inv_dte_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)588 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
589 uint16_t flag)
590 {
591
592 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
593 " Addr:0x%lx",
594 devid, domid, addr);
595 amdvi_decode_evt_flag(flag);
596 }
597
598 static void
amdvi_decode_pf_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)599 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
600 uint16_t flag)
601 {
602
603 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
604 " Addr:0x%lx",
605 devid, domid, addr);
606 amdvi_decode_evt_flag(flag);
607 }
608
609 static void
amdvi_decode_dte_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)610 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
611 uint64_t addr, uint16_t flag)
612 {
613
614 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
615 " Addr:0x%lx", devid, domid, addr);
616 amdvi_decode_evt_flag(flag);
617 amdvi_decode_evt_flag_type(flag);
618 }
619
620 static void
amdvi_decode_page_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)621 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
622 uint16_t flag)
623 {
624
625 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
626 " Addr:0x%lx", devid, domid, addr);
627 amdvi_decode_evt_flag(flag);
628 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
629 }
630
631 static void
amdvi_decode_evt(struct amdvi_event * evt)632 amdvi_decode_evt(struct amdvi_event *evt)
633 {
634 struct amdvi_cmd *cmd;
635
636 switch (evt->opcode) {
637 case AMDVI_EVENT_INVALID_DTE:
638 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
639 evt->addr, evt->flag);
640 break;
641
642 case AMDVI_EVENT_PFAULT:
643 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
644 evt->addr, evt->flag);
645 break;
646
647 case AMDVI_EVENT_DTE_HW_ERROR:
648 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
649 evt->addr, evt->flag);
650 break;
651
652 case AMDVI_EVENT_PAGE_HW_ERROR:
653 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
654 evt->addr, evt->flag);
655 break;
656
657 case AMDVI_EVENT_ILLEGAL_CMD:
658 /* FALL THROUGH */
659 case AMDVI_EVENT_CMD_HW_ERROR:
660 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
661 "ILLEGAL CMD" : "CMD HW ERR");
662 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
663 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
664 cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
665 break;
666
667 case AMDVI_EVENT_IOTLB_TIMEOUT:
668 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
669 evt->devid, evt->addr);
670 break;
671
672 case AMDVI_EVENT_INVALID_DTE_REQ:
673 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
674 evt->devid, evt->addr, evt->flag >> 9,
675 (evt->flag >> 8) & 1);
676 break;
677
678 case AMDVI_EVENT_INVALID_PPR_REQ:
679 case AMDVI_EVENT_COUNTER_ZERO:
680 printf("AMD-Vi: v2 events.\n");
681 break;
682
683 default:
684 printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
685 }
686 }
687
688 static void
amdvi_print_events(struct amdvi_softc * softc)689 amdvi_print_events(struct amdvi_softc *softc)
690 {
691 struct amdvi_ctrl *ctrl;
692 struct amdvi_event *event;
693 int i, size;
694
695 ctrl = softc->ctrl;
696 size = sizeof(struct amdvi_event);
697 for (i = 0; i < softc->event_max; i++) {
698 event = &softc->event[ctrl->evt_head / size];
699 if (!event->opcode)
700 break;
701 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
702 i, ctrl->evt_head, ctrl->evt_tail);
703 amdvi_decode_evt(event);
704 ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
705 softc->event_max);
706 }
707 }
708
709 static int
amdvi_init_dte(struct amdvi_softc * softc)710 amdvi_init_dte(struct amdvi_softc *softc)
711 {
712 struct amdvi_ctrl *ctrl;
713
714 ctrl = softc->ctrl;
715 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
716 ctrl->dte.size = 0x1FF; /* 2MB device table. */
717
718 return (0);
719 }
720
721 /*
722 * Not all capabilities of IOMMU are available in ACPI IVHD flag
723 * or EFR entry, read directly from device.
724 */
725 static int
amdvi_print_pci_cap(device_t dev)726 amdvi_print_pci_cap(device_t dev)
727 {
728 struct amdvi_softc *softc;
729 uint32_t off, cap;
730
731 softc = device_get_softc(dev);
732 off = softc->cap_off;
733
734 /*
735 * Section 3.7.1 of IOMMU sepc rev 2.0.
736 * Read capability from device.
737 */
738 cap = amdvi_pci_read(softc, off);
739
740 /* Make sure capability type[18:16] is 3. */
741 KASSERT((((cap >> 16) & 0x7) == 0x3),
742 ("Not a IOMMU capability 0x%x@0x%x", cap, off));
743
744 softc->pci_cap = cap >> 24;
745 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
746 cap, off, softc->pci_cap,
747 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
748
749 return (0);
750 }
751
752 static void
amdvi_event_intr(void * arg)753 amdvi_event_intr(void *arg)
754 {
755 struct amdvi_softc *softc;
756 struct amdvi_ctrl *ctrl;
757
758 softc = (struct amdvi_softc *)arg;
759 ctrl = softc->ctrl;
760 device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
761 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
762 ctrl->status, ctrl->evt_head, ctrl->evt_tail);
763 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
764 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
765
766 amdvi_print_events(softc);
767 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
768 }
769
770 static void
amdvi_free_evt_intr_res(device_t dev)771 amdvi_free_evt_intr_res(device_t dev)
772 {
773
774 struct amdvi_softc *softc;
775 device_t mmio_dev;
776
777 softc = device_get_softc(dev);
778 mmio_dev = softc->pci_dev;
779
780 IVHD_TEARDOWN_INTR(mmio_dev);
781 }
782
783 static bool
amdvi_alloc_intr_resources(struct amdvi_softc * softc)784 amdvi_alloc_intr_resources(struct amdvi_softc *softc)
785 {
786 struct amdvi_ctrl *ctrl;
787 device_t dev, mmio_dev;
788 int err;
789
790 dev = softc->dev;
791 mmio_dev = softc->pci_dev;
792
793 /* Clear interrupt status bits. */
794 ctrl = softc->ctrl;
795 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
796
797 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault");
798 if (err)
799 device_printf(dev, "Interrupt setup failed on %s\n",
800 device_get_nameunit(mmio_dev));
801 return (err);
802 }
803
804 static void
amdvi_print_dev_cap(struct amdvi_softc * softc)805 amdvi_print_dev_cap(struct amdvi_softc *softc)
806 {
807 struct ivhd_dev_cfg *cfg;
808 int i;
809
810 cfg = softc->dev_cfg;
811 for (i = 0; i < softc->dev_cfg_cnt; i++) {
812 device_printf(softc->dev, "device [0x%x - 0x%x] "
813 "config:%b%s\n", cfg->start_id, cfg->end_id,
814 cfg->data,
815 "\020\001INIT\002ExtInt\003NMI"
816 "\007LINT0\010LINT1",
817 cfg->enable_ats ? "ATS enabled" : "");
818 cfg++;
819 }
820 }
821
822 static int
amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)823 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
824 {
825 struct amdvi_softc *softc;
826 int result, type, error = 0;
827
828 softc = (struct amdvi_softc *)arg1;
829 type = arg2;
830
831 switch (type) {
832 case 0:
833 result = softc->ctrl->cmd_head;
834 error = sysctl_handle_int(oidp, &result, 0,
835 req);
836 break;
837 case 1:
838 result = softc->ctrl->cmd_tail;
839 error = sysctl_handle_int(oidp, &result, 0,
840 req);
841 break;
842 case 2:
843 result = softc->ctrl->evt_head;
844 error = sysctl_handle_int(oidp, &result, 0,
845 req);
846 break;
847 case 3:
848 result = softc->ctrl->evt_tail;
849 error = sysctl_handle_int(oidp, &result, 0,
850 req);
851 break;
852
853 default:
854 device_printf(softc->dev, "Unknown sysctl:%d\n", type);
855 }
856
857 return (error);
858 }
859
860 static void
amdvi_add_sysctl(struct amdvi_softc * softc)861 amdvi_add_sysctl(struct amdvi_softc *softc)
862 {
863 struct sysctl_oid_list *child;
864 struct sysctl_ctx_list *ctx;
865 device_t dev;
866
867 dev = softc->dev;
868 ctx = device_get_sysctl_ctx(dev);
869 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
870
871 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
872 &softc->event_intr_cnt, "Event interrupt count");
873 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
874 &softc->total_cmd, "Command submitted count");
875 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
876 &softc->pci_rid, 0, "IOMMU RID");
877 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
878 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0,
879 amdvi_handle_sysctl, "IU", "Command head");
880 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
881 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1,
882 amdvi_handle_sysctl, "IU", "Command tail");
883 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
884 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2,
885 amdvi_handle_sysctl, "IU", "Command head");
886 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
887 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3,
888 amdvi_handle_sysctl, "IU", "Command tail");
889 }
890
891 int
amdvi_setup_hw(struct amdvi_softc * softc)892 amdvi_setup_hw(struct amdvi_softc *softc)
893 {
894 device_t dev;
895 int status;
896
897 dev = softc->dev;
898
899 amdvi_hw_enable_iotlb(softc);
900
901 amdvi_print_dev_cap(softc);
902
903 if ((status = amdvi_print_pci_cap(dev)) != 0) {
904 device_printf(dev, "PCI capability.\n");
905 return (status);
906 }
907 if ((status = amdvi_init_cmd(softc)) != 0) {
908 device_printf(dev, "Couldn't configure command buffer.\n");
909 return (status);
910 }
911 if ((status = amdvi_init_event(softc)) != 0) {
912 device_printf(dev, "Couldn't configure event buffer.\n");
913 return (status);
914 }
915 if ((status = amdvi_init_dte(softc)) != 0) {
916 device_printf(dev, "Couldn't configure device table.\n");
917 return (status);
918 }
919 if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
920 return (status);
921 }
922 amdvi_add_sysctl(softc);
923 return (0);
924 }
925
926 int
amdvi_teardown_hw(struct amdvi_softc * softc)927 amdvi_teardown_hw(struct amdvi_softc *softc)
928 {
929 device_t dev;
930
931 dev = softc->dev;
932
933 /*
934 * Called after disable, h/w is stopped by now, free all the resources.
935 */
936 amdvi_free_evt_intr_res(dev);
937
938 if (softc->cmd)
939 free(softc->cmd, M_AMDVI);
940
941 if (softc->event)
942 free(softc->event, M_AMDVI);
943
944 return (0);
945 }
946
947 /*********** bhyve interfaces *********************/
948 static int
amdvi_init(void)949 amdvi_init(void)
950 {
951 if (!ivhd_count) {
952 return (EIO);
953 }
954 if (!amdvi_enable_user && ivhd_count) {
955 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
956 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
957 ivhd_count);
958 return (EINVAL);
959 }
960 return (0);
961 }
962
963 static void
amdvi_cleanup(void)964 amdvi_cleanup(void)
965 {
966 /* Nothing. */
967 }
968
969 static uint16_t
amdvi_domainId(void)970 amdvi_domainId(void)
971 {
972
973 /*
974 * If we hit maximum domain limit, rollover leaving host
975 * domain(0).
976 * XXX: make sure that this domain is not used.
977 */
978 if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
979 amdvi_dom_id = 1;
980
981 return ((uint16_t)amdvi_dom_id++);
982 }
983
984 static void
amdvi_do_inv_domain(uint16_t domain_id,bool create)985 amdvi_do_inv_domain(uint16_t domain_id, bool create)
986 {
987 struct amdvi_softc *softc;
988 int i;
989
990 for (i = 0; i < ivhd_count; i++) {
991 softc = device_get_softc(ivhd_devs[i]);
992 KASSERT(softc, ("softc is NULL"));
993 /*
994 * If not present pages are cached, invalidate page after
995 * creating domain.
996 */
997 #if 0
998 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
999 continue;
1000 #endif
1001 amdvi_inv_domain(softc, domain_id);
1002 amdvi_wait(softc);
1003 }
1004 }
1005
1006 static void *
amdvi_create_domain(vm_paddr_t maxaddr)1007 amdvi_create_domain(vm_paddr_t maxaddr)
1008 {
1009 struct amdvi_domain *dom;
1010
1011 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
1012 dom->id = amdvi_domainId();
1013 //dom->maxaddr = maxaddr;
1014 #ifdef AMDVI_DEBUG_CMD
1015 printf("Created domain #%d\n", dom->id);
1016 #endif
1017 /*
1018 * Host domain(#0) don't create translation table.
1019 */
1020 if (dom->id || amdvi_host_ptp)
1021 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1022
1023 dom->ptp_level = amdvi_ptp_level;
1024
1025 amdvi_do_inv_domain(dom->id, true);
1026 SLIST_INSERT_HEAD(&dom_head, dom, next);
1027
1028 return (dom);
1029 }
1030
1031 static void
amdvi_free_ptp(uint64_t * ptp,int level)1032 amdvi_free_ptp(uint64_t *ptp, int level)
1033 {
1034 int i;
1035
1036 if (level < 1)
1037 return;
1038
1039 for (i = 0; i < NPTEPG ; i++) {
1040 if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
1041 continue;
1042 /* XXX: Add super-page or PTE mapping > 4KB. */
1043 #ifdef notyet
1044 /* Super-page mapping. */
1045 if (AMDVI_PD_SUPER(ptp[i]))
1046 continue;
1047 #endif
1048
1049 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
1050 & AMDVI_PT_MASK), level - 1);
1051 }
1052
1053 free(ptp, M_AMDVI);
1054 }
1055
1056 static void
amdvi_destroy_domain(void * arg)1057 amdvi_destroy_domain(void *arg)
1058 {
1059 struct amdvi_domain *domain;
1060
1061 domain = (struct amdvi_domain *)arg;
1062 KASSERT(domain, ("domain is NULL"));
1063 #ifdef AMDVI_DEBUG_CMD
1064 printf("Destroying domain %d\n", domain->id);
1065 #endif
1066 if (domain->ptp)
1067 amdvi_free_ptp(domain->ptp, domain->ptp_level);
1068
1069 amdvi_do_inv_domain(domain->id, false);
1070 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
1071 free(domain, M_AMDVI);
1072 }
1073
1074 static uint64_t
amdvi_set_pt(uint64_t * pt,int level,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t pg_size,bool create)1075 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
1076 vm_paddr_t hpa, uint64_t pg_size, bool create)
1077 {
1078 uint64_t *page, pa;
1079 int shift, index;
1080 const int PT_SHIFT = 9;
1081 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */
1082
1083 if (!pg_size)
1084 return (0);
1085
1086 if (hpa & (pg_size - 1)) {
1087 printf("HPA is not size aligned.\n");
1088 return (0);
1089 }
1090 if (gpa & (pg_size - 1)) {
1091 printf("HPA is not size aligned.\n");
1092 return (0);
1093 }
1094 shift = PML4SHIFT;
1095 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
1096 index = (gpa >> shift) & PT_INDEX_MASK;
1097
1098 if ((pt[index] == 0) && create) {
1099 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1100 pa = vtophys(page);
1101 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
1102 ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1103 }
1104 #ifdef AMDVI_DEBUG_PTE
1105 if ((gpa % 0x1000000) == 0)
1106 printf("[level%d, shift = %d]PTE:0x%lx\n",
1107 level, shift, pt[index]);
1108 #endif
1109 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK)
1110 pa = PTE2PA(pt[index]);
1111 pt = (uint64_t *)PHYS_TO_DMAP(pa);
1112 shift -= PT_SHIFT;
1113 level--;
1114 }
1115
1116 /* Leaf entry. */
1117 index = (gpa >> shift) & PT_INDEX_MASK;
1118
1119 if (create) {
1120 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
1121 } else
1122 pt[index] = 0;
1123
1124 #ifdef AMDVI_DEBUG_PTE
1125 if ((gpa % 0x1000000) == 0)
1126 printf("[Last level%d, shift = %d]PTE:0x%lx\n",
1127 level, shift, pt[index]);
1128 #endif
1129 return (1ULL << shift);
1130 }
1131
1132 static uint64_t
amdvi_update_mapping(struct amdvi_domain * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t size,bool create)1133 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
1134 vm_paddr_t hpa, uint64_t size, bool create)
1135 {
1136 uint64_t mapped, *ptp, len;
1137 int level;
1138
1139 KASSERT(domain, ("domain is NULL"));
1140 level = domain->ptp_level;
1141 KASSERT(level, ("Page table level is 0"));
1142
1143 ptp = domain->ptp;
1144 KASSERT(ptp, ("PTP is NULL"));
1145 mapped = 0;
1146 while (mapped < size) {
1147 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1148 PAGE_SIZE, create);
1149 if (!len) {
1150 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
1151 hpa, gpa);
1152 return (0);
1153 }
1154 mapped += len;
1155 }
1156
1157 return (mapped);
1158 }
1159
1160 static uint64_t
amdvi_create_mapping(void * arg,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)1161 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
1162 uint64_t len)
1163 {
1164 struct amdvi_domain *domain;
1165
1166 domain = (struct amdvi_domain *)arg;
1167
1168 if (domain->id && !domain->ptp) {
1169 printf("ptp is NULL");
1170 return (-1);
1171 }
1172
1173 /*
1174 * If host domain is created w/o page table, skip IOMMU page
1175 * table set-up.
1176 */
1177 if (domain->ptp)
1178 return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1179 else
1180 return (len);
1181 }
1182
1183 static uint64_t
amdvi_remove_mapping(void * arg,vm_paddr_t gpa,uint64_t len)1184 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1185 {
1186 struct amdvi_domain *domain;
1187
1188 domain = (struct amdvi_domain *)arg;
1189 /*
1190 * If host domain is created w/o page table, skip IOMMU page
1191 * table set-up.
1192 */
1193 if (domain->ptp)
1194 return (amdvi_update_mapping(domain, gpa, 0, len, false));
1195 return
1196 (len);
1197 }
1198
1199 static struct amdvi_softc *
amdvi_find_iommu(uint16_t devid)1200 amdvi_find_iommu(uint16_t devid)
1201 {
1202 struct amdvi_softc *softc;
1203 int i, j;
1204
1205 for (i = 0; i < ivhd_count; i++) {
1206 softc = device_get_softc(ivhd_devs[i]);
1207 for (j = 0; j < softc->dev_cfg_cnt; j++)
1208 if ((devid >= softc->dev_cfg[j].start_id) &&
1209 (devid <= softc->dev_cfg[j].end_id))
1210 return (softc);
1211 }
1212
1213 return (NULL);
1214 }
1215
1216 /*
1217 * Set-up device table entry.
1218 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1219 * be set concurrently, e.g. read and write bits.
1220 */
1221 static void
amdvi_set_dte(struct amdvi_domain * domain,struct amdvi_softc * softc,uint16_t devid,bool enable)1222 amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc,
1223 uint16_t devid, bool enable)
1224 {
1225 struct amdvi_dte* temp;
1226
1227 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
1228 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
1229
1230 temp = &amdvi_dte[devid];
1231
1232 #ifdef AMDVI_ATS_ENABLE
1233 /* If IOMMU and device support IOTLB, enable it. */
1234 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1235 temp->iotlb_enable = 1;
1236 #endif
1237
1238 /* Avoid duplicate I/O faults. */
1239 temp->sup_second_io_fault = 1;
1240 temp->sup_all_io_fault = amdvi_disable_io_fault;
1241
1242 temp->dt_valid = 1;
1243 temp->domain_id = domain->id;
1244
1245 if (enable) {
1246 if (domain->ptp) {
1247 temp->pt_base = vtophys(domain->ptp) >> 12;
1248 temp->pt_level = amdvi_ptp_level;
1249 }
1250 /*
1251 * XXX: Page table valid[TV] bit must be set even if host domain
1252 * page tables are not enabled.
1253 */
1254 temp->pt_valid = 1;
1255 temp->read_allow = 1;
1256 temp->write_allow = 1;
1257 }
1258 }
1259
1260 static void
amdvi_inv_device(struct amdvi_softc * softc,uint16_t devid)1261 amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid)
1262 {
1263 KASSERT(softc, ("softc is NULL"));
1264
1265 amdvi_cmd_inv_dte(softc, devid);
1266 #ifdef AMDVI_ATS_ENABLE
1267 if (amdvi_dev_support_iotlb(softc, devid))
1268 amdvi_cmd_inv_iotlb(softc, devid);
1269 #endif
1270 amdvi_wait(softc);
1271 }
1272
1273 static void
amdvi_add_device(void * arg,uint16_t devid)1274 amdvi_add_device(void *arg, uint16_t devid)
1275 {
1276 struct amdvi_domain *domain;
1277 struct amdvi_softc *softc;
1278
1279 domain = (struct amdvi_domain *)arg;
1280 KASSERT(domain != NULL, ("domain is NULL"));
1281 #ifdef AMDVI_DEBUG_CMD
1282 printf("Assigning device(%d.%d.%d) to domain:%d\n",
1283 RID2PCI_STR(devid), domain->id);
1284 #endif
1285 softc = amdvi_find_iommu(devid);
1286 if (softc == NULL)
1287 return;
1288 amdvi_set_dte(domain, softc, devid, true);
1289 amdvi_inv_device(softc, devid);
1290 }
1291
1292 static void
amdvi_remove_device(void * arg,uint16_t devid)1293 amdvi_remove_device(void *arg, uint16_t devid)
1294 {
1295 struct amdvi_domain *domain;
1296 struct amdvi_softc *softc;
1297
1298 domain = (struct amdvi_domain *)arg;
1299 #ifdef AMDVI_DEBUG_CMD
1300 printf("Remove device(0x%x) from domain:%d\n",
1301 devid, domain->id);
1302 #endif
1303 softc = amdvi_find_iommu(devid);
1304 if (softc == NULL)
1305 return;
1306 amdvi_set_dte(domain, softc, devid, false);
1307 amdvi_inv_device(softc, devid);
1308 }
1309
1310 static void
amdvi_enable(void)1311 amdvi_enable(void)
1312 {
1313 struct amdvi_ctrl *ctrl;
1314 struct amdvi_softc *softc;
1315 uint64_t val;
1316 int i;
1317
1318 for (i = 0; i < ivhd_count; i++) {
1319 softc = device_get_softc(ivhd_devs[i]);
1320 KASSERT(softc, ("softc is NULL\n"));
1321 ctrl = softc->ctrl;
1322 KASSERT(ctrl, ("ctrl is NULL\n"));
1323
1324 val = ( AMDVI_CTRL_EN |
1325 AMDVI_CTRL_CMD |
1326 AMDVI_CTRL_ELOG |
1327 AMDVI_CTRL_ELOGINT |
1328 AMDVI_CTRL_INV_TO_1S);
1329
1330 if (softc->ivhd_flag & IVHD_FLAG_COH)
1331 val |= AMDVI_CTRL_COH;
1332 if (softc->ivhd_flag & IVHD_FLAG_HTT)
1333 val |= AMDVI_CTRL_HTT;
1334 if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1335 val |= AMDVI_CTRL_RPPW;
1336 if (softc->ivhd_flag & IVHD_FLAG_PPW)
1337 val |= AMDVI_CTRL_PPW;
1338 if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1339 val |= AMDVI_CTRL_ISOC;
1340
1341 ctrl->control = val;
1342 }
1343 }
1344
1345 static void
amdvi_disable(void)1346 amdvi_disable(void)
1347 {
1348 struct amdvi_ctrl *ctrl;
1349 struct amdvi_softc *softc;
1350 int i;
1351
1352 for (i = 0; i < ivhd_count; i++) {
1353 softc = device_get_softc(ivhd_devs[i]);
1354 KASSERT(softc, ("softc is NULL\n"));
1355 ctrl = softc->ctrl;
1356 KASSERT(ctrl, ("ctrl is NULL\n"));
1357
1358 ctrl->control = 0;
1359 }
1360 }
1361
1362 static void
amdvi_invalidate_tlb(void * arg)1363 amdvi_invalidate_tlb(void *arg)
1364 {
1365 struct amdvi_domain *domain;
1366
1367 domain = (struct amdvi_domain *)arg;
1368 KASSERT(domain, ("domain is NULL"));
1369 amdvi_do_inv_domain(domain->id, false);
1370 }
1371
1372 const struct iommu_ops iommu_ops_amd = {
1373 .init = amdvi_init,
1374 .cleanup = amdvi_cleanup,
1375 .enable = amdvi_enable,
1376 .disable = amdvi_disable,
1377 .create_domain = amdvi_create_domain,
1378 .destroy_domain = amdvi_destroy_domain,
1379 .create_mapping = amdvi_create_mapping,
1380 .remove_mapping = amdvi_remove_mapping,
1381 .add_device = amdvi_add_device,
1382 .remove_device = amdvi_remove_device,
1383 .invalidate_tlb = amdvi_invalidate_tlb
1384 };
1385