1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <[email protected]>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/malloc.h>
35 #include <sys/bus.h>
36 #include <sys/interrupt.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/memdesc.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 #include <sys/rwlock.h>
45 #include <sys/rman.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <vm/vm.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57 #include <vm/vm_map.h>
58 #include <contrib/dev/acpica/include/acpi.h>
59 #include <contrib/dev/acpica/include/accommon.h>
60 #include <dev/pci/pcireg.h>
61 #include <dev/pci/pcivar.h>
62 #include <machine/atomic.h>
63 #include <machine/bus.h>
64 #include <machine/md_var.h>
65 #include <machine/specialreg.h>
66 #include <x86/include/busdma_impl.h>
67 #include <dev/iommu/busdma_iommu.h>
68 #include <x86/iommu/intel_reg.h>
69 #include <x86/iommu/x86_iommu.h>
70 #include <x86/iommu/intel_dmar.h>
71
72 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
73 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain");
74
75 static void dmar_unref_domain_locked(struct dmar_unit *dmar,
76 struct dmar_domain *domain);
77 static void dmar_domain_destroy(struct dmar_domain *domain);
78
79 static void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
80 static void dmar_free_ctx(struct dmar_ctx *ctx);
81
82 static void
dmar_ensure_ctx_page(struct dmar_unit * dmar,int bus)83 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
84 {
85 struct sf_buf *sf;
86 dmar_root_entry_t *re;
87 vm_page_t ctxm;
88
89 /*
90 * Allocated context page must be linked.
91 */
92 ctxm = iommu_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC);
93 if (ctxm != NULL)
94 return;
95
96 /*
97 * Page not present, allocate and link. Note that other
98 * thread might execute this sequence in parallel. This
99 * should be safe, because the context entries written by both
100 * threads are equal.
101 */
102 TD_PREP_PINNED_ASSERT;
103 ctxm = iommu_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO |
104 IOMMU_PGF_WAITOK);
105 re = iommu_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf);
106 re += bus;
107 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK &
108 VM_PAGE_TO_PHYS(ctxm)));
109 dmar_flush_root_to_ram(dmar, re);
110 iommu_unmap_pgtbl(sf);
111 TD_PINNED_ASSERT;
112 }
113
114 static dmar_ctx_entry_t *
dmar_map_ctx_entry(struct dmar_ctx * ctx,struct sf_buf ** sfp)115 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
116 {
117 struct dmar_unit *dmar;
118 dmar_ctx_entry_t *ctxp;
119
120 dmar = CTX2DMAR(ctx);
121
122 ctxp = iommu_map_pgtbl(dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->context.rid),
123 IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp);
124 ctxp += ctx->context.rid & 0xff;
125 return (ctxp);
126 }
127
128 static void
ctx_id_entry_init_one(dmar_ctx_entry_t * ctxp,struct dmar_domain * domain,vm_page_t ctx_root)129 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain,
130 vm_page_t ctx_root)
131 {
132 /*
133 * For update due to move, the store is not atomic. It is
134 * possible that DMAR read upper doubleword, while low
135 * doubleword is not yet updated. The domain id is stored in
136 * the upper doubleword, while the table pointer in the lower.
137 *
138 * There is no good solution, for the same reason it is wrong
139 * to clear P bit in the ctx entry for update.
140 */
141 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) |
142 domain->awlvl);
143 if (ctx_root == NULL) {
144 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P);
145 } else {
146 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR |
147 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) |
148 DMAR_CTX1_P);
149 }
150 }
151
152 static void
ctx_id_entry_init(struct dmar_ctx * ctx,dmar_ctx_entry_t * ctxp,bool move,int busno)153 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move,
154 int busno)
155 {
156 struct dmar_unit *unit;
157 struct dmar_domain *domain;
158 vm_page_t ctx_root;
159 int i;
160
161 domain = CTX2DOM(ctx);
162 unit = DOM2DMAR(domain);
163 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0),
164 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
165 unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner),
166 pci_get_function(ctx->context.tag->owner),
167 ctxp->ctx1, ctxp->ctx2));
168
169 if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 &&
170 (unit->hw_ecap & DMAR_ECAP_PT) != 0) {
171 KASSERT(domain->pgtbl_obj == NULL,
172 ("ctx %p non-null pgtbl_obj", ctx));
173 ctx_root = NULL;
174 } else {
175 ctx_root = iommu_pgalloc(domain->pgtbl_obj, 0,
176 IOMMU_PGF_NOALLOC);
177 }
178
179 if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) {
180 MPASS(!move);
181 for (i = 0; i <= PCI_BUSMAX; i++) {
182 ctx_id_entry_init_one(&ctxp[i], domain, ctx_root);
183 }
184 } else {
185 ctx_id_entry_init_one(ctxp, domain, ctx_root);
186 }
187 dmar_flush_ctx_to_ram(unit, ctxp);
188 }
189
190 static int
dmar_flush_for_ctx_entry(struct dmar_unit * dmar,bool force)191 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force)
192 {
193 int error;
194
195 /*
196 * If dmar declares Caching Mode as Set, follow 11.5 "Caching
197 * Mode Consideration" and do the (global) invalidation of the
198 * negative TLB entries.
199 */
200 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force)
201 return (0);
202 if (dmar->qi_enabled) {
203 dmar_qi_invalidate_ctx_glob_locked(dmar);
204 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)
205 dmar_qi_invalidate_iotlb_glob_locked(dmar);
206 return (0);
207 }
208 error = dmar_inv_ctx_glob(dmar);
209 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force))
210 error = dmar_inv_iotlb_glob(dmar);
211 return (error);
212 }
213
214 static int
domain_init_rmrr(struct dmar_domain * domain,device_t dev,int bus,int slot,int func,int dev_domain,int dev_busno,const void * dev_path,int dev_path_len)215 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus,
216 int slot, int func, int dev_domain, int dev_busno,
217 const void *dev_path, int dev_path_len)
218 {
219 struct iommu_map_entries_tailq rmrr_entries;
220 struct iommu_map_entry *entry, *entry1;
221 vm_page_t *ma;
222 iommu_gaddr_t start, end;
223 vm_pindex_t size, i;
224 int error, error1;
225
226 if (!dmar_rmrr_enable)
227 return (0);
228
229 error = 0;
230 TAILQ_INIT(&rmrr_entries);
231 dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path,
232 dev_path_len, &rmrr_entries);
233 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, dmamap_link, entry1) {
234 /*
235 * VT-d specification requires that the start of an
236 * RMRR entry is 4k-aligned. Buggy BIOSes put
237 * anything into the start and end fields. Truncate
238 * and round as neccesary.
239 *
240 * We also allow the overlapping RMRR entries, see
241 * iommu_gas_alloc_region().
242 */
243 start = entry->start;
244 end = entry->end;
245 if (bootverbose)
246 printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n",
247 domain->iodom.iommu->unit, bus, slot, func,
248 (uintmax_t)start, (uintmax_t)end);
249 entry->start = trunc_page(start);
250 entry->end = round_page(end);
251 if (entry->start == entry->end) {
252 /* Workaround for some AMI (?) BIOSes */
253 if (bootverbose) {
254 if (dev != NULL)
255 device_printf(dev, "");
256 printf("pci%d:%d:%d ", bus, slot, func);
257 printf("BIOS bug: dmar%d RMRR "
258 "region (%jx, %jx) corrected\n",
259 domain->iodom.iommu->unit, start, end);
260 }
261 entry->end += IOMMU_PAGE_SIZE * 0x20;
262 }
263 size = OFF_TO_IDX(entry->end - entry->start);
264 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK);
265 for (i = 0; i < size; i++) {
266 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
267 VM_MEMATTR_DEFAULT);
268 }
269 error1 = iommu_gas_map_region(DOM2IODOM(domain), entry,
270 IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE,
271 IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma);
272 /*
273 * Non-failed RMRR entries are owned by context rb
274 * tree. Get rid of the failed entry, but do not stop
275 * the loop. Rest of the parsed RMRR entries are
276 * loaded and removed on the context destruction.
277 */
278 if (error1 == 0 && entry->end != entry->start) {
279 IOMMU_LOCK(domain->iodom.iommu);
280 domain->refs++; /* XXXKIB prevent free */
281 domain->iodom.flags |= IOMMU_DOMAIN_RMRR;
282 IOMMU_UNLOCK(domain->iodom.iommu);
283 } else {
284 if (error1 != 0) {
285 if (dev != NULL)
286 device_printf(dev, "");
287 printf("pci%d:%d:%d ", bus, slot, func);
288 printf(
289 "dmar%d failed to map RMRR region (%jx, %jx) %d\n",
290 domain->iodom.iommu->unit, start, end,
291 error1);
292 error = error1;
293 }
294 TAILQ_REMOVE(&rmrr_entries, entry, dmamap_link);
295 iommu_gas_free_entry(entry);
296 }
297 for (i = 0; i < size; i++)
298 vm_page_putfake(ma[i]);
299 free(ma, M_TEMP);
300 }
301 return (error);
302 }
303
304 /*
305 * PCI memory address space is shared between memory-mapped devices (MMIO) and
306 * host memory (which may be remapped by an IOMMU). Device accesses to an
307 * address within a memory aperture in a PCIe root port will be treated as
308 * peer-to-peer and not forwarded to an IOMMU. To avoid this, reserve the
309 * address space of the root port's memory apertures in the address space used
310 * by the IOMMU for remapping.
311 */
312 static int
dmar_reserve_pci_regions(struct dmar_domain * domain,device_t dev)313 dmar_reserve_pci_regions(struct dmar_domain *domain, device_t dev)
314 {
315 struct iommu_domain *iodom;
316 device_t root;
317 uint32_t val;
318 uint64_t base, limit;
319 int error;
320
321 iodom = DOM2IODOM(domain);
322
323 root = pci_find_pcie_root_port(dev);
324 if (root == NULL)
325 return (0);
326
327 /* Disable downstream memory */
328 base = PCI_PPBMEMBASE(0, pci_read_config(root, PCIR_MEMBASE_1, 2));
329 limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_MEMLIMIT_1, 2));
330 error = iommu_gas_reserve_region_extend(iodom, base, limit + 1);
331 if (bootverbose || error != 0)
332 device_printf(dev, "DMAR reserve [%#jx-%#jx] (error %d)\n",
333 base, limit + 1, error);
334 if (error != 0)
335 return (error);
336
337 /* Disable downstream prefetchable memory */
338 val = pci_read_config(root, PCIR_PMBASEL_1, 2);
339 if (val != 0 || pci_read_config(root, PCIR_PMLIMITL_1, 2) != 0) {
340 if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) {
341 base = PCI_PPBMEMBASE(
342 pci_read_config(root, PCIR_PMBASEH_1, 4),
343 val);
344 limit = PCI_PPBMEMLIMIT(
345 pci_read_config(root, PCIR_PMLIMITH_1, 4),
346 pci_read_config(root, PCIR_PMLIMITL_1, 2));
347 } else {
348 base = PCI_PPBMEMBASE(0, val);
349 limit = PCI_PPBMEMLIMIT(0,
350 pci_read_config(root, PCIR_PMLIMITL_1, 2));
351 }
352 error = iommu_gas_reserve_region_extend(iodom, base,
353 limit + 1);
354 if (bootverbose || error != 0)
355 device_printf(dev, "DMAR reserve [%#jx-%#jx] "
356 "(error %d)\n", base, limit + 1, error);
357 if (error != 0)
358 return (error);
359 }
360
361 return (error);
362 }
363
364 static struct dmar_domain *
dmar_domain_alloc(struct dmar_unit * dmar,bool id_mapped)365 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped)
366 {
367 struct iommu_domain *iodom;
368 struct iommu_unit *unit;
369 struct dmar_domain *domain;
370 int error, id, mgaw;
371
372 id = alloc_unr(dmar->domids);
373 if (id == -1)
374 return (NULL);
375 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO);
376 iodom = DOM2IODOM(domain);
377 unit = DMAR2IOMMU(dmar);
378 domain->domain = id;
379 LIST_INIT(&iodom->contexts);
380 iommu_domain_init(unit, iodom, &dmar_domain_map_ops);
381
382 domain->dmar = dmar;
383
384 /*
385 * For now, use the maximal usable physical address of the
386 * installed memory to calculate the mgaw on id_mapped domain.
387 * It is useful for the identity mapping, and less so for the
388 * virtualized bus address space.
389 */
390 domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR;
391 mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped);
392 error = domain_set_agaw(domain, mgaw);
393 if (error != 0)
394 goto fail;
395 if (!id_mapped)
396 /* Use all supported address space for remapping. */
397 domain->iodom.end = 1ULL << (domain->agaw - 1);
398
399 iommu_gas_init_domain(DOM2IODOM(domain));
400
401 if (id_mapped) {
402 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) {
403 domain->pgtbl_obj = dmar_get_idmap_pgtbl(domain,
404 domain->iodom.end);
405 }
406 domain->iodom.flags |= IOMMU_DOMAIN_IDMAP;
407 } else {
408 error = dmar_domain_alloc_pgtbl(domain);
409 if (error != 0)
410 goto fail;
411 /* Disable local apic region access */
412 error = iommu_gas_reserve_region(iodom, 0xfee00000,
413 0xfeefffff + 1, &iodom->msi_entry);
414 if (error != 0)
415 goto fail;
416 }
417 return (domain);
418
419 fail:
420 dmar_domain_destroy(domain);
421 return (NULL);
422 }
423
424 static struct dmar_ctx *
dmar_ctx_alloc(struct dmar_domain * domain,uint16_t rid)425 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid)
426 {
427 struct dmar_ctx *ctx;
428
429 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO);
430 ctx->context.domain = DOM2IODOM(domain);
431 ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu),
432 M_DMAR_CTX, M_WAITOK | M_ZERO);
433 ctx->context.rid = rid;
434 ctx->context.refs = 1;
435 return (ctx);
436 }
437
438 static void
dmar_ctx_link(struct dmar_ctx * ctx)439 dmar_ctx_link(struct dmar_ctx *ctx)
440 {
441 struct dmar_domain *domain;
442
443 domain = CTX2DOM(ctx);
444 IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
445 KASSERT(domain->refs >= domain->ctx_cnt,
446 ("dom %p ref underflow %d %d", domain, domain->refs,
447 domain->ctx_cnt));
448 domain->refs++;
449 domain->ctx_cnt++;
450 LIST_INSERT_HEAD(&domain->iodom.contexts, &ctx->context, link);
451 }
452
453 static void
dmar_ctx_unlink(struct dmar_ctx * ctx)454 dmar_ctx_unlink(struct dmar_ctx *ctx)
455 {
456 struct dmar_domain *domain;
457
458 domain = CTX2DOM(ctx);
459 IOMMU_ASSERT_LOCKED(domain->iodom.iommu);
460 KASSERT(domain->refs > 0,
461 ("domain %p ctx dtr refs %d", domain, domain->refs));
462 KASSERT(domain->ctx_cnt >= domain->refs,
463 ("domain %p ctx dtr refs %d ctx_cnt %d", domain,
464 domain->refs, domain->ctx_cnt));
465 domain->refs--;
466 domain->ctx_cnt--;
467 LIST_REMOVE(&ctx->context, link);
468 }
469
470 static void
dmar_domain_destroy(struct dmar_domain * domain)471 dmar_domain_destroy(struct dmar_domain *domain)
472 {
473 struct iommu_domain *iodom;
474 struct dmar_unit *dmar;
475
476 iodom = DOM2IODOM(domain);
477
478 KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries),
479 ("unfinished unloads %p", domain));
480 KASSERT(LIST_EMPTY(&iodom->contexts),
481 ("destroying dom %p with contexts", domain));
482 KASSERT(domain->ctx_cnt == 0,
483 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt));
484 KASSERT(domain->refs == 0,
485 ("destroying dom %p with refs %d", domain, domain->refs));
486 if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
487 DMAR_DOMAIN_LOCK(domain);
488 iommu_gas_fini_domain(iodom);
489 DMAR_DOMAIN_UNLOCK(domain);
490 }
491 if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) {
492 if (domain->pgtbl_obj != NULL)
493 DMAR_DOMAIN_PGLOCK(domain);
494 dmar_domain_free_pgtbl(domain);
495 }
496 iommu_domain_fini(iodom);
497 dmar = DOM2DMAR(domain);
498 free_unr(dmar->domids, domain->domain);
499 free(domain, M_DMAR_DOMAIN);
500 }
501
502 static struct dmar_ctx *
dmar_get_ctx_for_dev1(struct dmar_unit * dmar,device_t dev,uint16_t rid,int dev_domain,int dev_busno,const void * dev_path,int dev_path_len,bool id_mapped,bool rmrr_init)503 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid,
504 int dev_domain, int dev_busno, const void *dev_path, int dev_path_len,
505 bool id_mapped, bool rmrr_init)
506 {
507 struct dmar_domain *domain, *domain1;
508 struct dmar_ctx *ctx, *ctx1;
509 struct iommu_unit *unit __diagused;
510 dmar_ctx_entry_t *ctxp;
511 struct sf_buf *sf;
512 int bus, slot, func, error;
513 bool enable;
514
515 if (dev != NULL) {
516 bus = pci_get_bus(dev);
517 slot = pci_get_slot(dev);
518 func = pci_get_function(dev);
519 } else {
520 bus = PCI_RID2BUS(rid);
521 slot = PCI_RID2SLOT(rid);
522 func = PCI_RID2FUNC(rid);
523 }
524 enable = false;
525 TD_PREP_PINNED_ASSERT;
526 unit = DMAR2IOMMU(dmar);
527 DMAR_LOCK(dmar);
528 KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0),
529 ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus,
530 slot, func));
531 ctx = dmar_find_ctx_locked(dmar, rid);
532 error = 0;
533 if (ctx == NULL) {
534 /*
535 * Perform the allocations which require sleep or have
536 * higher chance to succeed if the sleep is allowed.
537 */
538 DMAR_UNLOCK(dmar);
539 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid));
540 domain1 = dmar_domain_alloc(dmar, id_mapped);
541 if (domain1 == NULL) {
542 TD_PINNED_ASSERT;
543 return (NULL);
544 }
545 if (!id_mapped) {
546 error = domain_init_rmrr(domain1, dev, bus,
547 slot, func, dev_domain, dev_busno, dev_path,
548 dev_path_len);
549 if (error == 0 && dev != NULL)
550 error = dmar_reserve_pci_regions(domain1, dev);
551 if (error != 0) {
552 dmar_domain_destroy(domain1);
553 TD_PINNED_ASSERT;
554 return (NULL);
555 }
556 }
557 ctx1 = dmar_ctx_alloc(domain1, rid);
558 ctxp = dmar_map_ctx_entry(ctx1, &sf);
559 DMAR_LOCK(dmar);
560
561 /*
562 * Recheck the contexts, other thread might have
563 * already allocated needed one.
564 */
565 ctx = dmar_find_ctx_locked(dmar, rid);
566 if (ctx == NULL) {
567 domain = domain1;
568 ctx = ctx1;
569 dmar_ctx_link(ctx);
570 ctx->context.tag->owner = dev;
571 iommu_device_tag_init(CTX2IOCTX(ctx), dev);
572
573 /*
574 * This is the first activated context for the
575 * DMAR unit. Enable the translation after
576 * everything is set up.
577 */
578 if (LIST_EMPTY(&dmar->domains))
579 enable = true;
580 LIST_INSERT_HEAD(&dmar->domains, domain, link);
581 ctx_id_entry_init(ctx, ctxp, false, bus);
582 if (dev != NULL) {
583 device_printf(dev,
584 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d "
585 "agaw %d %s-mapped\n",
586 dmar->iommu.unit, dmar->segment, bus, slot,
587 func, rid, domain->domain, domain->mgaw,
588 domain->agaw, id_mapped ? "id" : "re");
589 }
590 iommu_unmap_pgtbl(sf);
591 } else {
592 iommu_unmap_pgtbl(sf);
593 dmar_domain_destroy(domain1);
594 /* Nothing needs to be done to destroy ctx1. */
595 free(ctx1, M_DMAR_CTX);
596 domain = CTX2DOM(ctx);
597 ctx->context.refs++; /* tag referenced us */
598 }
599 } else {
600 domain = CTX2DOM(ctx);
601 if (ctx->context.tag->owner == NULL)
602 ctx->context.tag->owner = dev;
603 ctx->context.refs++; /* tag referenced us */
604 }
605
606 error = dmar_flush_for_ctx_entry(dmar, enable);
607 if (error != 0) {
608 dmar_free_ctx_locked(dmar, ctx);
609 TD_PINNED_ASSERT;
610 return (NULL);
611 }
612
613 /*
614 * The dmar lock was potentially dropped between check for the
615 * empty context list and now. Recheck the state of GCMD_TE
616 * to avoid unneeded command.
617 */
618 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
619 error = dmar_disable_protected_regions(dmar);
620 if (error != 0)
621 printf("dmar%d: Failed to disable protected regions\n",
622 dmar->iommu.unit);
623 error = dmar_enable_translation(dmar);
624 if (error == 0) {
625 if (bootverbose) {
626 printf("dmar%d: enabled translation\n",
627 dmar->iommu.unit);
628 }
629 } else {
630 printf("dmar%d: enabling translation failed, "
631 "error %d\n", dmar->iommu.unit, error);
632 dmar_free_ctx_locked(dmar, ctx);
633 TD_PINNED_ASSERT;
634 return (NULL);
635 }
636 }
637 DMAR_UNLOCK(dmar);
638 TD_PINNED_ASSERT;
639 return (ctx);
640 }
641
642 struct dmar_ctx *
dmar_get_ctx_for_dev(struct dmar_unit * dmar,device_t dev,uint16_t rid,bool id_mapped,bool rmrr_init)643 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid,
644 bool id_mapped, bool rmrr_init)
645 {
646 int dev_domain, dev_path_len, dev_busno;
647
648 dev_domain = pci_get_domain(dev);
649 dev_path_len = dmar_dev_depth(dev);
650 ACPI_DMAR_PCI_PATH dev_path[dev_path_len];
651 dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len);
652 return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno,
653 dev_path, dev_path_len, id_mapped, rmrr_init));
654 }
655
656 struct dmar_ctx *
dmar_get_ctx_for_devpath(struct dmar_unit * dmar,uint16_t rid,int dev_domain,int dev_busno,const void * dev_path,int dev_path_len,bool id_mapped,bool rmrr_init)657 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid,
658 int dev_domain, int dev_busno,
659 const void *dev_path, int dev_path_len,
660 bool id_mapped, bool rmrr_init)
661 {
662
663 return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno,
664 dev_path, dev_path_len, id_mapped, rmrr_init));
665 }
666
667 int
dmar_move_ctx_to_domain(struct dmar_domain * domain,struct dmar_ctx * ctx)668 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx)
669 {
670 struct dmar_unit *dmar;
671 struct dmar_domain *old_domain;
672 dmar_ctx_entry_t *ctxp;
673 struct sf_buf *sf;
674 int error;
675
676 dmar = domain->dmar;
677 old_domain = CTX2DOM(ctx);
678 if (domain == old_domain)
679 return (0);
680 KASSERT(old_domain->iodom.iommu == domain->iodom.iommu,
681 ("domain %p %u moving between dmars %u %u", domain,
682 domain->domain, old_domain->iodom.iommu->unit,
683 domain->iodom.iommu->unit));
684 TD_PREP_PINNED_ASSERT;
685
686 ctxp = dmar_map_ctx_entry(ctx, &sf);
687 DMAR_LOCK(dmar);
688 dmar_ctx_unlink(ctx);
689 ctx->context.domain = &domain->iodom;
690 dmar_ctx_link(ctx);
691 ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100);
692 iommu_unmap_pgtbl(sf);
693 error = dmar_flush_for_ctx_entry(dmar, true);
694 /* If flush failed, rolling back would not work as well. */
695 printf("dmar%d rid %x domain %d->%d %s-mapped\n",
696 dmar->iommu.unit, ctx->context.rid, old_domain->domain,
697 domain->domain, (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ?
698 "id" : "re");
699 dmar_unref_domain_locked(dmar, old_domain);
700 TD_PINNED_ASSERT;
701 return (error);
702 }
703
704 static void
dmar_unref_domain_locked(struct dmar_unit * dmar,struct dmar_domain * domain)705 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain)
706 {
707
708 DMAR_ASSERT_LOCKED(dmar);
709 KASSERT(domain->refs >= 1,
710 ("dmar %d domain %p refs %u", dmar->iommu.unit, domain,
711 domain->refs));
712 KASSERT(domain->refs > domain->ctx_cnt,
713 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain,
714 domain->refs, domain->ctx_cnt));
715
716 if (domain->refs > 1) {
717 domain->refs--;
718 DMAR_UNLOCK(dmar);
719 return;
720 }
721
722 KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0,
723 ("lost ref on RMRR domain %p", domain));
724
725 LIST_REMOVE(domain, link);
726 DMAR_UNLOCK(dmar);
727
728 taskqueue_drain(dmar->iommu.delayed_taskqueue,
729 &domain->iodom.unload_task);
730 dmar_domain_destroy(domain);
731 }
732
733 static void
dmar_free_ctx_locked(struct dmar_unit * dmar,struct dmar_ctx * ctx)734 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
735 {
736 struct sf_buf *sf;
737 dmar_ctx_entry_t *ctxp;
738 struct dmar_domain *domain;
739
740 DMAR_ASSERT_LOCKED(dmar);
741 KASSERT(ctx->context.refs >= 1,
742 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->context.refs));
743
744 /*
745 * If our reference is not last, only the dereference should
746 * be performed.
747 */
748 if (ctx->context.refs > 1) {
749 ctx->context.refs--;
750 DMAR_UNLOCK(dmar);
751 return;
752 }
753
754 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
755 ("lost ref on disabled ctx %p", ctx));
756
757 /*
758 * Otherwise, the context entry must be cleared before the
759 * page table is destroyed. The mapping of the context
760 * entries page could require sleep, unlock the dmar.
761 */
762 DMAR_UNLOCK(dmar);
763 TD_PREP_PINNED_ASSERT;
764 ctxp = dmar_map_ctx_entry(ctx, &sf);
765 DMAR_LOCK(dmar);
766 KASSERT(ctx->context.refs >= 1,
767 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->context.refs));
768
769 /*
770 * Other thread might have referenced the context, in which
771 * case again only the dereference should be performed.
772 */
773 if (ctx->context.refs > 1) {
774 ctx->context.refs--;
775 DMAR_UNLOCK(dmar);
776 iommu_unmap_pgtbl(sf);
777 TD_PINNED_ASSERT;
778 return;
779 }
780
781 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0,
782 ("lost ref on disabled ctx %p", ctx));
783
784 /*
785 * Clear the context pointer and flush the caches.
786 * XXXKIB: cannot do this if any RMRR entries are still present.
787 */
788 dmar_pte_clear(&ctxp->ctx1);
789 ctxp->ctx2 = 0;
790 dmar_flush_ctx_to_ram(dmar, ctxp);
791 dmar_inv_ctx_glob(dmar);
792 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
793 if (dmar->qi_enabled)
794 dmar_qi_invalidate_iotlb_glob_locked(dmar);
795 else
796 dmar_inv_iotlb_glob(dmar);
797 }
798 iommu_unmap_pgtbl(sf);
799 domain = CTX2DOM(ctx);
800 dmar_ctx_unlink(ctx);
801 free(ctx->context.tag, M_DMAR_CTX);
802 free(ctx, M_DMAR_CTX);
803 dmar_unref_domain_locked(dmar, domain);
804 TD_PINNED_ASSERT;
805 }
806
807 static void
dmar_free_ctx(struct dmar_ctx * ctx)808 dmar_free_ctx(struct dmar_ctx *ctx)
809 {
810 struct dmar_unit *dmar;
811
812 dmar = CTX2DMAR(ctx);
813 DMAR_LOCK(dmar);
814 dmar_free_ctx_locked(dmar, ctx);
815 }
816
817 /*
818 * Returns with the domain locked.
819 */
820 struct dmar_ctx *
dmar_find_ctx_locked(struct dmar_unit * dmar,uint16_t rid)821 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
822 {
823 struct dmar_domain *domain;
824 struct iommu_ctx *ctx;
825
826 DMAR_ASSERT_LOCKED(dmar);
827
828 LIST_FOREACH(domain, &dmar->domains, link) {
829 LIST_FOREACH(ctx, &domain->iodom.contexts, link) {
830 if (ctx->rid == rid)
831 return (IOCTX2CTX(ctx));
832 }
833 }
834 return (NULL);
835 }
836
837 /*
838 * If the given value for "free" is true, then the caller must not be using
839 * the entry's dmamap_link field.
840 */
841 void
dmar_domain_unload_entry(struct iommu_map_entry * entry,bool free,bool cansleep)842 dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free,
843 bool cansleep)
844 {
845 struct dmar_domain *domain;
846 struct dmar_unit *unit;
847
848 domain = IODOM2DOM(entry->domain);
849 unit = DOM2DMAR(domain);
850
851 /*
852 * If "free" is false, then the IOTLB invalidation must be performed
853 * synchronously. Otherwise, the caller might free the entry before
854 * dmar_qi_task() is finished processing it.
855 */
856 if (unit->qi_enabled) {
857 if (free) {
858 DMAR_LOCK(unit);
859 iommu_qi_invalidate_locked(&domain->iodom, entry,
860 true);
861 DMAR_UNLOCK(unit);
862 } else {
863 iommu_qi_invalidate_sync(&domain->iodom, entry->start,
864 entry->end - entry->start, cansleep);
865 iommu_domain_free_entry(entry, false);
866 }
867 } else {
868 dmar_flush_iotlb_sync(domain, entry->start, entry->end -
869 entry->start);
870 iommu_domain_free_entry(entry, free);
871 }
872 }
873
874 static bool
dmar_domain_unload_emit_wait(struct dmar_domain * domain,struct iommu_map_entry * entry)875 dmar_domain_unload_emit_wait(struct dmar_domain *domain,
876 struct iommu_map_entry *entry)
877 {
878
879 if (TAILQ_NEXT(entry, dmamap_link) == NULL)
880 return (true);
881 return (domain->batch_no++ % iommu_qi_batch_coalesce == 0);
882 }
883
884 void
dmar_domain_unload(struct iommu_domain * iodom,struct iommu_map_entries_tailq * entries,bool cansleep)885 dmar_domain_unload(struct iommu_domain *iodom,
886 struct iommu_map_entries_tailq *entries, bool cansleep)
887 {
888 struct dmar_domain *domain;
889 struct dmar_unit *unit;
890 struct iommu_map_entry *entry, *entry1;
891 int error __diagused;
892
893 domain = IODOM2DOM(iodom);
894 unit = DOM2DMAR(domain);
895
896 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
897 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
898 ("not mapped entry %p %p", domain, entry));
899 error = iodom->ops->unmap(iodom, entry,
900 cansleep ? IOMMU_PGF_WAITOK : 0);
901 KASSERT(error == 0, ("unmap %p error %d", domain, error));
902 if (!unit->qi_enabled) {
903 dmar_flush_iotlb_sync(domain, entry->start,
904 entry->end - entry->start);
905 TAILQ_REMOVE(entries, entry, dmamap_link);
906 iommu_domain_free_entry(entry, true);
907 }
908 }
909 if (TAILQ_EMPTY(entries))
910 return;
911
912 KASSERT(unit->qi_enabled, ("loaded entry left"));
913 DMAR_LOCK(unit);
914 while ((entry = TAILQ_FIRST(entries)) != NULL) {
915 TAILQ_REMOVE(entries, entry, dmamap_link);
916 iommu_qi_invalidate_locked(&domain->iodom, entry,
917 dmar_domain_unload_emit_wait(domain, entry));
918 }
919 DMAR_UNLOCK(unit);
920 }
921
922 struct iommu_ctx *
dmar_get_ctx(struct iommu_unit * iommu,device_t dev,uint16_t rid,bool id_mapped,bool rmrr_init)923 dmar_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
924 bool id_mapped, bool rmrr_init)
925 {
926 struct dmar_unit *dmar;
927 struct dmar_ctx *ret;
928
929 dmar = IOMMU2DMAR(iommu);
930 ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init);
931 return (CTX2IOCTX(ret));
932 }
933
934 void
dmar_free_ctx_locked_method(struct iommu_unit * iommu,struct iommu_ctx * context)935 dmar_free_ctx_locked_method(struct iommu_unit *iommu,
936 struct iommu_ctx *context)
937 {
938 struct dmar_unit *dmar;
939 struct dmar_ctx *ctx;
940
941 dmar = IOMMU2DMAR(iommu);
942 ctx = IOCTX2CTX(context);
943 dmar_free_ctx_locked(dmar, ctx);
944 }
945
946 void
dmar_free_ctx_method(struct iommu_ctx * context)947 dmar_free_ctx_method(struct iommu_ctx *context)
948 {
949 struct dmar_ctx *ctx;
950
951 ctx = IOCTX2CTX(context);
952 dmar_free_ctx(ctx);
953 }
954