1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019-2020 Ruslan Bukin <[email protected]>
5 *
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory (Department of Computer Science and
8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
9 * DARPA SSITH research programme.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /*
34 * Hardware overview.
35 *
36 * An incoming transaction from a peripheral device has an address, size,
37 * attributes and StreamID.
38 *
39 * In case of PCI-based devices, StreamID is a PCI rid.
40 *
41 * The StreamID is used to select a Stream Table Entry (STE) in a Stream table,
42 * which contains per-device configuration.
43 *
44 * Stream table is a linear or 2-level walk table (this driver supports both).
45 * Note that a linear table could occupy 1GB or more of memory depending on
46 * sid_bits value.
47 *
48 * STE is used to locate a Context Descriptor, which is a struct in memory
49 * that describes stages of translation, translation table type, pointer to
50 * level 0 of page tables, ASID, etc.
51 *
52 * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2):
53 * o S1 is used for the host machine traffic translation
54 * o S2 is for a hypervisor
55 *
56 * This driver enables S1 stage with standard AArch64 page tables.
57 *
58 * Note that SMMU does not share TLB with a main CPU.
59 * Command queue is used by this driver to Invalidate SMMU TLB, STE cache.
60 *
61 * An arm64 SoC could have more than one SMMU instance.
62 * ACPI IORT table describes which SMMU unit is assigned for a particular
63 * peripheral device.
64 *
65 * Queues.
66 *
67 * Register interface and Memory-based circular buffer queues are used
68 * to inferface SMMU.
69 *
70 * These are a Command queue for commands to send to the SMMU and an Event
71 * queue for event/fault reports from the SMMU. Optionally PRI queue is
72 * designed for PCIe page requests reception.
73 *
74 * Note that not every hardware supports PRI services. For instance they were
75 * not found in Neoverse N1 SDP machine.
76 * (This drivers does not implement PRI queue.)
77 *
78 * All SMMU queues are arranged as circular buffers in memory. They are used
79 * in a producer-consumer fashion so that an output queue contains data
80 * produced by the SMMU and consumed by software.
81 * An input queue contains data produced by software, consumed by the SMMU.
82 *
83 * Interrupts.
84 *
85 * Interrupts are not required by this driver for normal operation.
86 * The standard wired interrupt is only triggered when an event comes from
87 * the SMMU, which is only in a case of errors (e.g. translation fault).
88 */
89
90 #include "opt_platform.h"
91 #include "opt_acpi.h"
92
93 #include <sys/cdefs.h>
94 __FBSDID("$FreeBSD$");
95
96 #include <sys/param.h>
97 #include <sys/bitstring.h>
98 #include <sys/bus.h>
99 #include <sys/kernel.h>
100 #include <sys/malloc.h>
101 #include <sys/rman.h>
102 #include <sys/lock.h>
103 #include <sys/tree.h>
104 #include <sys/taskqueue.h>
105 #include <vm/vm.h>
106 #include <vm/vm_page.h>
107 #if DEV_ACPI
108 #include <contrib/dev/acpica/include/acpi.h>
109 #include <dev/acpica/acpivar.h>
110 #endif
111 #include <dev/pci/pcireg.h>
112 #include <dev/pci/pcivar.h>
113 #include <dev/iommu/iommu.h>
114
115 #include "iommu.h"
116 #include "iommu_if.h"
117
118 #include "smmureg.h"
119 #include "smmuvar.h"
120
121 #define STRTAB_L1_SZ_SHIFT 20
122 #define STRTAB_SPLIT 8
123
124 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6)
125 #define STRTAB_L1_DESC_DWORDS 1
126
127 #define STRTAB_STE_DWORDS 8
128
129 #define CMDQ_ENTRY_DWORDS 2
130 #define EVTQ_ENTRY_DWORDS 4
131 #define PRIQ_ENTRY_DWORDS 2
132
133 #define CD_DWORDS 8
134
135 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2))
136 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1))
137 #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */
138
139 #define SMMU_Q_ALIGN (64 * 1024)
140
141 static struct resource_spec smmu_spec[] = {
142 { SYS_RES_MEMORY, 0, RF_ACTIVE },
143 { SYS_RES_IRQ, 0, RF_ACTIVE },
144 { SYS_RES_IRQ, 1, RF_ACTIVE },
145 { SYS_RES_IRQ, 2, RF_ACTIVE },
146 RESOURCE_SPEC_END
147 };
148
149 MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR);
150
151 #define dprintf(fmt, ...)
152
153 struct smmu_event {
154 int ident;
155 char *str;
156 char *msg;
157 };
158
159 static struct smmu_event events[] = {
160 { 0x01, "F_UUT",
161 "Unsupported Upstream Transaction."},
162 { 0x02, "C_BAD_STREAMID",
163 "Transaction StreamID out of range."},
164 { 0x03, "F_STE_FETCH",
165 "Fetch of STE caused external abort."},
166 { 0x04, "C_BAD_STE",
167 "Used STE invalid."},
168 { 0x05, "F_BAD_ATS_TREQ",
169 "Address Translation Request disallowed for a StreamID "
170 "and a PCIe ATS Translation Request received."},
171 { 0x06, "F_STREAM_DISABLED",
172 "The STE of a transaction marks non-substream transactions "
173 "disabled."},
174 { 0x07, "F_TRANSL_FORBIDDEN",
175 "An incoming PCIe transaction is marked Translated but "
176 "SMMU bypass is disallowed for this StreamID."},
177 { 0x08, "C_BAD_SUBSTREAMID",
178 "Incoming SubstreamID present, but configuration is invalid."},
179 { 0x09, "F_CD_FETCH",
180 "Fetch of CD caused external abort."},
181 { 0x0a, "C_BAD_CD",
182 "Fetched CD invalid."},
183 { 0x0b, "F_WALK_EABT",
184 "An external abort occurred fetching (or updating) "
185 "a translation table descriptor."},
186 { 0x10, "F_TRANSLATION",
187 "Translation fault."},
188 { 0x11, "F_ADDR_SIZE",
189 "Address Size fault."},
190 { 0x12, "F_ACCESS",
191 "Access flag fault due to AF == 0 in a page or block TTD."},
192 { 0x13, "F_PERMISSION",
193 "Permission fault occurred on page access."},
194 { 0x20, "F_TLB_CONFLICT",
195 "A TLB conflict occurred because of the transaction."},
196 { 0x21, "F_CFG_CONFLICT",
197 "A configuration cache conflict occurred due to "
198 "the transaction."},
199 { 0x24, "E_PAGE_REQUEST",
200 "Speculative page request hint."},
201 { 0x25, "F_VMS_FETCH",
202 "Fetch of VMS caused external abort."},
203 { 0, NULL, NULL },
204 };
205
206 static int
smmu_q_has_space(struct smmu_queue * q)207 smmu_q_has_space(struct smmu_queue *q)
208 {
209
210 /*
211 * See 6.3.27 SMMU_CMDQ_PROD
212 *
213 * There is space in the queue for additional commands if:
214 * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR ||
215 * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP
216 */
217
218 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||
219 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
220 return (1);
221
222 return (0);
223 }
224
225 static int
smmu_q_empty(struct smmu_queue * q)226 smmu_q_empty(struct smmu_queue *q)
227 {
228
229 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&
230 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
231 return (1);
232
233 return (0);
234 }
235
236 static int __unused
smmu_q_consumed(struct smmu_queue * q,uint32_t prod)237 smmu_q_consumed(struct smmu_queue *q, uint32_t prod)
238 {
239
240 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&
241 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))
242 return (1);
243
244 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&
245 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))
246 return (1);
247
248 return (0);
249 }
250
251 static uint32_t
smmu_q_inc_cons(struct smmu_queue * q)252 smmu_q_inc_cons(struct smmu_queue *q)
253 {
254 uint32_t cons;
255 uint32_t val;
256
257 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;
258 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));
259
260 return (val);
261 }
262
263 static uint32_t
smmu_q_inc_prod(struct smmu_queue * q)264 smmu_q_inc_prod(struct smmu_queue *q)
265 {
266 uint32_t prod;
267 uint32_t val;
268
269 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;
270 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));
271
272 return (val);
273 }
274
275 static int
smmu_write_ack(struct smmu_softc * sc,uint32_t reg,uint32_t reg_ack,uint32_t val)276 smmu_write_ack(struct smmu_softc *sc, uint32_t reg,
277 uint32_t reg_ack, uint32_t val)
278 {
279 uint32_t v;
280 int timeout;
281
282 timeout = 100000;
283
284 bus_write_4(sc->res[0], reg, val);
285
286 do {
287 v = bus_read_4(sc->res[0], reg_ack);
288 if (v == val)
289 break;
290 } while (timeout--);
291
292 if (timeout <= 0) {
293 device_printf(sc->dev, "Failed to write reg.\n");
294 return (-1);
295 }
296
297 return (0);
298 }
299
300 static inline int
ilog2(long x)301 ilog2(long x)
302 {
303
304 KASSERT(x > 0 && powerof2(x), ("%s: invalid arg %ld", __func__, x));
305
306 return (flsl(x) - 1);
307 }
308
309 static int
smmu_init_queue(struct smmu_softc * sc,struct smmu_queue * q,uint32_t prod_off,uint32_t cons_off,uint32_t dwords)310 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q,
311 uint32_t prod_off, uint32_t cons_off, uint32_t dwords)
312 {
313 int sz;
314
315 sz = (1 << q->size_log2) * dwords * 8;
316
317 /* Set up the command circular buffer */
318 q->vaddr = contigmalloc(sz, M_SMMU,
319 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);
320 if (q->vaddr == NULL) {
321 device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
322 return (-1);
323 }
324
325 q->prod_off = prod_off;
326 q->cons_off = cons_off;
327 q->paddr = vtophys(q->vaddr);
328
329 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;
330 q->base |= q->paddr & Q_BASE_ADDR_M;
331 q->base |= q->size_log2 << Q_LOG2SIZE_S;
332
333 return (0);
334 }
335
336 static int
smmu_init_queues(struct smmu_softc * sc)337 smmu_init_queues(struct smmu_softc *sc)
338 {
339 int err;
340
341 /* Command queue. */
342 err = smmu_init_queue(sc, &sc->cmdq,
343 SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS);
344 if (err)
345 return (ENXIO);
346
347 /* Event queue. */
348 err = smmu_init_queue(sc, &sc->evtq,
349 SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS);
350 if (err)
351 return (ENXIO);
352
353 if (!(sc->features & SMMU_FEATURE_PRI))
354 return (0);
355
356 /* PRI queue. */
357 err = smmu_init_queue(sc, &sc->priq,
358 SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS);
359 if (err)
360 return (ENXIO);
361
362 return (0);
363 }
364
365 /*
366 * Dump 2LVL or linear STE.
367 */
368 static void
smmu_dump_ste(struct smmu_softc * sc,int sid)369 smmu_dump_ste(struct smmu_softc *sc, int sid)
370 {
371 struct smmu_strtab *strtab;
372 struct l1_desc *l1_desc;
373 uint64_t *ste, *l1;
374 int i;
375
376 strtab = &sc->strtab;
377
378 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
379 i = sid >> STRTAB_SPLIT;
380 l1 = (void *)((uint64_t)strtab->vaddr +
381 STRTAB_L1_DESC_DWORDS * 8 * i);
382 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);
383
384 l1_desc = &strtab->l1[i];
385 ste = l1_desc->va;
386 if (ste == NULL) /* L2 is not initialized */
387 return;
388 } else {
389 ste = (void *)((uint64_t)strtab->vaddr +
390 sid * (STRTAB_STE_DWORDS << 3));
391 }
392
393 /* Dump L2 or linear STE. */
394 for (i = 0; i < STRTAB_STE_DWORDS; i++)
395 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);
396 }
397
398 static void __unused
smmu_dump_cd(struct smmu_softc * sc,struct smmu_cd * cd)399 smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd)
400 {
401 uint64_t *vaddr;
402 int i;
403
404 device_printf(sc->dev, "%s\n", __func__);
405
406 vaddr = cd->vaddr;
407 for (i = 0; i < CD_DWORDS; i++)
408 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);
409 }
410
411 static void
smmu_evtq_dequeue(struct smmu_softc * sc,uint32_t * evt)412 smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt)
413 {
414 struct smmu_queue *evtq;
415 void *entry_addr;
416
417 evtq = &sc->evtq;
418
419 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);
420 entry_addr = (void *)((uint64_t)evtq->vaddr +
421 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);
422 memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8);
423 evtq->lc.cons = smmu_q_inc_cons(evtq);
424 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);
425 }
426
427 static void
smmu_print_event(struct smmu_softc * sc,uint32_t * evt)428 smmu_print_event(struct smmu_softc *sc, uint32_t *evt)
429 {
430 struct smmu_event *ev;
431 uintptr_t input_addr;
432 uint8_t event_id;
433 device_t dev;
434 int sid;
435 int i;
436
437 dev = sc->dev;
438
439 ev = NULL;
440 event_id = evt[0] & 0xff;
441 for (i = 0; events[i].ident != 0; i++) {
442 if (events[i].ident == event_id) {
443 ev = &events[i];
444 break;
445 }
446 }
447
448 sid = evt[1];
449 input_addr = evt[5];
450 input_addr <<= 32;
451 input_addr |= evt[4];
452
453 if (smmu_quirks_check(dev, sid, event_id, input_addr)) {
454 /* The event is known. Don't print anything. */
455 return;
456 }
457
458 if (ev) {
459 device_printf(sc->dev,
460 "Event %s (%s) received.\n", ev->str, ev->msg);
461 } else
462 device_printf(sc->dev, "Event 0x%x received\n", event_id);
463
464 device_printf(sc->dev, "SID %x, Input Address: %jx\n",
465 sid, input_addr);
466
467 for (i = 0; i < 8; i++)
468 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);
469
470 smmu_dump_ste(sc, sid);
471 }
472
473 static void
make_cmd(struct smmu_softc * sc,uint64_t * cmd,struct smmu_cmdq_entry * entry)474 make_cmd(struct smmu_softc *sc, uint64_t *cmd,
475 struct smmu_cmdq_entry *entry)
476 {
477
478 memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8);
479 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;
480
481 switch (entry->opcode) {
482 case CMD_TLBI_NH_VA:
483 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
484 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;
485 if (entry->tlbi.leaf) {
486 /*
487 * Leaf flag means that only cached entries
488 * for the last level of translation table walk
489 * are required to be invalidated.
490 */
491 cmd[1] |= TLBI_1_LEAF;
492 }
493 break;
494 case CMD_TLBI_NH_ASID:
495 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
496 break;
497 case CMD_TLBI_NSNH_ALL:
498 case CMD_TLBI_NH_ALL:
499 case CMD_TLBI_EL2_ALL:
500 break;
501 case CMD_CFGI_CD:
502 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);
503 /* FALLTROUGH */
504 case CMD_CFGI_STE:
505 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);
506 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);
507 break;
508 case CMD_CFGI_STE_RANGE:
509 cmd[1] = (31 << CFGI_1_STE_RANGE_S);
510 break;
511 case CMD_SYNC:
512 cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB;
513 if (entry->sync.msiaddr) {
514 cmd[0] |= SYNC_0_CS_SIG_IRQ;
515 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);
516 } else
517 cmd[0] |= SYNC_0_CS_SIG_SEV;
518 break;
519 case CMD_PREFETCH_CONFIG:
520 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);
521 break;
522 };
523 }
524
525 static void
smmu_cmdq_enqueue_cmd(struct smmu_softc * sc,struct smmu_cmdq_entry * entry)526 smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry)
527 {
528 uint64_t cmd[CMDQ_ENTRY_DWORDS];
529 struct smmu_queue *cmdq;
530 void *entry_addr;
531
532 cmdq = &sc->cmdq;
533
534 make_cmd(sc, cmd, entry);
535
536 SMMU_LOCK(sc);
537
538 /* Ensure that a space is available. */
539 do {
540 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);
541 } while (smmu_q_has_space(cmdq) == 0);
542
543 /* Write the command to the current prod entry. */
544 entry_addr = (void *)((uint64_t)cmdq->vaddr +
545 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);
546 memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8);
547
548 /* Increment prod index. */
549 cmdq->lc.prod = smmu_q_inc_prod(cmdq);
550 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);
551
552 SMMU_UNLOCK(sc);
553 }
554
555 static void __unused
smmu_poll_until_consumed(struct smmu_softc * sc,struct smmu_queue * q)556 smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q)
557 {
558
559 while (1) {
560 q->lc.val = bus_read_8(sc->res[0], q->prod_off);
561 if (smmu_q_empty(q))
562 break;
563 cpu_spinwait();
564 }
565 }
566
567 static int
smmu_sync(struct smmu_softc * sc)568 smmu_sync(struct smmu_softc *sc)
569 {
570 struct smmu_cmdq_entry cmd;
571 struct smmu_queue *q;
572 uint32_t *base;
573 int timeout;
574 int prod;
575
576 q = &sc->cmdq;
577 prod = q->lc.prod;
578
579 /* Enqueue sync command. */
580 cmd.opcode = CMD_SYNC;
581 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;
582 smmu_cmdq_enqueue_cmd(sc, &cmd);
583
584 /* Wait for the sync completion. */
585 base = (void *)((uint64_t)q->vaddr +
586 Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8);
587
588 /*
589 * It takes around 200 loops (6 instructions each)
590 * on Neoverse N1 to complete the sync.
591 */
592 timeout = 10000;
593
594 do {
595 if (*base == 0) {
596 /* MSI write completed. */
597 break;
598 }
599 cpu_spinwait();
600 } while (timeout--);
601
602 if (timeout < 0)
603 device_printf(sc->dev, "Failed to sync\n");
604
605 return (0);
606 }
607
608 static int
smmu_sync_cd(struct smmu_softc * sc,int sid,int ssid,bool leaf)609 smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf)
610 {
611 struct smmu_cmdq_entry cmd;
612
613 cmd.opcode = CMD_CFGI_CD;
614 cmd.cfgi.sid = sid;
615 cmd.cfgi.ssid = ssid;
616 cmd.cfgi.leaf = leaf;
617 smmu_cmdq_enqueue_cmd(sc, &cmd);
618
619 return (0);
620 }
621
622 static void
smmu_invalidate_all_sid(struct smmu_softc * sc)623 smmu_invalidate_all_sid(struct smmu_softc *sc)
624 {
625 struct smmu_cmdq_entry cmd;
626
627 /* Invalidate cached config */
628 cmd.opcode = CMD_CFGI_STE_RANGE;
629 smmu_cmdq_enqueue_cmd(sc, &cmd);
630 smmu_sync(sc);
631 }
632
633 static void
smmu_tlbi_all(struct smmu_softc * sc)634 smmu_tlbi_all(struct smmu_softc *sc)
635 {
636 struct smmu_cmdq_entry cmd;
637
638 /* Invalidate entire TLB */
639 cmd.opcode = CMD_TLBI_NSNH_ALL;
640 smmu_cmdq_enqueue_cmd(sc, &cmd);
641 smmu_sync(sc);
642 }
643
644 static void
smmu_tlbi_asid(struct smmu_softc * sc,uint16_t asid)645 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid)
646 {
647 struct smmu_cmdq_entry cmd;
648
649 /* Invalidate TLB for an ASID. */
650 cmd.opcode = CMD_TLBI_NH_ASID;
651 cmd.tlbi.asid = asid;
652 smmu_cmdq_enqueue_cmd(sc, &cmd);
653 smmu_sync(sc);
654 }
655
656 static void
smmu_tlbi_va(struct smmu_softc * sc,vm_offset_t va,uint16_t asid)657 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid)
658 {
659 struct smmu_cmdq_entry cmd;
660
661 /* Invalidate specific range */
662 cmd.opcode = CMD_TLBI_NH_VA;
663 cmd.tlbi.asid = asid;
664 cmd.tlbi.vmid = 0;
665 cmd.tlbi.leaf = true; /* We change only L3. */
666 cmd.tlbi.addr = va;
667 smmu_cmdq_enqueue_cmd(sc, &cmd);
668 }
669
670 static void
smmu_invalidate_sid(struct smmu_softc * sc,uint32_t sid)671 smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid)
672 {
673 struct smmu_cmdq_entry cmd;
674
675 /* Invalidate cached config */
676 cmd.opcode = CMD_CFGI_STE;
677 cmd.cfgi.sid = sid;
678 smmu_cmdq_enqueue_cmd(sc, &cmd);
679 smmu_sync(sc);
680 }
681
682 static void
smmu_prefetch_sid(struct smmu_softc * sc,uint32_t sid)683 smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid)
684 {
685 struct smmu_cmdq_entry cmd;
686
687 cmd.opcode = CMD_PREFETCH_CONFIG;
688 cmd.prefetch.sid = sid;
689 smmu_cmdq_enqueue_cmd(sc, &cmd);
690 smmu_sync(sc);
691 }
692
693 /*
694 * Init STE in bypass mode. Traffic is not translated for the sid.
695 */
696 static void
smmu_init_ste_bypass(struct smmu_softc * sc,uint32_t sid,uint64_t * ste)697 smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste)
698 {
699 uint64_t val;
700
701 val = STE0_VALID | STE0_CONFIG_BYPASS;
702
703 ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS;
704 ste[2] = 0;
705 ste[3] = 0;
706 ste[4] = 0;
707 ste[5] = 0;
708 ste[6] = 0;
709 ste[7] = 0;
710
711 smmu_invalidate_sid(sc, sid);
712 ste[0] = val;
713 dsb(sy);
714 smmu_invalidate_sid(sc, sid);
715
716 smmu_prefetch_sid(sc, sid);
717 }
718
719 /*
720 * Enable Stage1 (S1) translation for the sid.
721 */
722 static int
smmu_init_ste_s1(struct smmu_softc * sc,struct smmu_cd * cd,uint32_t sid,uint64_t * ste)723 smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd,
724 uint32_t sid, uint64_t *ste)
725 {
726 uint64_t val;
727
728 val = STE0_VALID;
729
730 /* S1 */
731 ste[1] = STE1_EATS_FULLATS |
732 STE1_S1CSH_IS |
733 STE1_S1CIR_WBRA |
734 STE1_S1COR_WBRA |
735 STE1_STRW_NS_EL1;
736 ste[2] = 0;
737 ste[3] = 0;
738 ste[4] = 0;
739 ste[5] = 0;
740 ste[6] = 0;
741 ste[7] = 0;
742
743 if (sc->features & SMMU_FEATURE_STALL &&
744 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))
745 ste[1] |= STE1_S1STALLD;
746
747 /* Configure STE */
748 val |= (cd->paddr & STE0_S1CONTEXTPTR_M);
749 val |= STE0_CONFIG_S1_TRANS;
750
751 smmu_invalidate_sid(sc, sid);
752
753 /* The STE[0] has to be written in a single blast, last of all. */
754 ste[0] = val;
755 dsb(sy);
756
757 smmu_invalidate_sid(sc, sid);
758 smmu_sync_cd(sc, sid, 0, true);
759 smmu_invalidate_sid(sc, sid);
760
761 /* The sid will be used soon most likely. */
762 smmu_prefetch_sid(sc, sid);
763
764 return (0);
765 }
766
767 static int
smmu_init_ste(struct smmu_softc * sc,struct smmu_cd * cd,int sid,bool bypass)768 smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass)
769 {
770 struct smmu_strtab *strtab;
771 struct l1_desc *l1_desc;
772 uint64_t *addr;
773
774 strtab = &sc->strtab;
775
776 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
777 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
778 addr = l1_desc->va;
779 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
780 } else {
781 addr = (void *)((uint64_t)strtab->vaddr +
782 STRTAB_STE_DWORDS * 8 * sid);
783 };
784
785 if (bypass)
786 smmu_init_ste_bypass(sc, sid, addr);
787 else
788 smmu_init_ste_s1(sc, cd, sid, addr);
789
790 smmu_sync(sc);
791
792 return (0);
793 }
794
795 static int
smmu_init_cd(struct smmu_softc * sc,struct smmu_domain * domain)796 smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain)
797 {
798 vm_paddr_t paddr;
799 uint64_t *ptr;
800 uint64_t val;
801 vm_size_t size;
802 struct smmu_cd *cd;
803 pmap_t p;
804
805 size = 1 * (CD_DWORDS << 3);
806
807 p = &domain->p;
808 cd = domain->cd = malloc(sizeof(struct smmu_cd),
809 M_SMMU, M_WAITOK | M_ZERO);
810
811 cd->vaddr = contigmalloc(size, M_SMMU,
812 M_WAITOK | M_ZERO, /* flags */
813 0, /* low */
814 (1ul << 40) - 1, /* high */
815 size, /* alignment */
816 0); /* boundary */
817 if (cd->vaddr == NULL) {
818 device_printf(sc->dev, "Failed to allocate CD\n");
819 return (ENXIO);
820 }
821
822 cd->size = size;
823 cd->paddr = vtophys(cd->vaddr);
824
825 ptr = cd->vaddr;
826
827 val = CD0_VALID;
828 val |= CD0_AA64;
829 val |= CD0_R;
830 val |= CD0_A;
831 val |= CD0_ASET;
832 val |= (uint64_t)domain->asid << CD0_ASID_S;
833 val |= CD0_TG0_4KB;
834 val |= CD0_EPD1; /* Disable TT1 */
835 val |= ((64 - sc->ias) << CD0_T0SZ_S);
836 val |= CD0_IPS_48BITS;
837
838 paddr = p->pm_l0_paddr & CD1_TTB0_M;
839 KASSERT(paddr == p->pm_l0_paddr, ("bad allocation 1"));
840
841 ptr[1] = paddr;
842 ptr[2] = 0;
843 ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) |
844 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) |
845 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) |
846 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH);
847
848 /* Install the CD. */
849 ptr[0] = val;
850
851 return (0);
852 }
853
854 static int
smmu_init_strtab_linear(struct smmu_softc * sc)855 smmu_init_strtab_linear(struct smmu_softc *sc)
856 {
857 struct smmu_strtab *strtab;
858 vm_paddr_t base;
859 uint32_t size;
860 uint64_t reg;
861
862 strtab = &sc->strtab;
863 strtab->num_l1_entries = (1 << sc->sid_bits);
864
865 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);
866
867 if (bootverbose)
868 device_printf(sc->dev,
869 "%s: linear strtab size %d, num_l1_entries %d\n",
870 __func__, size, strtab->num_l1_entries);
871
872 strtab->vaddr = contigmalloc(size, M_SMMU,
873 M_WAITOK | M_ZERO, /* flags */
874 0, /* low */
875 (1ul << 48) - 1, /* high */
876 size, /* alignment */
877 0); /* boundary */
878 if (strtab->vaddr == NULL) {
879 device_printf(sc->dev, "failed to allocate strtab\n");
880 return (ENXIO);
881 }
882
883 reg = STRTAB_BASE_CFG_FMT_LINEAR;
884 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;
885 strtab->base_cfg = (uint32_t)reg;
886
887 base = vtophys(strtab->vaddr);
888
889 reg = base & STRTAB_BASE_ADDR_M;
890 KASSERT(reg == base, ("bad allocation 2"));
891 reg |= STRTAB_BASE_RA;
892 strtab->base = reg;
893
894 return (0);
895 }
896
897 static int
smmu_init_strtab_2lvl(struct smmu_softc * sc)898 smmu_init_strtab_2lvl(struct smmu_softc *sc)
899 {
900 struct smmu_strtab *strtab;
901 vm_paddr_t base;
902 uint64_t reg_base;
903 uint32_t l1size;
904 uint32_t size;
905 uint32_t reg;
906 int sz;
907
908 strtab = &sc->strtab;
909
910 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
911 size = min(size, sc->sid_bits - STRTAB_SPLIT);
912 strtab->num_l1_entries = (1 << size);
913 size += STRTAB_SPLIT;
914
915 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);
916
917 if (bootverbose)
918 device_printf(sc->dev,
919 "%s: size %d, l1 entries %d, l1size %d\n",
920 __func__, size, strtab->num_l1_entries, l1size);
921
922 strtab->vaddr = contigmalloc(l1size, M_SMMU,
923 M_WAITOK | M_ZERO, /* flags */
924 0, /* low */
925 (1ul << 48) - 1, /* high */
926 l1size, /* alignment */
927 0); /* boundary */
928 if (strtab->vaddr == NULL) {
929 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");
930 return (ENOMEM);
931 }
932
933 sz = strtab->num_l1_entries * sizeof(struct l1_desc);
934
935 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);
936 if (strtab->l1 == NULL) {
937 contigfree(strtab->vaddr, l1size, M_SMMU);
938 return (ENOMEM);
939 }
940
941 reg = STRTAB_BASE_CFG_FMT_2LVL;
942 reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S;
943 reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S;
944 strtab->base_cfg = (uint32_t)reg;
945
946 base = vtophys(strtab->vaddr);
947
948 reg_base = base & STRTAB_BASE_ADDR_M;
949 KASSERT(reg_base == base, ("bad allocation 3"));
950 reg_base |= STRTAB_BASE_RA;
951 strtab->base = reg_base;
952
953 return (0);
954 }
955
956 static int
smmu_init_strtab(struct smmu_softc * sc)957 smmu_init_strtab(struct smmu_softc *sc)
958 {
959 int error;
960
961 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)
962 error = smmu_init_strtab_2lvl(sc);
963 else
964 error = smmu_init_strtab_linear(sc);
965
966 return (error);
967 }
968
969 static int
smmu_init_l1_entry(struct smmu_softc * sc,int sid)970 smmu_init_l1_entry(struct smmu_softc *sc, int sid)
971 {
972 struct smmu_strtab *strtab;
973 struct l1_desc *l1_desc;
974 uint64_t *addr;
975 uint64_t val;
976 size_t size;
977 int i;
978
979 strtab = &sc->strtab;
980 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
981
982 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
983
984 l1_desc->span = STRTAB_SPLIT + 1;
985 l1_desc->size = size;
986 l1_desc->va = contigmalloc(size, M_SMMU,
987 M_WAITOK | M_ZERO, /* flags */
988 0, /* low */
989 (1ul << 48) - 1, /* high */
990 size, /* alignment */
991 0); /* boundary */
992 if (l1_desc->va == NULL) {
993 device_printf(sc->dev, "failed to allocate l2 entry\n");
994 return (ENXIO);
995 }
996
997 l1_desc->pa = vtophys(l1_desc->va);
998
999 i = sid >> STRTAB_SPLIT;
1000 addr = (void *)((uint64_t)strtab->vaddr +
1001 STRTAB_L1_DESC_DWORDS * 8 * i);
1002
1003 /* Install the L1 entry. */
1004 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;
1005 KASSERT(val == l1_desc->pa, ("bad allocation 4"));
1006 val |= l1_desc->span;
1007 *addr = val;
1008
1009 return (0);
1010 }
1011
1012 static void
smmu_deinit_l1_entry(struct smmu_softc * sc,int sid)1013 smmu_deinit_l1_entry(struct smmu_softc *sc, int sid)
1014 {
1015 struct smmu_strtab *strtab;
1016 struct l1_desc *l1_desc;
1017 uint64_t *addr;
1018 int i;
1019
1020 strtab = &sc->strtab;
1021
1022 i = sid >> STRTAB_SPLIT;
1023 addr = (void *)((uint64_t)strtab->vaddr +
1024 STRTAB_L1_DESC_DWORDS * 8 * i);
1025 *addr = 0;
1026
1027 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
1028 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
1029 contigfree(l1_desc->va, l1_desc->size, M_SMMU);
1030 }
1031 }
1032
1033 static int
smmu_disable(struct smmu_softc * sc)1034 smmu_disable(struct smmu_softc *sc)
1035 {
1036 uint32_t reg;
1037 int error;
1038
1039 /* Disable SMMU */
1040 reg = bus_read_4(sc->res[0], SMMU_CR0);
1041 reg &= ~CR0_SMMUEN;
1042 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1043 if (error)
1044 device_printf(sc->dev, "Could not disable SMMU.\n");
1045
1046 return (0);
1047 }
1048
1049 static int
smmu_event_intr(void * arg)1050 smmu_event_intr(void *arg)
1051 {
1052 uint32_t evt[EVTQ_ENTRY_DWORDS * 2];
1053 struct smmu_softc *sc;
1054
1055 sc = arg;
1056
1057 do {
1058 smmu_evtq_dequeue(sc, evt);
1059 smmu_print_event(sc, evt);
1060 } while (!smmu_q_empty(&sc->evtq));
1061
1062 return (FILTER_HANDLED);
1063 }
1064
1065 static int __unused
smmu_sync_intr(void * arg)1066 smmu_sync_intr(void *arg)
1067 {
1068 struct smmu_softc *sc;
1069
1070 sc = arg;
1071
1072 device_printf(sc->dev, "%s\n", __func__);
1073
1074 return (FILTER_HANDLED);
1075 }
1076
1077 static int
smmu_gerr_intr(void * arg)1078 smmu_gerr_intr(void *arg)
1079 {
1080 struct smmu_softc *sc;
1081
1082 sc = arg;
1083
1084 device_printf(sc->dev, "SMMU Global Error\n");
1085
1086 return (FILTER_HANDLED);
1087 }
1088
1089 static int
smmu_enable_interrupts(struct smmu_softc * sc)1090 smmu_enable_interrupts(struct smmu_softc *sc)
1091 {
1092 uint32_t reg;
1093 int error;
1094
1095 /* Disable MSI. */
1096 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);
1097 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);
1098 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);
1099
1100 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);
1101 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);
1102 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);
1103
1104 if (sc->features & CR0_PRIQEN) {
1105 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);
1106 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);
1107 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);
1108 }
1109
1110 /* Disable any interrupts. */
1111 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0);
1112 if (error) {
1113 device_printf(sc->dev, "Could not disable interrupts.\n");
1114 return (ENXIO);
1115 }
1116
1117 /* Enable interrupts. */
1118 reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
1119 if (sc->features & SMMU_FEATURE_PRI)
1120 reg |= IRQ_CTRL_PRIQ_IRQEN;
1121
1122 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg);
1123 if (error) {
1124 device_printf(sc->dev, "Could not enable interrupts.\n");
1125 return (ENXIO);
1126 }
1127
1128 return (0);
1129 }
1130
1131 #if DEV_ACPI
1132 static void
smmu_configure_intr(struct smmu_softc * sc,struct resource * res)1133 smmu_configure_intr(struct smmu_softc *sc, struct resource *res)
1134 {
1135 struct intr_map_data_acpi *ad;
1136 struct intr_map_data *data;
1137
1138 data = rman_get_virtual(res);
1139 KASSERT(data != NULL, ("data is NULL"));
1140
1141 if (data->type == INTR_MAP_DATA_ACPI) {
1142 ad = (struct intr_map_data_acpi *)data;
1143 ad->trig = INTR_TRIGGER_EDGE;
1144 ad->pol = INTR_POLARITY_HIGH;
1145 }
1146 }
1147 #endif
1148
1149 static int
smmu_setup_interrupts(struct smmu_softc * sc)1150 smmu_setup_interrupts(struct smmu_softc *sc)
1151 {
1152 device_t dev;
1153 int error;
1154
1155 dev = sc->dev;
1156
1157 #if DEV_ACPI
1158 /*
1159 * Configure SMMU interrupts as EDGE triggered manually
1160 * as ACPI tables carries no information for that.
1161 */
1162 smmu_configure_intr(sc, sc->res[1]);
1163 smmu_configure_intr(sc, sc->res[2]);
1164 smmu_configure_intr(sc, sc->res[3]);
1165 #endif
1166
1167 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
1168 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);
1169 if (error) {
1170 device_printf(dev, "Couldn't setup Event interrupt handler\n");
1171 return (ENXIO);
1172 }
1173
1174 error = bus_setup_intr(dev, sc->res[3], INTR_TYPE_MISC,
1175 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);
1176 if (error) {
1177 device_printf(dev, "Couldn't setup Gerr interrupt handler\n");
1178 return (ENXIO);
1179 }
1180
1181 return (0);
1182 }
1183
1184 static int
smmu_reset(struct smmu_softc * sc)1185 smmu_reset(struct smmu_softc *sc)
1186 {
1187 struct smmu_cmdq_entry cmd;
1188 struct smmu_strtab *strtab;
1189 int error;
1190 int reg;
1191
1192 reg = bus_read_4(sc->res[0], SMMU_CR0);
1193
1194 if (reg & CR0_SMMUEN)
1195 device_printf(sc->dev,
1196 "%s: Warning: SMMU is enabled\n", __func__);
1197
1198 error = smmu_disable(sc);
1199 if (error)
1200 device_printf(sc->dev,
1201 "%s: Could not disable SMMU.\n", __func__);
1202
1203 if (smmu_enable_interrupts(sc) != 0) {
1204 device_printf(sc->dev, "Could not enable interrupts.\n");
1205 return (ENXIO);
1206 }
1207
1208 reg = CR1_TABLE_SH_IS |
1209 CR1_TABLE_OC_WBC |
1210 CR1_TABLE_IC_WBC |
1211 CR1_QUEUE_SH_IS |
1212 CR1_QUEUE_OC_WBC |
1213 CR1_QUEUE_IC_WBC;
1214 bus_write_4(sc->res[0], SMMU_CR1, reg);
1215
1216 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
1217 bus_write_4(sc->res[0], SMMU_CR2, reg);
1218
1219 /* Stream table. */
1220 strtab = &sc->strtab;
1221 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);
1222 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);
1223
1224 /* Command queue. */
1225 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);
1226 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);
1227 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);
1228
1229 reg = CR0_CMDQEN;
1230 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1231 if (error) {
1232 device_printf(sc->dev, "Could not enable command queue\n");
1233 return (ENXIO);
1234 }
1235
1236 /* Invalidate cached configuration. */
1237 smmu_invalidate_all_sid(sc);
1238
1239 if (sc->features & SMMU_FEATURE_HYP) {
1240 cmd.opcode = CMD_TLBI_EL2_ALL;
1241 smmu_cmdq_enqueue_cmd(sc, &cmd);
1242 };
1243
1244 /* Invalidate TLB. */
1245 smmu_tlbi_all(sc);
1246
1247 /* Event queue */
1248 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);
1249 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);
1250 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);
1251
1252 reg |= CR0_EVENTQEN;
1253 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1254 if (error) {
1255 device_printf(sc->dev, "Could not enable event queue\n");
1256 return (ENXIO);
1257 }
1258
1259 if (sc->features & SMMU_FEATURE_PRI) {
1260 /* PRI queue */
1261 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);
1262 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);
1263 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);
1264
1265 reg |= CR0_PRIQEN;
1266 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1267 if (error) {
1268 device_printf(sc->dev, "Could not enable PRI queue\n");
1269 return (ENXIO);
1270 }
1271 }
1272
1273 if (sc->features & SMMU_FEATURE_ATS) {
1274 reg |= CR0_ATSCHK;
1275 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1276 if (error) {
1277 device_printf(sc->dev, "Could not enable ATS check.\n");
1278 return (ENXIO);
1279 }
1280 }
1281
1282 reg |= CR0_SMMUEN;
1283 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1284 if (error) {
1285 device_printf(sc->dev, "Could not enable SMMU.\n");
1286 return (ENXIO);
1287 }
1288
1289 return (0);
1290 }
1291
1292 static int
smmu_check_features(struct smmu_softc * sc)1293 smmu_check_features(struct smmu_softc *sc)
1294 {
1295 uint32_t reg;
1296 uint32_t val;
1297
1298 sc->features = 0;
1299
1300 reg = bus_read_4(sc->res[0], SMMU_IDR0);
1301
1302 if (reg & IDR0_ST_LVL_2) {
1303 if (bootverbose)
1304 device_printf(sc->dev,
1305 "2-level stream table supported.\n");
1306 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;
1307 }
1308
1309 if (reg & IDR0_CD2L) {
1310 if (bootverbose)
1311 device_printf(sc->dev,
1312 "2-level CD table supported.\n");
1313 sc->features |= SMMU_FEATURE_2_LVL_CD;
1314 }
1315
1316 switch (reg & IDR0_TTENDIAN_M) {
1317 case IDR0_TTENDIAN_MIXED:
1318 if (bootverbose)
1319 device_printf(sc->dev, "Mixed endianess supported.\n");
1320 sc->features |= SMMU_FEATURE_TT_LE;
1321 sc->features |= SMMU_FEATURE_TT_BE;
1322 break;
1323 case IDR0_TTENDIAN_LITTLE:
1324 if (bootverbose)
1325 device_printf(sc->dev,
1326 "Little endian supported only.\n");
1327 sc->features |= SMMU_FEATURE_TT_LE;
1328 break;
1329 case IDR0_TTENDIAN_BIG:
1330 if (bootverbose)
1331 device_printf(sc->dev, "Big endian supported only.\n");
1332 sc->features |= SMMU_FEATURE_TT_BE;
1333 break;
1334 default:
1335 device_printf(sc->dev, "Unsupported endianness.\n");
1336 return (ENXIO);
1337 }
1338
1339 if (reg & IDR0_SEV)
1340 sc->features |= SMMU_FEATURE_SEV;
1341
1342 if (reg & IDR0_MSI) {
1343 if (bootverbose)
1344 device_printf(sc->dev, "MSI feature present.\n");
1345 sc->features |= SMMU_FEATURE_MSI;
1346 }
1347
1348 if (reg & IDR0_HYP) {
1349 if (bootverbose)
1350 device_printf(sc->dev, "HYP feature present.\n");
1351 sc->features |= SMMU_FEATURE_HYP;
1352 }
1353
1354 if (reg & IDR0_ATS)
1355 sc->features |= SMMU_FEATURE_ATS;
1356
1357 if (reg & IDR0_PRI)
1358 sc->features |= SMMU_FEATURE_PRI;
1359
1360 switch (reg & IDR0_STALL_MODEL_M) {
1361 case IDR0_STALL_MODEL_FORCE:
1362 /* Stall is forced. */
1363 sc->features |= SMMU_FEATURE_STALL_FORCE;
1364 /* FALLTHROUGH */
1365 case IDR0_STALL_MODEL_STALL:
1366 sc->features |= SMMU_FEATURE_STALL;
1367 break;
1368 }
1369
1370 /* Grab translation stages supported. */
1371 if (reg & IDR0_S1P) {
1372 if (bootverbose)
1373 device_printf(sc->dev,
1374 "Stage 1 translation supported.\n");
1375 sc->features |= SMMU_FEATURE_S1P;
1376 }
1377 if (reg & IDR0_S2P) {
1378 if (bootverbose)
1379 device_printf(sc->dev,
1380 "Stage 2 translation supported.\n");
1381 sc->features |= SMMU_FEATURE_S2P;
1382 }
1383
1384 switch (reg & IDR0_TTF_M) {
1385 case IDR0_TTF_ALL:
1386 case IDR0_TTF_AA64:
1387 sc->ias = 40;
1388 break;
1389 default:
1390 device_printf(sc->dev, "No AArch64 table format support.\n");
1391 return (ENXIO);
1392 }
1393
1394 if (reg & IDR0_ASID16)
1395 sc->asid_bits = 16;
1396 else
1397 sc->asid_bits = 8;
1398
1399 if (bootverbose)
1400 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);
1401
1402 if (reg & IDR0_VMID16)
1403 sc->vmid_bits = 16;
1404 else
1405 sc->vmid_bits = 8;
1406
1407 reg = bus_read_4(sc->res[0], SMMU_IDR1);
1408
1409 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
1410 device_printf(sc->dev,
1411 "Embedded implementations not supported by this driver.\n");
1412 return (ENXIO);
1413 }
1414
1415 val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S;
1416 sc->cmdq.size_log2 = val;
1417 if (bootverbose)
1418 device_printf(sc->dev, "CMD queue bits %d\n", val);
1419
1420 val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S;
1421 sc->evtq.size_log2 = val;
1422 if (bootverbose)
1423 device_printf(sc->dev, "EVENT queue bits %d\n", val);
1424
1425 if (sc->features & SMMU_FEATURE_PRI) {
1426 val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S;
1427 sc->priq.size_log2 = val;
1428 if (bootverbose)
1429 device_printf(sc->dev, "PRI queue bits %d\n", val);
1430 }
1431
1432 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;
1433 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;
1434
1435 if (sc->sid_bits <= STRTAB_SPLIT)
1436 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;
1437
1438 if (bootverbose) {
1439 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);
1440 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);
1441 }
1442
1443 /* IDR3 */
1444 reg = bus_read_4(sc->res[0], SMMU_IDR3);
1445 if (reg & IDR3_RIL)
1446 sc->features |= SMMU_FEATURE_RANGE_INV;
1447
1448 /* IDR5 */
1449 reg = bus_read_4(sc->res[0], SMMU_IDR5);
1450
1451 switch (reg & IDR5_OAS_M) {
1452 case IDR5_OAS_32:
1453 sc->oas = 32;
1454 break;
1455 case IDR5_OAS_36:
1456 sc->oas = 36;
1457 break;
1458 case IDR5_OAS_40:
1459 sc->oas = 40;
1460 break;
1461 case IDR5_OAS_42:
1462 sc->oas = 42;
1463 break;
1464 case IDR5_OAS_44:
1465 sc->oas = 44;
1466 break;
1467 case IDR5_OAS_48:
1468 sc->oas = 48;
1469 break;
1470 case IDR5_OAS_52:
1471 sc->oas = 52;
1472 break;
1473 }
1474
1475 sc->pgsizes = 0;
1476 if (reg & IDR5_GRAN64K)
1477 sc->pgsizes |= 64 * 1024;
1478 if (reg & IDR5_GRAN16K)
1479 sc->pgsizes |= 16 * 1024;
1480 if (reg & IDR5_GRAN4K)
1481 sc->pgsizes |= 4 * 1024;
1482
1483 if ((reg & IDR5_VAX_M) == IDR5_VAX_52)
1484 sc->features |= SMMU_FEATURE_VAX;
1485
1486 return (0);
1487 }
1488
1489 static void
smmu_init_asids(struct smmu_softc * sc)1490 smmu_init_asids(struct smmu_softc *sc)
1491 {
1492
1493 sc->asid_set_size = (1 << sc->asid_bits);
1494 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);
1495 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);
1496 }
1497
1498 static int
smmu_asid_alloc(struct smmu_softc * sc,int * new_asid)1499 smmu_asid_alloc(struct smmu_softc *sc, int *new_asid)
1500 {
1501
1502 mtx_lock_spin(&sc->asid_set_mutex);
1503 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);
1504 if (*new_asid == -1) {
1505 mtx_unlock_spin(&sc->asid_set_mutex);
1506 return (ENOMEM);
1507 }
1508 bit_set(sc->asid_set, *new_asid);
1509 mtx_unlock_spin(&sc->asid_set_mutex);
1510
1511 return (0);
1512 }
1513
1514 static void
smmu_asid_free(struct smmu_softc * sc,int asid)1515 smmu_asid_free(struct smmu_softc *sc, int asid)
1516 {
1517
1518 mtx_lock_spin(&sc->asid_set_mutex);
1519 bit_clear(sc->asid_set, asid);
1520 mtx_unlock_spin(&sc->asid_set_mutex);
1521 }
1522
1523 /*
1524 * Device interface.
1525 */
1526 int
smmu_attach(device_t dev)1527 smmu_attach(device_t dev)
1528 {
1529 struct smmu_softc *sc;
1530 int error;
1531
1532 sc = device_get_softc(dev);
1533 sc->dev = dev;
1534
1535 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);
1536
1537 error = bus_alloc_resources(dev, smmu_spec, sc->res);
1538 if (error) {
1539 device_printf(dev, "Couldn't allocate resources.\n");
1540 return (ENXIO);
1541 }
1542
1543 error = smmu_setup_interrupts(sc);
1544 if (error) {
1545 bus_release_resources(dev, smmu_spec, sc->res);
1546 return (ENXIO);
1547 }
1548
1549 error = smmu_check_features(sc);
1550 if (error) {
1551 device_printf(dev, "Some features are required "
1552 "but not supported by hardware.\n");
1553 return (ENXIO);
1554 }
1555
1556 smmu_init_asids(sc);
1557
1558 error = smmu_init_queues(sc);
1559 if (error) {
1560 device_printf(dev, "Couldn't allocate queues.\n");
1561 return (ENXIO);
1562 }
1563
1564 error = smmu_init_strtab(sc);
1565 if (error) {
1566 device_printf(dev, "Couldn't allocate strtab.\n");
1567 return (ENXIO);
1568 }
1569
1570 error = smmu_reset(sc);
1571 if (error) {
1572 device_printf(dev, "Couldn't reset SMMU.\n");
1573 return (ENXIO);
1574 }
1575
1576 return (0);
1577 }
1578
1579 int
smmu_detach(device_t dev)1580 smmu_detach(device_t dev)
1581 {
1582 struct smmu_softc *sc;
1583
1584 sc = device_get_softc(dev);
1585
1586 bus_release_resources(dev, smmu_spec, sc->res);
1587
1588 return (0);
1589 }
1590
1591 static int
smmu_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)1592 smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
1593 {
1594 struct smmu_softc *sc;
1595
1596 sc = device_get_softc(dev);
1597
1598 device_printf(sc->dev, "%s\n", __func__);
1599
1600 return (ENOENT);
1601 }
1602
1603 static int
smmu_unmap(device_t dev,struct iommu_domain * iodom,vm_offset_t va,bus_size_t size)1604 smmu_unmap(device_t dev, struct iommu_domain *iodom,
1605 vm_offset_t va, bus_size_t size)
1606 {
1607 struct smmu_domain *domain;
1608 struct smmu_softc *sc;
1609 int err;
1610 int i;
1611
1612 sc = device_get_softc(dev);
1613
1614 domain = (struct smmu_domain *)iodom;
1615
1616 err = 0;
1617
1618 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);
1619
1620 for (i = 0; i < size; i += PAGE_SIZE) {
1621 if (pmap_sremove(&domain->p, va) == 0) {
1622 /* pmap entry removed, invalidate TLB. */
1623 smmu_tlbi_va(sc, va, domain->asid);
1624 } else {
1625 err = ENOENT;
1626 break;
1627 }
1628 va += PAGE_SIZE;
1629 }
1630
1631 smmu_sync(sc);
1632
1633 return (err);
1634 }
1635
1636 static int
smmu_map(device_t dev,struct iommu_domain * iodom,vm_offset_t va,vm_page_t * ma,vm_size_t size,vm_prot_t prot)1637 smmu_map(device_t dev, struct iommu_domain *iodom,
1638 vm_offset_t va, vm_page_t *ma, vm_size_t size,
1639 vm_prot_t prot)
1640 {
1641 struct smmu_domain *domain;
1642 struct smmu_softc *sc;
1643 vm_paddr_t pa;
1644 int error;
1645 int i;
1646
1647 sc = device_get_softc(dev);
1648
1649 domain = (struct smmu_domain *)iodom;
1650
1651 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,
1652 domain->asid);
1653
1654 for (i = 0; size > 0; size -= PAGE_SIZE) {
1655 pa = VM_PAGE_TO_PHYS(ma[i++]);
1656 error = pmap_senter(&domain->p, va, pa, prot, 0);
1657 if (error)
1658 return (error);
1659 smmu_tlbi_va(sc, va, domain->asid);
1660 va += PAGE_SIZE;
1661 }
1662
1663 smmu_sync(sc);
1664
1665 return (0);
1666 }
1667
1668 static struct iommu_domain *
smmu_domain_alloc(device_t dev,struct iommu_unit * iommu)1669 smmu_domain_alloc(device_t dev, struct iommu_unit *iommu)
1670 {
1671 struct smmu_domain *domain;
1672 struct smmu_unit *unit;
1673 struct smmu_softc *sc;
1674 int error;
1675 int new_asid;
1676
1677 sc = device_get_softc(dev);
1678
1679 unit = (struct smmu_unit *)iommu;
1680
1681 domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO);
1682
1683 error = smmu_asid_alloc(sc, &new_asid);
1684 if (error) {
1685 free(domain, M_SMMU);
1686 device_printf(sc->dev,
1687 "Could not allocate ASID for a new domain.\n");
1688 return (NULL);
1689 }
1690
1691 domain->asid = (uint16_t)new_asid;
1692
1693 pmap_pinit(&domain->p);
1694 PMAP_LOCK_INIT(&domain->p);
1695
1696 error = smmu_init_cd(sc, domain);
1697 if (error) {
1698 free(domain, M_SMMU);
1699 device_printf(sc->dev, "Could not initialize CD\n");
1700 return (NULL);
1701 }
1702
1703 smmu_tlbi_asid(sc, domain->asid);
1704
1705 LIST_INIT(&domain->ctx_list);
1706
1707 IOMMU_LOCK(iommu);
1708 LIST_INSERT_HEAD(&unit->domain_list, domain, next);
1709 IOMMU_UNLOCK(iommu);
1710
1711 return (&domain->iodom);
1712 }
1713
1714 static void
smmu_domain_free(device_t dev,struct iommu_domain * iodom)1715 smmu_domain_free(device_t dev, struct iommu_domain *iodom)
1716 {
1717 struct smmu_domain *domain;
1718 struct smmu_softc *sc;
1719 struct smmu_cd *cd;
1720
1721 sc = device_get_softc(dev);
1722
1723 domain = (struct smmu_domain *)iodom;
1724
1725 LIST_REMOVE(domain, next);
1726
1727 cd = domain->cd;
1728
1729 pmap_sremove_pages(&domain->p);
1730 pmap_release(&domain->p);
1731
1732 smmu_tlbi_asid(sc, domain->asid);
1733 smmu_asid_free(sc, domain->asid);
1734
1735 contigfree(cd->vaddr, cd->size, M_SMMU);
1736 free(cd, M_SMMU);
1737
1738 free(domain, M_SMMU);
1739 }
1740
1741 static int
smmu_set_buswide(device_t dev,struct smmu_domain * domain,struct smmu_ctx * ctx)1742 smmu_set_buswide(device_t dev, struct smmu_domain *domain,
1743 struct smmu_ctx *ctx)
1744 {
1745 struct smmu_softc *sc;
1746 int i;
1747
1748 sc = device_get_softc(dev);
1749
1750 for (i = 0; i < PCI_SLOTMAX; i++)
1751 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass);
1752
1753 return (0);
1754 }
1755
1756 static struct iommu_ctx *
smmu_ctx_alloc(device_t dev,struct iommu_domain * iodom,device_t child,bool disabled)1757 smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child,
1758 bool disabled)
1759 {
1760 struct smmu_domain *domain;
1761 struct smmu_softc *sc;
1762 struct smmu_ctx *ctx;
1763 uint16_t rid;
1764 u_int xref, sid;
1765 int seg;
1766 int err;
1767
1768 sc = device_get_softc(dev);
1769 domain = (struct smmu_domain *)iodom;
1770
1771 seg = pci_get_domain(child);
1772 rid = pci_get_rid(child);
1773 err = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
1774 if (err)
1775 return (NULL);
1776
1777 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
1778 err = smmu_init_l1_entry(sc, sid);
1779 if (err)
1780 return (NULL);
1781 }
1782
1783 ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO);
1784 ctx->vendor = pci_get_vendor(child);
1785 ctx->device = pci_get_device(child);
1786 ctx->dev = child;
1787 ctx->sid = sid;
1788 ctx->domain = domain;
1789 if (disabled)
1790 ctx->bypass = true;
1791
1792 /*
1793 * Neoverse N1 SDP:
1794 * 0x800 xhci
1795 * 0x700 re
1796 * 0x600 sata
1797 */
1798
1799 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);
1800
1801 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))
1802 smmu_set_buswide(dev, domain, ctx);
1803
1804 IOMMU_DOMAIN_LOCK(iodom);
1805 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
1806 IOMMU_DOMAIN_UNLOCK(iodom);
1807
1808 return (&ctx->ioctx);
1809 }
1810
1811 static void
smmu_ctx_free(device_t dev,struct iommu_ctx * ioctx)1812 smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx)
1813 {
1814 struct smmu_softc *sc;
1815 struct smmu_ctx *ctx;
1816
1817 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
1818
1819 sc = device_get_softc(dev);
1820 ctx = (struct smmu_ctx *)ioctx;
1821
1822 smmu_deinit_l1_entry(sc, ctx->sid);
1823
1824 LIST_REMOVE(ctx, next);
1825
1826 free(ctx, M_SMMU);
1827 }
1828
1829 struct smmu_ctx *
smmu_ctx_lookup_by_sid(device_t dev,u_int sid)1830 smmu_ctx_lookup_by_sid(device_t dev, u_int sid)
1831 {
1832 struct smmu_softc *sc;
1833 struct smmu_domain *domain;
1834 struct smmu_unit *unit;
1835 struct smmu_ctx *ctx;
1836
1837 sc = device_get_softc(dev);
1838
1839 unit = &sc->unit;
1840
1841 LIST_FOREACH(domain, &unit->domain_list, next) {
1842 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1843 if (ctx->sid == sid)
1844 return (ctx);
1845 }
1846 }
1847
1848 return (NULL);
1849 }
1850
1851 static struct iommu_ctx *
smmu_ctx_lookup(device_t dev,device_t child)1852 smmu_ctx_lookup(device_t dev, device_t child)
1853 {
1854 struct iommu_unit *iommu;
1855 struct smmu_softc *sc;
1856 struct smmu_domain *domain;
1857 struct smmu_unit *unit;
1858 struct smmu_ctx *ctx;
1859
1860 sc = device_get_softc(dev);
1861
1862 unit = &sc->unit;
1863 iommu = &unit->iommu;
1864
1865 IOMMU_ASSERT_LOCKED(iommu);
1866
1867 LIST_FOREACH(domain, &unit->domain_list, next) {
1868 IOMMU_DOMAIN_LOCK(&domain->iodom);
1869 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1870 if (ctx->dev == child) {
1871 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1872 return (&ctx->ioctx);
1873 }
1874 }
1875 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1876 }
1877
1878 return (NULL);
1879 }
1880
1881 static int
smmu_find(device_t dev,device_t child)1882 smmu_find(device_t dev, device_t child)
1883 {
1884 struct smmu_softc *sc;
1885 u_int xref, sid;
1886 uint16_t rid;
1887 int error;
1888 int seg;
1889
1890 sc = device_get_softc(dev);
1891
1892 rid = pci_get_rid(child);
1893 seg = pci_get_domain(child);
1894
1895 /*
1896 * Find an xref of an IOMMU controller that serves traffic for dev.
1897 */
1898 #ifdef DEV_ACPI
1899 error = acpi_iort_map_pci_smmuv3(seg, rid, &xref, &sid);
1900 if (error) {
1901 /* Could not find reference to an SMMU device. */
1902 return (ENOENT);
1903 }
1904 #else
1905 /* TODO: add FDT support. */
1906 return (ENXIO);
1907 #endif
1908
1909 /* Check if xref is ours. */
1910 if (xref != sc->xref)
1911 return (EFAULT);
1912
1913 return (0);
1914 }
1915
1916 static device_method_t smmu_methods[] = {
1917 /* Device interface */
1918 DEVMETHOD(device_detach, smmu_detach),
1919
1920 /* SMMU interface */
1921 DEVMETHOD(iommu_find, smmu_find),
1922 DEVMETHOD(iommu_map, smmu_map),
1923 DEVMETHOD(iommu_unmap, smmu_unmap),
1924 DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc),
1925 DEVMETHOD(iommu_domain_free, smmu_domain_free),
1926 DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc),
1927 DEVMETHOD(iommu_ctx_free, smmu_ctx_free),
1928 DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup),
1929
1930 /* Bus interface */
1931 DEVMETHOD(bus_read_ivar, smmu_read_ivar),
1932
1933 /* End */
1934 DEVMETHOD_END
1935 };
1936
1937 DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc));
1938