1 /*-
2 * Copyright (c) 2015-2016 The FreeBSD Foundation
3 * Copyright (c) 2023 Arm Ltd
4 *
5 * This software was developed by Andrew Turner under
6 * the sponsorship of the FreeBSD Foundation.
7 *
8 * This software was developed by Semihalf under
9 * the sponsorship of the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include "opt_acpi.h"
34 #include "opt_platform.h"
35 #include "opt_iommu.h"
36
37 #include <sys/cdefs.h>
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/bus.h>
41 #include <sys/cpuset.h>
42 #include <sys/domainset.h>
43 #include <sys/endian.h>
44 #include <sys/kernel.h>
45 #include <sys/lock.h>
46 #include <sys/malloc.h>
47 #include <sys/module.h>
48 #include <sys/mutex.h>
49 #include <sys/physmem.h>
50 #include <sys/proc.h>
51 #include <sys/taskqueue.h>
52 #include <sys/tree.h>
53 #include <sys/queue.h>
54 #include <sys/rman.h>
55 #include <sys/sbuf.h>
56 #include <sys/smp.h>
57 #include <sys/sysctl.h>
58 #include <sys/vmem.h>
59
60 #include <vm/vm.h>
61 #include <vm/pmap.h>
62 #include <vm/vm_page.h>
63
64 #include <machine/bus.h>
65 #include <machine/intr.h>
66
67 #include <arm/arm/gic_common.h>
68 #include <arm64/arm64/gic_v3_reg.h>
69 #include <arm64/arm64/gic_v3_var.h>
70
71 #ifdef FDT
72 #include <dev/ofw/openfirm.h>
73 #include <dev/ofw/ofw_bus.h>
74 #include <dev/ofw/ofw_bus_subr.h>
75 #endif
76 #include <dev/pci/pcireg.h>
77 #include <dev/pci/pcivar.h>
78
79 #ifdef IOMMU
80 #include <dev/iommu/iommu.h>
81 #include <dev/iommu/iommu_gas.h>
82 #endif
83
84 #include "pcib_if.h"
85 #include "pic_if.h"
86 #include "msi_if.h"
87
88 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
89 "ARM GICv3 Interrupt Translation Service");
90
91 #define LPI_NIRQS (64 * 1024)
92
93 /* The size and alignment of the command circular buffer */
94 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */
95 #define ITS_CMDQ_ALIGN (64 * 1024)
96
97 #define LPI_CONFTAB_SIZE LPI_NIRQS
98 #define LPI_CONFTAB_ALIGN (64 * 1024)
99 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
100
101 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
102 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
103 #define LPI_PENDTAB_ALIGN (64 * 1024)
104 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
105
106 #define LPI_INT_TRANS_TAB_ALIGN 256
107 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
108
109 /* ITS commands encoding */
110 #define ITS_CMD_MOVI (0x01)
111 #define ITS_CMD_SYNC (0x05)
112 #define ITS_CMD_MAPD (0x08)
113 #define ITS_CMD_MAPC (0x09)
114 #define ITS_CMD_MAPTI (0x0a)
115 #define ITS_CMD_MAPI (0x0b)
116 #define ITS_CMD_INV (0x0c)
117 #define ITS_CMD_INVALL (0x0d)
118 /* Command */
119 #define CMD_COMMAND_MASK (0xFFUL)
120 /* PCI device ID */
121 #define CMD_DEVID_SHIFT (32)
122 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
123 /* Size of IRQ ID bitfield */
124 #define CMD_SIZE_MASK (0xFFUL)
125 /* Virtual LPI ID */
126 #define CMD_ID_MASK (0xFFFFFFFFUL)
127 /* Physical LPI ID */
128 #define CMD_PID_SHIFT (32)
129 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT)
130 /* Collection */
131 #define CMD_COL_MASK (0xFFFFUL)
132 /* Target (CPU or Re-Distributor) */
133 #define CMD_TARGET_SHIFT (16)
134 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
135 /* Interrupt Translation Table address */
136 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL)
137 /* Valid command bit */
138 #define CMD_VALID_SHIFT (63)
139 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT)
140
141 #define ITS_TARGET_NONE 0xFBADBEEF
142
143 /* LPI chunk owned by ITS device */
144 struct lpi_chunk {
145 u_int lpi_base;
146 u_int lpi_free; /* First free LPI in set */
147 u_int lpi_num; /* Total number of LPIs in chunk */
148 u_int lpi_busy; /* Number of busy LPIs in chink */
149 };
150
151 /* ITS device */
152 struct its_dev {
153 TAILQ_ENTRY(its_dev) entry;
154 /* PCI device */
155 device_t pci_dev;
156 /* Device ID (i.e. PCI device ID) */
157 uint32_t devid;
158 /* List of assigned LPIs */
159 struct lpi_chunk lpis;
160 /* Virtual address of ITT */
161 vm_offset_t itt;
162 size_t itt_size;
163 };
164
165 /*
166 * ITS command descriptor.
167 * Idea for command description passing taken from Linux.
168 */
169 struct its_cmd_desc {
170 uint8_t cmd_type;
171
172 union {
173 struct {
174 struct its_dev *its_dev;
175 struct its_col *col;
176 uint32_t id;
177 } cmd_desc_movi;
178
179 struct {
180 struct its_col *col;
181 } cmd_desc_sync;
182
183 struct {
184 struct its_col *col;
185 uint8_t valid;
186 } cmd_desc_mapc;
187
188 struct {
189 struct its_dev *its_dev;
190 struct its_col *col;
191 uint32_t pid;
192 uint32_t id;
193 } cmd_desc_mapvi;
194
195 struct {
196 struct its_dev *its_dev;
197 struct its_col *col;
198 uint32_t pid;
199 } cmd_desc_mapi;
200
201 struct {
202 struct its_dev *its_dev;
203 uint8_t valid;
204 } cmd_desc_mapd;
205
206 struct {
207 struct its_dev *its_dev;
208 struct its_col *col;
209 uint32_t pid;
210 } cmd_desc_inv;
211
212 struct {
213 struct its_col *col;
214 } cmd_desc_invall;
215 };
216 };
217
218 /* ITS command. Each command is 32 bytes long */
219 struct its_cmd {
220 uint64_t cmd_dword[4]; /* ITS command double word */
221 };
222
223 /* An ITS private table */
224 struct its_ptable {
225 vm_offset_t ptab_vaddr;
226 /* Size of the L1 and L2 tables */
227 size_t ptab_l1_size;
228 size_t ptab_l2_size;
229 /* Number of L1 and L2 entries */
230 int ptab_l1_nidents;
231 int ptab_l2_nidents;
232
233 int ptab_page_size;
234 int ptab_share;
235 bool ptab_indirect;
236 };
237
238 /* ITS collection description. */
239 struct its_col {
240 uint64_t col_target; /* Target Re-Distributor */
241 uint64_t col_id; /* Collection ID */
242 };
243
244 struct gicv3_its_irqsrc {
245 struct intr_irqsrc gi_isrc;
246 u_int gi_id;
247 u_int gi_lpi;
248 struct its_dev *gi_its_dev;
249 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
250 };
251
252 struct gicv3_its_softc {
253 device_t dev;
254 struct intr_pic *sc_pic;
255 struct resource *sc_its_res;
256
257 cpuset_t sc_cpus;
258 struct domainset *sc_ds;
259 u_int gic_irq_cpu;
260 int sc_devbits;
261 int sc_dev_table_idx;
262
263 struct its_ptable sc_its_ptab[GITS_BASER_NUM];
264 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */
265
266 /*
267 * TODO: We should get these from the parent as we only want a
268 * single copy of each across the interrupt controller.
269 */
270 uint8_t *sc_conf_base;
271 vm_offset_t sc_pend_base[MAXCPU];
272
273 /* Command handling */
274 struct mtx sc_its_cmd_lock;
275 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
276 size_t sc_its_cmd_next_idx;
277
278 vmem_t *sc_irq_alloc;
279 struct gicv3_its_irqsrc **sc_irqs;
280 u_int sc_irq_base;
281 u_int sc_irq_length;
282 u_int sc_irq_count;
283
284 struct mtx sc_its_dev_lock;
285 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
286 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
287
288 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001
289 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002
290 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004
291 #define ITS_FLAGS_LPI_PREALLOC 0x00000008
292 u_int sc_its_flags;
293 bool trace_enable;
294 vm_page_t ma; /* fake msi page */
295 };
296
297 typedef void (its_quirk_func_t)(device_t);
298 static its_quirk_func_t its_quirk_cavium_22375;
299
300 static const struct {
301 const char *desc;
302 uint32_t iidr;
303 uint32_t iidr_mask;
304 its_quirk_func_t *func;
305 } its_quirks[] = {
306 {
307 /* Cavium ThunderX Pass 1.x */
308 .desc = "Cavium ThunderX errata: 22375, 24313",
309 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
310 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
311 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
312 .func = its_quirk_cavium_22375,
313 },
314 };
315
316 #define gic_its_read_4(sc, reg) \
317 bus_read_4((sc)->sc_its_res, (reg))
318 #define gic_its_read_8(sc, reg) \
319 bus_read_8((sc)->sc_its_res, (reg))
320
321 #define gic_its_write_4(sc, reg, val) \
322 bus_write_4((sc)->sc_its_res, (reg), (val))
323 #define gic_its_write_8(sc, reg, val) \
324 bus_write_8((sc)->sc_its_res, (reg), (val))
325
326 static device_attach_t gicv3_its_attach;
327 static device_detach_t gicv3_its_detach;
328
329 static pic_disable_intr_t gicv3_its_disable_intr;
330 static pic_enable_intr_t gicv3_its_enable_intr;
331 static pic_map_intr_t gicv3_its_map_intr;
332 static pic_setup_intr_t gicv3_its_setup_intr;
333 static pic_post_filter_t gicv3_its_post_filter;
334 static pic_post_ithread_t gicv3_its_post_ithread;
335 static pic_pre_ithread_t gicv3_its_pre_ithread;
336 static pic_bind_intr_t gicv3_its_bind_intr;
337 #ifdef SMP
338 static pic_init_secondary_t gicv3_its_init_secondary;
339 #endif
340 static msi_alloc_msi_t gicv3_its_alloc_msi;
341 static msi_release_msi_t gicv3_its_release_msi;
342 static msi_alloc_msix_t gicv3_its_alloc_msix;
343 static msi_release_msix_t gicv3_its_release_msix;
344 static msi_map_msi_t gicv3_its_map_msi;
345 #ifdef IOMMU
346 static msi_iommu_init_t gicv3_iommu_init;
347 static msi_iommu_deinit_t gicv3_iommu_deinit;
348 #endif
349
350 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
351 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
352 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
353 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
354 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
355 static void its_cmd_invall(device_t, struct its_col *);
356
357 static device_method_t gicv3_its_methods[] = {
358 /* Device interface */
359 DEVMETHOD(device_detach, gicv3_its_detach),
360
361 /* Interrupt controller interface */
362 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr),
363 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr),
364 DEVMETHOD(pic_map_intr, gicv3_its_map_intr),
365 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr),
366 DEVMETHOD(pic_post_filter, gicv3_its_post_filter),
367 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread),
368 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread),
369 #ifdef SMP
370 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr),
371 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary),
372 #endif
373
374 /* MSI/MSI-X */
375 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi),
376 DEVMETHOD(msi_release_msi, gicv3_its_release_msi),
377 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix),
378 DEVMETHOD(msi_release_msix, gicv3_its_release_msix),
379 DEVMETHOD(msi_map_msi, gicv3_its_map_msi),
380 #ifdef IOMMU
381 DEVMETHOD(msi_iommu_init, gicv3_iommu_init),
382 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit),
383 #endif
384
385 /* End */
386 DEVMETHOD_END
387 };
388
389 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
390 sizeof(struct gicv3_its_softc));
391
392 static void
gicv3_its_cmdq_init(struct gicv3_its_softc * sc)393 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
394 {
395 vm_paddr_t cmd_paddr;
396 uint64_t reg, tmp;
397
398 /* Set up the command circular buffer */
399 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
400 sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
401 0);
402 sc->sc_its_cmd_next_idx = 0;
403
404 cmd_paddr = vtophys(sc->sc_its_cmd_base);
405
406 /* Set the base of the command buffer */
407 reg = GITS_CBASER_VALID |
408 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
409 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
410 (ITS_CMDQ_SIZE / 4096 - 1);
411 gic_its_write_8(sc, GITS_CBASER, reg);
412
413 /* Read back to check for fixed value fields */
414 tmp = gic_its_read_8(sc, GITS_CBASER);
415
416 if ((tmp & GITS_CBASER_SHARE_MASK) !=
417 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
418 /* Check if the hardware reported non-shareable */
419 if ((tmp & GITS_CBASER_SHARE_MASK) ==
420 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
421 /* If so remove the cache attribute */
422 reg &= ~GITS_CBASER_CACHE_MASK;
423 reg &= ~GITS_CBASER_SHARE_MASK;
424 /* Set to Non-cacheable, Non-shareable */
425 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
426 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
427
428 gic_its_write_8(sc, GITS_CBASER, reg);
429 }
430
431 /* The command queue has to be flushed after each command */
432 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
433 }
434
435 /* Get the next command from the start of the buffer */
436 gic_its_write_8(sc, GITS_CWRITER, 0x0);
437 }
438
439 static int
gicv3_its_table_page_size(struct gicv3_its_softc * sc,int table)440 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
441 {
442 uint64_t reg, tmp;
443 int page_size;
444
445 page_size = PAGE_SIZE_64K;
446 reg = gic_its_read_8(sc, GITS_BASER(table));
447
448 while (1) {
449 reg &= GITS_BASER_PSZ_MASK;
450 switch (page_size) {
451 case PAGE_SIZE_4K: /* 4KB */
452 reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
453 break;
454 case PAGE_SIZE_16K: /* 16KB */
455 reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
456 break;
457 case PAGE_SIZE_64K: /* 64KB */
458 reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
459 break;
460 }
461
462 /* Write the new page size */
463 gic_its_write_8(sc, GITS_BASER(table), reg);
464
465 /* Read back to check */
466 tmp = gic_its_read_8(sc, GITS_BASER(table));
467
468 /* The page size is correct */
469 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
470 return (page_size);
471
472 switch (page_size) {
473 default:
474 return (-1);
475 case PAGE_SIZE_16K:
476 page_size = PAGE_SIZE_4K;
477 break;
478 case PAGE_SIZE_64K:
479 page_size = PAGE_SIZE_16K;
480 break;
481 }
482 }
483 }
484
485 static bool
gicv3_its_table_supports_indirect(struct gicv3_its_softc * sc,int table)486 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table)
487 {
488 uint64_t reg;
489
490 reg = gic_its_read_8(sc, GITS_BASER(table));
491
492 /* Try setting the indirect flag */
493 reg |= GITS_BASER_INDIRECT;
494 gic_its_write_8(sc, GITS_BASER(table), reg);
495
496 /* Read back to check */
497 reg = gic_its_read_8(sc, GITS_BASER(table));
498 return ((reg & GITS_BASER_INDIRECT) != 0);
499 }
500
501
502 static int
gicv3_its_table_init(device_t dev,struct gicv3_its_softc * sc)503 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
504 {
505 vm_offset_t table;
506 vm_paddr_t paddr;
507 uint64_t cache, reg, share, tmp, type;
508 size_t its_tbl_size, nitspages, npages;
509 size_t l1_esize, l2_esize, l1_nidents, l2_nidents;
510 int i, page_size;
511 int devbits;
512 bool indirect;
513
514 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
515 /*
516 * GITS_TYPER[17:13] of ThunderX reports that device IDs
517 * are to be 21 bits in length. The entry size of the ITS
518 * table can be read from GITS_BASERn[52:48] and on ThunderX
519 * is supposed to be 8 bytes in length (for device table).
520 * Finally the page size that is to be used by ITS to access
521 * this table will be set to 64KB.
522 *
523 * This gives 0x200000 entries of size 0x8 bytes covered by
524 * 256 pages each of which 64KB in size. The number of pages
525 * (minus 1) should then be written to GITS_BASERn[7:0]. In
526 * that case this value would be 0xFF but on ThunderX the
527 * maximum value that HW accepts is 0xFD.
528 *
529 * Set an arbitrary number of device ID bits to 20 in order
530 * to limit the number of entries in ITS device table to
531 * 0x100000 and the table size to 8MB.
532 */
533 devbits = 20;
534 cache = 0;
535 } else {
536 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
537 cache = GITS_BASER_CACHE_WAWB;
538 }
539 sc->sc_devbits = devbits;
540 share = GITS_BASER_SHARE_IS;
541
542 for (i = 0; i < GITS_BASER_NUM; i++) {
543 reg = gic_its_read_8(sc, GITS_BASER(i));
544 /* The type of table */
545 type = GITS_BASER_TYPE(reg);
546 if (type == GITS_BASER_TYPE_UNIMPL)
547 continue;
548
549 /* The table entry size */
550 l1_esize = GITS_BASER_ESIZE(reg);
551
552 /* Find the tables page size */
553 page_size = gicv3_its_table_page_size(sc, i);
554 if (page_size == -1) {
555 device_printf(dev, "No valid page size for table %d\n",
556 i);
557 return (EINVAL);
558 }
559
560 indirect = false;
561 l2_nidents = 0;
562 l2_esize = 0;
563 switch(type) {
564 case GITS_BASER_TYPE_DEV:
565 if (sc->sc_dev_table_idx != -1)
566 device_printf(dev,
567 "Warning: Multiple device tables found\n");
568
569 sc->sc_dev_table_idx = i;
570 l1_nidents = (1 << devbits);
571 if ((l1_esize * l1_nidents) > (page_size * 2)) {
572 indirect =
573 gicv3_its_table_supports_indirect(sc, i);
574 if (indirect) {
575 /*
576 * Each l1 entry is 8 bytes and points
577 * to an l2 table of size page_size.
578 * Calculate how many entries this is
579 * and use this to find how many
580 * 8 byte l1 idents we need.
581 */
582 l2_esize = l1_esize;
583 l2_nidents = page_size / l2_esize;
584 l1_nidents = l1_nidents / l2_nidents;
585 l1_esize = GITS_INDIRECT_L1_ESIZE;
586 }
587 }
588 its_tbl_size = l1_esize * l1_nidents;
589 its_tbl_size = roundup2(its_tbl_size, page_size);
590 break;
591 case GITS_BASER_TYPE_VP:
592 case GITS_BASER_TYPE_PP: /* Undocumented? */
593 case GITS_BASER_TYPE_IC:
594 its_tbl_size = page_size;
595 break;
596 default:
597 if (bootverbose)
598 device_printf(dev, "Unhandled table type %lx\n",
599 type);
600 continue;
601 }
602 npages = howmany(its_tbl_size, PAGE_SIZE);
603
604 /* Allocate the table */
605 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE,
606 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
607 (1ul << 48) - 1, PAGE_SIZE_64K, 0);
608
609 sc->sc_its_ptab[i].ptab_vaddr = table;
610 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size;
611 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents;
612 sc->sc_its_ptab[i].ptab_l2_size = page_size;
613 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents;
614
615 sc->sc_its_ptab[i].ptab_indirect = indirect;
616 sc->sc_its_ptab[i].ptab_page_size = page_size;
617
618 paddr = vtophys(table);
619
620 while (1) {
621 nitspages = howmany(its_tbl_size, page_size);
622
623 /* Clear the fields we will be setting */
624 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
625 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
626 GITS_BASER_PA_MASK |
627 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
628 GITS_BASER_SIZE_MASK);
629 /* Set the new values */
630 reg |= GITS_BASER_VALID |
631 (indirect ? GITS_BASER_INDIRECT : 0) |
632 (cache << GITS_BASER_CACHE_SHIFT) |
633 (type << GITS_BASER_TYPE_SHIFT) |
634 paddr | (share << GITS_BASER_SHARE_SHIFT) |
635 (nitspages - 1);
636
637 switch (page_size) {
638 case PAGE_SIZE_4K: /* 4KB */
639 reg |=
640 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
641 break;
642 case PAGE_SIZE_16K: /* 16KB */
643 reg |=
644 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
645 break;
646 case PAGE_SIZE_64K: /* 64KB */
647 reg |=
648 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
649 break;
650 }
651
652 gic_its_write_8(sc, GITS_BASER(i), reg);
653
654 /* Read back to check */
655 tmp = gic_its_read_8(sc, GITS_BASER(i));
656
657 /* Do the shareability masks line up? */
658 if ((tmp & GITS_BASER_SHARE_MASK) !=
659 (reg & GITS_BASER_SHARE_MASK)) {
660 share = (tmp & GITS_BASER_SHARE_MASK) >>
661 GITS_BASER_SHARE_SHIFT;
662 continue;
663 }
664
665 if (tmp != reg) {
666 device_printf(dev, "GITS_BASER%d: "
667 "unable to be updated: %lx != %lx\n",
668 i, reg, tmp);
669 return (ENXIO);
670 }
671
672 sc->sc_its_ptab[i].ptab_share = share;
673 /* We should have made all needed changes */
674 break;
675 }
676 }
677
678 return (0);
679 }
680
681 static void
gicv3_its_conftable_init(struct gicv3_its_softc * sc)682 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
683 {
684 /* note: we assume the ITS children are serialized by the parent */
685 static void *conf_table;
686 int extra_flags = 0;
687 device_t gicv3;
688 uint32_t ctlr;
689 vm_paddr_t conf_pa;
690 vm_offset_t conf_va;
691
692 /*
693 * The PROPBASER is a singleton in our parent. We only set it up the
694 * first time through. conf_table is effectively global to all the units
695 * and we rely on subr_bus to serialize probe/attach.
696 */
697 if (conf_table != NULL) {
698 sc->sc_conf_base = conf_table;
699 return;
700 }
701
702 gicv3 = device_get_parent(sc->dev);
703 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
704 if ((ctlr & GICR_CTLR_LPI_ENABLE) != 0) {
705 conf_pa = gic_r_read_8(gicv3, GICR_PROPBASER);
706 conf_pa &= GICR_PROPBASER_PA_MASK;
707 /*
708 * If there was a pre-existing PROPBASER, then we need to honor
709 * it because implemenetation defined behavior in gicv3 makes it
710 * impossible to quiesce to change it out. We will only see a
711 * pre-existing one when we've been kexec'd from a Linux kernel,
712 * or from a LinuxBoot environment.
713 *
714 * Linux provides us with a MEMRESERVE table that we put into
715 * the excluded physmem area. If PROPBASER isn't in this tabke,
716 * the system cannot run due to random memory corruption,
717 * so we panic for this case.
718 */
719 if (!physmem_excluded(conf_pa, LPI_CONFTAB_SIZE))
720 panic("gicv3 PROPBASER needs to reuse %#lx, but not reserved\n",
721 conf_pa);
722 conf_va = PHYS_TO_DMAP(conf_pa);
723 if (!pmap_klookup(conf_va, NULL))
724 panic("Can't mapped prior LPI mapping into VA\n");
725 conf_table = (void *)conf_va;
726 extra_flags = ITS_FLAGS_LPI_PREALLOC | ITS_FLAGS_LPI_CONF_FLUSH;
727 if (bootverbose)
728 device_printf(sc->dev,
729 "LPI enabled, conf table using pa %#lx va %lx\n",
730 conf_pa, conf_va);
731 } else {
732
733 /*
734 * Otherwise just allocate contiguous pages. We'll configure the
735 * PROPBASER register later in its_init_cpu_lpi().
736 */
737 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
738 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
739 LPI_CONFTAB_ALIGN, 0);
740 }
741 sc->sc_conf_base = conf_table;
742 sc->sc_its_flags |= extra_flags;
743
744 /* Set the default configuration */
745 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
746 LPI_CONFTAB_SIZE);
747
748 /* Flush the table to memory */
749 cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE);
750 }
751
752 static void
gicv3_its_pendtables_init(struct gicv3_its_softc * sc)753 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
754 {
755
756 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
757 for (int i = 0; i <= mp_maxid; i++) {
758 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
759 continue;
760
761 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc(
762 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
763 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
764
765 /* Flush so the ITS can see the memory */
766 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i],
767 LPI_PENDTAB_SIZE);
768 }
769 }
770 }
771
772 static void
its_init_cpu_lpi(device_t dev,struct gicv3_its_softc * sc)773 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
774 {
775 device_t gicv3;
776 uint64_t xbaser, tmp, size;
777 uint32_t ctlr;
778 u_int cpuid;
779
780 gicv3 = device_get_parent(dev);
781 cpuid = PCPU_GET(cpuid);
782
783 /*
784 * Set the redistributor base. If we're reusing what we found on boot
785 * since the gic was already running, then don't touch it here. We also
786 * don't need to disable / enable LPI if we're not changing PROPBASER,
787 * so only do that if we're not prealloced.
788 */
789 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
790 /* Disable LPIs */
791 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
792 ctlr &= ~GICR_CTLR_LPI_ENABLE;
793 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
794
795 /* Make sure changes are observable my the GIC */
796 dsb(sy);
797
798 size = (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
799
800 xbaser = vtophys(sc->sc_conf_base) |
801 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
802 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
803 size;
804
805 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
806
807 /* Check the cache attributes we set */
808 tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
809
810 if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
811 (xbaser & GICR_PROPBASER_SHARE_MASK)) {
812 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
813 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
814 /* We need to mark as non-cacheable */
815 xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
816 GICR_PROPBASER_CACHE_MASK);
817 /* Non-cacheable */
818 xbaser |= GICR_PROPBASER_CACHE_NIN <<
819 GICR_PROPBASER_CACHE_SHIFT;
820 /* Non-shareable */
821 xbaser |= GICR_PROPBASER_SHARE_NS <<
822 GICR_PROPBASER_SHARE_SHIFT;
823 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
824 }
825 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
826 }
827
828 /*
829 * Set the LPI pending table base
830 */
831 xbaser = vtophys(sc->sc_pend_base[cpuid]) |
832 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
833 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
834
835 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
836
837 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
838
839 if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
840 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
841 /* Clear the cahce and shareability bits */
842 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
843 GICR_PENDBASER_SHARE_MASK);
844 /* Mark as non-shareable */
845 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
846 /* And non-cacheable */
847 xbaser |= GICR_PENDBASER_CACHE_NIN <<
848 GICR_PENDBASER_CACHE_SHIFT;
849 }
850
851 /* Enable LPIs */
852 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
853 ctlr |= GICR_CTLR_LPI_ENABLE;
854 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
855
856 /* Make sure the GIC has seen everything */
857 dsb(sy);
858 } else {
859 KASSERT(sc->sc_pend_base[cpuid] == 0,
860 ("PREALLOC too soon cpuid %d", cpuid));
861 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
862 tmp &= GICR_PENDBASER_PA_MASK;
863 if (!physmem_excluded(tmp, LPI_PENDTAB_SIZE))
864 panic("gicv3 PENDBASER on cpu %d needs to reuse 0x%#lx, but not reserved\n",
865 cpuid, tmp);
866 sc->sc_pend_base[cpuid] = PHYS_TO_DMAP(tmp);
867 }
868
869
870 if (bootverbose)
871 device_printf(gicv3, "using %sPENDBASE of %#lx on cpu %d\n",
872 (sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) ? "pre-existing " : "",
873 vtophys(sc->sc_pend_base[cpuid]), cpuid);
874 }
875
876 static int
its_init_cpu(device_t dev,struct gicv3_its_softc * sc)877 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
878 {
879 device_t gicv3;
880 vm_paddr_t target;
881 u_int cpuid;
882 struct redist_pcpu *rpcpu;
883
884 gicv3 = device_get_parent(dev);
885 cpuid = PCPU_GET(cpuid);
886 if (!CPU_ISSET(cpuid, &sc->sc_cpus))
887 return (0);
888
889 /* Check if the ITS is enabled on this CPU */
890 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
891 return (ENXIO);
892
893 rpcpu = gicv3_get_redist(dev);
894
895 /* Do per-cpu LPI init once */
896 if (!rpcpu->lpi_enabled) {
897 its_init_cpu_lpi(dev, sc);
898 rpcpu->lpi_enabled = true;
899 }
900
901 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
902 /* This ITS wants the redistributor physical address */
903 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
904 rpcpu->offset);
905 } else {
906 /* This ITS wants the unique processor number */
907 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
908 CMD_TARGET_SHIFT;
909 }
910
911 sc->sc_its_cols[cpuid]->col_target = target;
912 sc->sc_its_cols[cpuid]->col_id = cpuid;
913
914 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
915 its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
916
917 return (0);
918 }
919
920 static int
gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)921 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
922 {
923 struct gicv3_its_softc *sc;
924 int rv;
925
926 sc = arg1;
927
928 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
929 if (rv != 0 || req->newptr == NULL)
930 return (rv);
931 if (sc->trace_enable)
932 gic_its_write_8(sc, GITS_TRKCTLR, 3);
933 else
934 gic_its_write_8(sc, GITS_TRKCTLR, 0);
935
936 return (0);
937 }
938
939 static int
gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)940 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
941 {
942 struct gicv3_its_softc *sc;
943 struct sbuf *sb;
944 int err;
945
946 sc = arg1;
947 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
948 if (sb == NULL) {
949 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
950 return (ENOMEM);
951 }
952 sbuf_cat(sb, "\n");
953 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
954 gic_its_read_4(sc, GITS_TRKCTLR));
955 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n",
956 gic_its_read_4(sc, GITS_TRKR));
957 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
958 gic_its_read_4(sc, GITS_TRKDIDR));
959 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
960 gic_its_read_4(sc, GITS_TRKPIDR));
961 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
962 gic_its_read_4(sc, GITS_TRKVIDR));
963 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
964 gic_its_read_4(sc, GITS_TRKTGTR));
965
966 err = sbuf_finish(sb);
967 if (err)
968 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
969 sbuf_delete(sb);
970 return(err);
971 }
972
973 static int
gicv3_its_init_sysctl(struct gicv3_its_softc * sc)974 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
975 {
976 struct sysctl_oid *oid, *child;
977 struct sysctl_ctx_list *ctx_list;
978
979 ctx_list = device_get_sysctl_ctx(sc->dev);
980 child = device_get_sysctl_tree(sc->dev);
981 oid = SYSCTL_ADD_NODE(ctx_list,
982 SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
983 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
984 if (oid == NULL)
985 return (ENXIO);
986
987 /* Add registers */
988 SYSCTL_ADD_PROC(ctx_list,
989 SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
990 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
991 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
992 SYSCTL_ADD_PROC(ctx_list,
993 SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
994 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
995 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
996
997 return (0);
998 }
999
1000 static int
gicv3_its_attach(device_t dev)1001 gicv3_its_attach(device_t dev)
1002 {
1003 struct gicv3_its_softc *sc;
1004 int domain, err, i, rid;
1005 uint64_t phys;
1006 uint32_t ctlr, iidr;
1007
1008 sc = device_get_softc(dev);
1009
1010 sc->sc_dev_table_idx = -1;
1011 sc->sc_irq_length = gicv3_get_nirqs(dev);
1012 sc->sc_irq_base = GIC_FIRST_LPI;
1013 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
1014
1015 rid = 0;
1016 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1017 RF_ACTIVE);
1018 if (sc->sc_its_res == NULL) {
1019 device_printf(dev, "Could not allocate memory\n");
1020 return (ENXIO);
1021 }
1022
1023 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
1024 GITS_TRANSLATER, PAGE_SIZE);
1025 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
1026 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
1027
1028 CPU_COPY(&all_cpus, &sc->sc_cpus);
1029 iidr = gic_its_read_4(sc, GITS_IIDR);
1030 for (i = 0; i < nitems(its_quirks); i++) {
1031 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
1032 if (bootverbose) {
1033 device_printf(dev, "Applying %s\n",
1034 its_quirks[i].desc);
1035 }
1036 its_quirks[i].func(dev);
1037 break;
1038 }
1039 }
1040
1041 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
1042 sc->sc_ds = DOMAINSET_PREF(domain);
1043 } else {
1044 sc->sc_ds = DOMAINSET_RR();
1045 }
1046
1047 /*
1048 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
1049 * coming in via, for instance, a kexec/kboot style setup where a
1050 * previous kernel has configured then relinquished control. Clear it
1051 * so that we can reconfigure GITS_BASER*.
1052 */
1053 ctlr = gic_its_read_4(sc, GITS_CTLR);
1054 if ((ctlr & GITS_CTLR_EN) != 0) {
1055 ctlr &= ~GITS_CTLR_EN;
1056 gic_its_write_4(sc, GITS_CTLR, ctlr);
1057 }
1058
1059 /* Allocate the private tables */
1060 err = gicv3_its_table_init(dev, sc);
1061 if (err != 0)
1062 return (err);
1063
1064 /* Protects access to the device list */
1065 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
1066
1067 /* Protects access to the ITS command circular buffer. */
1068 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
1069
1070 /* Allocate the command circular buffer */
1071 gicv3_its_cmdq_init(sc);
1072
1073 /* Allocate the per-CPU collections */
1074 for (int cpu = 0; cpu <= mp_maxid; cpu++)
1075 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
1076 sc->sc_its_cols[cpu] = malloc_domainset(
1077 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
1078 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
1079 M_WAITOK | M_ZERO);
1080 else
1081 sc->sc_its_cols[cpu] = NULL;
1082
1083 /* Enable the ITS */
1084 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
1085
1086 /* Create the LPI configuration table */
1087 gicv3_its_conftable_init(sc);
1088
1089 /* And the pending tebles */
1090 gicv3_its_pendtables_init(sc);
1091
1092 /* Enable LPIs on this CPU */
1093 its_init_cpu(dev, sc);
1094
1095 TAILQ_INIT(&sc->sc_its_dev_list);
1096 TAILQ_INIT(&sc->sc_free_irqs);
1097
1098 /*
1099 * Create the vmem object to allocate INTRNG IRQs from. We try to
1100 * use all IRQs not already used by the GICv3.
1101 * XXX: This assumes there are no other interrupt controllers in the
1102 * system.
1103 */
1104 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
1105 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
1106
1107 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
1108 M_GICV3_ITS, M_WAITOK | M_ZERO);
1109
1110 /* For GIC-500 install tracking sysctls. */
1111 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
1112 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
1113 gicv3_its_init_sysctl(sc);
1114
1115 return (0);
1116 }
1117
1118 static int
gicv3_its_detach(device_t dev)1119 gicv3_its_detach(device_t dev)
1120 {
1121
1122 return (ENXIO);
1123 }
1124
1125 static void
its_quirk_cavium_22375(device_t dev)1126 its_quirk_cavium_22375(device_t dev)
1127 {
1128 struct gicv3_its_softc *sc;
1129 int domain;
1130
1131 sc = device_get_softc(dev);
1132 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
1133
1134 /*
1135 * We need to limit which CPUs we send these interrupts to on
1136 * the original dual socket ThunderX as it is unable to
1137 * forward them between the two sockets.
1138 */
1139 if (bus_get_domain(dev, &domain) == 0) {
1140 if (domain < MAXMEMDOM) {
1141 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1142 } else {
1143 CPU_ZERO(&sc->sc_cpus);
1144 }
1145 }
1146 }
1147
1148 static void
gicv3_its_disable_intr(device_t dev,struct intr_irqsrc * isrc)1149 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1150 {
1151 struct gicv3_its_softc *sc;
1152 struct gicv3_its_irqsrc *girq;
1153 uint8_t *conf;
1154
1155 sc = device_get_softc(dev);
1156 girq = (struct gicv3_its_irqsrc *)isrc;
1157 conf = sc->sc_conf_base;
1158
1159 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1160
1161 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1162 /* Clean D-cache under command. */
1163 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1164 } else {
1165 /* DSB inner shareable, store */
1166 dsb(ishst);
1167 }
1168
1169 its_cmd_inv(dev, girq->gi_its_dev, girq);
1170 }
1171
1172 static void
gicv3_its_enable_intr(device_t dev,struct intr_irqsrc * isrc)1173 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1174 {
1175 struct gicv3_its_softc *sc;
1176 struct gicv3_its_irqsrc *girq;
1177 uint8_t *conf;
1178
1179 sc = device_get_softc(dev);
1180 girq = (struct gicv3_its_irqsrc *)isrc;
1181 conf = sc->sc_conf_base;
1182
1183 conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1184
1185 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1186 /* Clean D-cache under command. */
1187 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1);
1188 } else {
1189 /* DSB inner shareable, store */
1190 dsb(ishst);
1191 }
1192
1193 its_cmd_inv(dev, girq->gi_its_dev, girq);
1194 }
1195
1196 static int
gicv3_its_intr(void * arg,uintptr_t irq)1197 gicv3_its_intr(void *arg, uintptr_t irq)
1198 {
1199 struct gicv3_its_softc *sc = arg;
1200 struct gicv3_its_irqsrc *girq;
1201 struct trapframe *tf;
1202
1203 irq -= sc->sc_irq_base;
1204 girq = sc->sc_irqs[irq];
1205 if (girq == NULL)
1206 panic("gicv3_its_intr: Invalid interrupt %ld",
1207 irq + sc->sc_irq_base);
1208
1209 tf = curthread->td_intr_frame;
1210 intr_isrc_dispatch(&girq->gi_isrc, tf);
1211 return (FILTER_HANDLED);
1212 }
1213
1214 static void
gicv3_its_pre_ithread(device_t dev,struct intr_irqsrc * isrc)1215 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1216 {
1217 struct gicv3_its_irqsrc *girq;
1218
1219 girq = (struct gicv3_its_irqsrc *)isrc;
1220 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1221 }
1222
1223 static void
gicv3_its_post_ithread(device_t dev,struct intr_irqsrc * isrc)1224 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1225 {
1226
1227 }
1228
1229 static void
gicv3_its_post_filter(device_t dev,struct intr_irqsrc * isrc)1230 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1231 {
1232 struct gicv3_its_irqsrc *girq;
1233
1234 girq = (struct gicv3_its_irqsrc *)isrc;
1235 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1236 }
1237
1238 static int
gicv3_its_select_cpu(device_t dev,struct intr_irqsrc * isrc)1239 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1240 {
1241 struct gicv3_its_softc *sc;
1242
1243 sc = device_get_softc(dev);
1244 if (CPU_EMPTY(&isrc->isrc_cpu)) {
1245 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1246 &sc->sc_cpus);
1247 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1248 }
1249
1250 return (0);
1251 }
1252
1253 static int
gicv3_its_bind_intr(device_t dev,struct intr_irqsrc * isrc)1254 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1255 {
1256 struct gicv3_its_irqsrc *girq;
1257
1258 gicv3_its_select_cpu(dev, isrc);
1259
1260 girq = (struct gicv3_its_irqsrc *)isrc;
1261 its_cmd_movi(dev, girq);
1262 return (0);
1263 }
1264
1265 static int
gicv3_its_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)1266 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1267 struct intr_irqsrc **isrcp)
1268 {
1269
1270 /*
1271 * This should never happen, we only call this function to map
1272 * interrupts found before the controller driver is ready.
1273 */
1274 panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1275 }
1276
1277 static int
gicv3_its_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)1278 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1279 struct resource *res, struct intr_map_data *data)
1280 {
1281
1282 /* Bind the interrupt to a CPU */
1283 gicv3_its_bind_intr(dev, isrc);
1284
1285 return (0);
1286 }
1287
1288 #ifdef SMP
1289 static void
gicv3_its_init_secondary(device_t dev)1290 gicv3_its_init_secondary(device_t dev)
1291 {
1292 struct gicv3_its_softc *sc;
1293
1294 sc = device_get_softc(dev);
1295
1296 /*
1297 * This is fatal as otherwise we may bind interrupts to this CPU.
1298 * We need a way to tell the interrupt framework to only bind to a
1299 * subset of given CPUs when it performs the shuffle.
1300 */
1301 if (its_init_cpu(dev, sc) != 0)
1302 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1303 PCPU_GET(cpuid));
1304 }
1305 #endif
1306
1307 static uint32_t
its_get_devid(device_t pci_dev)1308 its_get_devid(device_t pci_dev)
1309 {
1310 uintptr_t id;
1311
1312 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1313 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1314 device_get_nameunit(pci_dev));
1315
1316 return (id);
1317 }
1318
1319 static struct its_dev *
its_device_find(device_t dev,device_t child)1320 its_device_find(device_t dev, device_t child)
1321 {
1322 struct gicv3_its_softc *sc;
1323 struct its_dev *its_dev = NULL;
1324
1325 sc = device_get_softc(dev);
1326
1327 mtx_lock_spin(&sc->sc_its_dev_lock);
1328 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1329 if (its_dev->pci_dev == child)
1330 break;
1331 }
1332 mtx_unlock_spin(&sc->sc_its_dev_lock);
1333
1334 return (its_dev);
1335 }
1336
1337 static bool
its_device_alloc(struct gicv3_its_softc * sc,int devid)1338 its_device_alloc(struct gicv3_its_softc *sc, int devid)
1339 {
1340 struct its_ptable *ptable;
1341 vm_offset_t l2_table;
1342 uint64_t *table;
1343 uint32_t index;
1344 bool shareable;
1345
1346 /* No device table */
1347 if (sc->sc_dev_table_idx < 0) {
1348 if (devid >= (1 << sc->sc_devbits)) {
1349 if (bootverbose) {
1350 device_printf(sc->dev,
1351 "%s: Device out of range for hardware "
1352 "(%x >= %x)\n", __func__, devid,
1353 1 << sc->sc_devbits);
1354 }
1355 return (false);
1356 }
1357 return (true);
1358 }
1359
1360 ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx];
1361 /* Check the devid is within the table limit */
1362 if (!ptable->ptab_indirect) {
1363 if (devid >= ptable->ptab_l1_nidents) {
1364 if (bootverbose) {
1365 device_printf(sc->dev,
1366 "%s: Device out of range for table "
1367 "(%x >= %x)\n", __func__, devid,
1368 ptable->ptab_l1_nidents);
1369 }
1370 return (false);
1371 }
1372
1373 return (true);
1374 }
1375
1376 /* Check the devid is within the allocated range */
1377 index = devid / ptable->ptab_l2_nidents;
1378 if (index >= ptable->ptab_l1_nidents) {
1379 if (bootverbose) {
1380 device_printf(sc->dev,
1381 "%s: Index out of range for table (%x >= %x)\n",
1382 __func__, index, ptable->ptab_l1_nidents);
1383 }
1384 return (false);
1385 }
1386
1387 table = (uint64_t *)ptable->ptab_vaddr;
1388 /* We have an second level table */
1389 if ((table[index] & GITS_BASER_VALID) != 0)
1390 return (true);
1391
1392 shareable = true;
1393 if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS)
1394 shareable = false;
1395
1396 l2_table = (vm_offset_t)contigmalloc_domainset(ptable->ptab_l2_size,
1397 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
1398 ptable->ptab_page_size, 0);
1399
1400 if (!shareable)
1401 cpu_dcache_wb_range((vm_offset_t)l2_table,
1402 ptable->ptab_l2_size);
1403
1404 table[index] = vtophys(l2_table) | GITS_BASER_VALID;
1405 if (!shareable)
1406 cpu_dcache_wb_range((vm_offset_t)&table[index],
1407 sizeof(table[index]));
1408
1409 dsb(sy);
1410 return (true);
1411 }
1412
1413 static struct its_dev *
its_device_get(device_t dev,device_t child,u_int nvecs)1414 its_device_get(device_t dev, device_t child, u_int nvecs)
1415 {
1416 struct gicv3_its_softc *sc;
1417 struct its_dev *its_dev;
1418 vmem_addr_t irq_base;
1419 size_t esize;
1420
1421 sc = device_get_softc(dev);
1422
1423 its_dev = its_device_find(dev, child);
1424 if (its_dev != NULL)
1425 return (its_dev);
1426
1427 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1428 if (its_dev == NULL)
1429 return (NULL);
1430
1431 its_dev->pci_dev = child;
1432 its_dev->devid = its_get_devid(child);
1433
1434 its_dev->lpis.lpi_busy = 0;
1435 its_dev->lpis.lpi_num = nvecs;
1436 its_dev->lpis.lpi_free = nvecs;
1437
1438 if (!its_device_alloc(sc, its_dev->devid)) {
1439 free(its_dev, M_GICV3_ITS);
1440 return (NULL);
1441 }
1442
1443 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1444 &irq_base) != 0) {
1445 free(its_dev, M_GICV3_ITS);
1446 return (NULL);
1447 }
1448 its_dev->lpis.lpi_base = irq_base;
1449
1450 /* Get ITT entry size */
1451 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1452
1453 /*
1454 * Allocate ITT for this device.
1455 * PA has to be 256 B aligned. At least two entries for device.
1456 */
1457 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1458 its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size,
1459 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1460 LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1461 if (its_dev->itt == 0) {
1462 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1463 free(its_dev, M_GICV3_ITS);
1464 return (NULL);
1465 }
1466
1467 /* Make sure device sees zeroed ITT. */
1468 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1469 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1470
1471 mtx_lock_spin(&sc->sc_its_dev_lock);
1472 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1473 mtx_unlock_spin(&sc->sc_its_dev_lock);
1474
1475 /* Map device to its ITT */
1476 its_cmd_mapd(dev, its_dev, 1);
1477
1478 return (its_dev);
1479 }
1480
1481 static void
its_device_release(device_t dev,struct its_dev * its_dev)1482 its_device_release(device_t dev, struct its_dev *its_dev)
1483 {
1484 struct gicv3_its_softc *sc;
1485
1486 KASSERT(its_dev->lpis.lpi_busy == 0,
1487 ("its_device_release: Trying to release an inuse ITS device"));
1488
1489 /* Unmap device in ITS */
1490 its_cmd_mapd(dev, its_dev, 0);
1491
1492 sc = device_get_softc(dev);
1493
1494 /* Remove the device from the list of devices */
1495 mtx_lock_spin(&sc->sc_its_dev_lock);
1496 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1497 mtx_unlock_spin(&sc->sc_its_dev_lock);
1498
1499 /* Free ITT */
1500 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device"));
1501 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1502
1503 /* Free the IRQ allocation */
1504 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1505 its_dev->lpis.lpi_num);
1506
1507 free(its_dev, M_GICV3_ITS);
1508 }
1509
1510 static struct gicv3_its_irqsrc *
gicv3_its_alloc_irqsrc(device_t dev,struct gicv3_its_softc * sc,u_int irq)1511 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1512 {
1513 struct gicv3_its_irqsrc *girq = NULL;
1514
1515 KASSERT(sc->sc_irqs[irq] == NULL,
1516 ("%s: Interrupt %u already allocated", __func__, irq));
1517 mtx_lock_spin(&sc->sc_its_dev_lock);
1518 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1519 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1520 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1521 }
1522 mtx_unlock_spin(&sc->sc_its_dev_lock);
1523 if (girq == NULL) {
1524 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1525 M_NOWAIT | M_ZERO);
1526 if (girq == NULL)
1527 return (NULL);
1528 girq->gi_id = -1;
1529 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1530 "%s,%u", device_get_nameunit(dev), irq) != 0) {
1531 free(girq, M_GICV3_ITS);
1532 return (NULL);
1533 }
1534 }
1535 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1536 sc->sc_irqs[irq] = girq;
1537
1538 return (girq);
1539 }
1540
1541 static void
gicv3_its_release_irqsrc(struct gicv3_its_softc * sc,struct gicv3_its_irqsrc * girq)1542 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1543 struct gicv3_its_irqsrc *girq)
1544 {
1545 u_int irq;
1546
1547 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1548
1549 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1550 sc->sc_irqs[irq] = NULL;
1551
1552 girq->gi_id = -1;
1553 girq->gi_its_dev = NULL;
1554 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1555 }
1556
1557 static int
gicv3_its_alloc_msi(device_t dev,device_t child,int count,int maxcount,device_t * pic,struct intr_irqsrc ** srcs)1558 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1559 device_t *pic, struct intr_irqsrc **srcs)
1560 {
1561 struct gicv3_its_softc *sc;
1562 struct gicv3_its_irqsrc *girq;
1563 struct its_dev *its_dev;
1564 u_int irq;
1565 int i;
1566
1567 its_dev = its_device_get(dev, child, count);
1568 if (its_dev == NULL)
1569 return (ENXIO);
1570
1571 KASSERT(its_dev->lpis.lpi_free >= count,
1572 ("gicv3_its_alloc_msi: No free LPIs"));
1573 sc = device_get_softc(dev);
1574 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1575 its_dev->lpis.lpi_free;
1576
1577 /* Allocate the irqsrc for each MSI */
1578 for (i = 0; i < count; i++, irq++) {
1579 its_dev->lpis.lpi_free--;
1580 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1581 sc, irq);
1582 if (srcs[i] == NULL)
1583 break;
1584 }
1585
1586 /* The allocation failed, release them */
1587 if (i != count) {
1588 mtx_lock_spin(&sc->sc_its_dev_lock);
1589 for (i = 0; i < count; i++) {
1590 girq = (struct gicv3_its_irqsrc *)srcs[i];
1591 if (girq == NULL)
1592 break;
1593 gicv3_its_release_irqsrc(sc, girq);
1594 srcs[i] = NULL;
1595 }
1596 mtx_unlock_spin(&sc->sc_its_dev_lock);
1597 return (ENXIO);
1598 }
1599
1600 /* Finish the allocation now we have all MSI irqsrcs */
1601 for (i = 0; i < count; i++) {
1602 girq = (struct gicv3_its_irqsrc *)srcs[i];
1603 girq->gi_id = i;
1604 girq->gi_its_dev = its_dev;
1605
1606 /* Map the message to the given IRQ */
1607 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1608 its_cmd_mapti(dev, girq);
1609 }
1610 its_dev->lpis.lpi_busy += count;
1611 *pic = dev;
1612
1613 return (0);
1614 }
1615
1616 static int
gicv3_its_release_msi(device_t dev,device_t child,int count,struct intr_irqsrc ** isrc)1617 gicv3_its_release_msi(device_t dev, device_t child, int count,
1618 struct intr_irqsrc **isrc)
1619 {
1620 struct gicv3_its_softc *sc;
1621 struct gicv3_its_irqsrc *girq;
1622 struct its_dev *its_dev;
1623 int i;
1624
1625 its_dev = its_device_find(dev, child);
1626
1627 KASSERT(its_dev != NULL,
1628 ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1629 "no ITS device"));
1630 KASSERT(its_dev->lpis.lpi_busy >= count,
1631 ("gicv3_its_release_msi: Releasing more interrupts than "
1632 "were allocated: releasing %d, allocated %d", count,
1633 its_dev->lpis.lpi_busy));
1634
1635 sc = device_get_softc(dev);
1636 mtx_lock_spin(&sc->sc_its_dev_lock);
1637 for (i = 0; i < count; i++) {
1638 girq = (struct gicv3_its_irqsrc *)isrc[i];
1639 gicv3_its_release_irqsrc(sc, girq);
1640 }
1641 mtx_unlock_spin(&sc->sc_its_dev_lock);
1642 its_dev->lpis.lpi_busy -= count;
1643
1644 if (its_dev->lpis.lpi_busy == 0)
1645 its_device_release(dev, its_dev);
1646
1647 return (0);
1648 }
1649
1650 static int
gicv3_its_alloc_msix(device_t dev,device_t child,device_t * pic,struct intr_irqsrc ** isrcp)1651 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1652 struct intr_irqsrc **isrcp)
1653 {
1654 struct gicv3_its_softc *sc;
1655 struct gicv3_its_irqsrc *girq;
1656 struct its_dev *its_dev;
1657 u_int nvecs, irq;
1658
1659 nvecs = pci_msix_count(child);
1660 its_dev = its_device_get(dev, child, nvecs);
1661 if (its_dev == NULL)
1662 return (ENXIO);
1663
1664 KASSERT(its_dev->lpis.lpi_free > 0,
1665 ("gicv3_its_alloc_msix: No free LPIs"));
1666 sc = device_get_softc(dev);
1667 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1668 its_dev->lpis.lpi_free;
1669
1670 girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1671 if (girq == NULL)
1672 return (ENXIO);
1673 girq->gi_id = its_dev->lpis.lpi_busy;
1674 girq->gi_its_dev = its_dev;
1675
1676 its_dev->lpis.lpi_free--;
1677 its_dev->lpis.lpi_busy++;
1678
1679 /* Map the message to the given IRQ */
1680 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1681 its_cmd_mapti(dev, girq);
1682
1683 *pic = dev;
1684 *isrcp = (struct intr_irqsrc *)girq;
1685
1686 return (0);
1687 }
1688
1689 static int
gicv3_its_release_msix(device_t dev,device_t child,struct intr_irqsrc * isrc)1690 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1691 {
1692 struct gicv3_its_softc *sc;
1693 struct gicv3_its_irqsrc *girq;
1694 struct its_dev *its_dev;
1695
1696 its_dev = its_device_find(dev, child);
1697
1698 KASSERT(its_dev != NULL,
1699 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1700 "no ITS device"));
1701 KASSERT(its_dev->lpis.lpi_busy > 0,
1702 ("gicv3_its_release_msix: Releasing more interrupts than "
1703 "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1704
1705 sc = device_get_softc(dev);
1706 girq = (struct gicv3_its_irqsrc *)isrc;
1707 mtx_lock_spin(&sc->sc_its_dev_lock);
1708 gicv3_its_release_irqsrc(sc, girq);
1709 mtx_unlock_spin(&sc->sc_its_dev_lock);
1710 its_dev->lpis.lpi_busy--;
1711
1712 if (its_dev->lpis.lpi_busy == 0)
1713 its_device_release(dev, its_dev);
1714
1715 return (0);
1716 }
1717
1718 static int
gicv3_its_map_msi(device_t dev,device_t child,struct intr_irqsrc * isrc,uint64_t * addr,uint32_t * data)1719 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1720 uint64_t *addr, uint32_t *data)
1721 {
1722 struct gicv3_its_softc *sc;
1723 struct gicv3_its_irqsrc *girq;
1724
1725 sc = device_get_softc(dev);
1726 girq = (struct gicv3_its_irqsrc *)isrc;
1727
1728 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1729 *data = girq->gi_id;
1730
1731 return (0);
1732 }
1733
1734 #ifdef IOMMU
1735 static int
gicv3_iommu_init(device_t dev,device_t child,struct iommu_domain ** domain)1736 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1737 {
1738 struct gicv3_its_softc *sc;
1739 struct iommu_ctx *ctx;
1740 int error;
1741
1742 sc = device_get_softc(dev);
1743 ctx = iommu_get_dev_ctx(child);
1744 if (ctx == NULL)
1745 return (ENXIO);
1746 /* Map the page containing the GITS_TRANSLATER register. */
1747 error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1748 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1749 *domain = iommu_get_ctx_domain(ctx);
1750
1751 return (error);
1752 }
1753
1754 static void
gicv3_iommu_deinit(device_t dev,device_t child)1755 gicv3_iommu_deinit(device_t dev, device_t child)
1756 {
1757 struct iommu_ctx *ctx;
1758
1759 ctx = iommu_get_dev_ctx(child);
1760 if (ctx == NULL)
1761 return;
1762
1763 iommu_unmap_msi(ctx);
1764 }
1765 #endif
1766
1767 /*
1768 * Commands handling.
1769 */
1770
1771 static __inline void
cmd_format_command(struct its_cmd * cmd,uint8_t cmd_type)1772 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1773 {
1774 /* Command field: DW0 [7:0] */
1775 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1776 cmd->cmd_dword[0] |= htole64(cmd_type);
1777 }
1778
1779 static __inline void
cmd_format_devid(struct its_cmd * cmd,uint32_t devid)1780 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1781 {
1782 /* Device ID field: DW0 [63:32] */
1783 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1784 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1785 }
1786
1787 static __inline void
cmd_format_size(struct its_cmd * cmd,uint16_t size)1788 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1789 {
1790 /* Size field: DW1 [4:0] */
1791 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1792 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1793 }
1794
1795 static __inline void
cmd_format_id(struct its_cmd * cmd,uint32_t id)1796 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1797 {
1798 /* ID field: DW1 [31:0] */
1799 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1800 cmd->cmd_dword[1] |= htole64(id);
1801 }
1802
1803 static __inline void
cmd_format_pid(struct its_cmd * cmd,uint32_t pid)1804 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1805 {
1806 /* Physical ID field: DW1 [63:32] */
1807 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1808 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1809 }
1810
1811 static __inline void
cmd_format_col(struct its_cmd * cmd,uint16_t col_id)1812 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1813 {
1814 /* Collection field: DW2 [16:0] */
1815 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1816 cmd->cmd_dword[2] |= htole64(col_id);
1817 }
1818
1819 static __inline void
cmd_format_target(struct its_cmd * cmd,uint64_t target)1820 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1821 {
1822 /* Target Address field: DW2 [47:16] */
1823 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1824 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1825 }
1826
1827 static __inline void
cmd_format_itt(struct its_cmd * cmd,uint64_t itt)1828 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1829 {
1830 /* ITT Address field: DW2 [47:8] */
1831 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1832 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1833 }
1834
1835 static __inline void
cmd_format_valid(struct its_cmd * cmd,uint8_t valid)1836 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1837 {
1838 /* Valid field: DW2 [63] */
1839 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1840 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1841 }
1842
1843 static inline bool
its_cmd_queue_full(struct gicv3_its_softc * sc)1844 its_cmd_queue_full(struct gicv3_its_softc *sc)
1845 {
1846 size_t read_idx, next_write_idx;
1847
1848 /* Get the index of the next command */
1849 next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1850 (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1851 /* And the index of the current command being read */
1852 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1853
1854 /*
1855 * The queue is full when the write offset points
1856 * at the command before the current read offset.
1857 */
1858 return (next_write_idx == read_idx);
1859 }
1860
1861 static inline void
its_cmd_sync(struct gicv3_its_softc * sc,struct its_cmd * cmd)1862 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1863 {
1864
1865 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1866 /* Clean D-cache under command. */
1867 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd));
1868 } else {
1869 /* DSB inner shareable, store */
1870 dsb(ishst);
1871 }
1872
1873 }
1874
1875 static inline uint64_t
its_cmd_cwriter_offset(struct gicv3_its_softc * sc,struct its_cmd * cmd)1876 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1877 {
1878 uint64_t off;
1879
1880 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1881
1882 return (off);
1883 }
1884
1885 static void
its_cmd_wait_completion(device_t dev,struct its_cmd * cmd_first,struct its_cmd * cmd_last)1886 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1887 struct its_cmd *cmd_last)
1888 {
1889 struct gicv3_its_softc *sc;
1890 uint64_t first, last, read;
1891 size_t us_left;
1892
1893 sc = device_get_softc(dev);
1894
1895 /*
1896 * XXX ARM64TODO: This is obviously a significant delay.
1897 * The reason for that is that currently the time frames for
1898 * the command to complete are not known.
1899 */
1900 us_left = 1000000;
1901
1902 first = its_cmd_cwriter_offset(sc, cmd_first);
1903 last = its_cmd_cwriter_offset(sc, cmd_last);
1904
1905 for (;;) {
1906 read = gic_its_read_8(sc, GITS_CREADR);
1907 if (first < last) {
1908 if (read < first || read >= last)
1909 break;
1910 } else if (read < first && read >= last)
1911 break;
1912
1913 if (us_left-- == 0) {
1914 /* This means timeout */
1915 device_printf(dev,
1916 "Timeout while waiting for CMD completion.\n");
1917 return;
1918 }
1919 DELAY(1);
1920 }
1921 }
1922
1923 static struct its_cmd *
its_cmd_alloc_locked(device_t dev)1924 its_cmd_alloc_locked(device_t dev)
1925 {
1926 struct gicv3_its_softc *sc;
1927 struct its_cmd *cmd;
1928 size_t us_left;
1929
1930 sc = device_get_softc(dev);
1931
1932 /*
1933 * XXX ARM64TODO: This is obviously a significant delay.
1934 * The reason for that is that currently the time frames for
1935 * the command to complete (and therefore free the descriptor)
1936 * are not known.
1937 */
1938 us_left = 1000000;
1939
1940 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1941 while (its_cmd_queue_full(sc)) {
1942 if (us_left-- == 0) {
1943 /* Timeout while waiting for free command */
1944 device_printf(dev,
1945 "Timeout while waiting for free command\n");
1946 return (NULL);
1947 }
1948 DELAY(1);
1949 }
1950
1951 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1952 sc->sc_its_cmd_next_idx++;
1953 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1954
1955 return (cmd);
1956 }
1957
1958 static uint64_t
its_cmd_prepare(struct its_cmd * cmd,struct its_cmd_desc * desc)1959 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1960 {
1961 uint64_t target;
1962 uint8_t cmd_type;
1963 u_int size;
1964
1965 cmd_type = desc->cmd_type;
1966 target = ITS_TARGET_NONE;
1967
1968 switch (cmd_type) {
1969 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */
1970 target = desc->cmd_desc_movi.col->col_target;
1971 cmd_format_command(cmd, ITS_CMD_MOVI);
1972 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1973 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1974 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1975 break;
1976 case ITS_CMD_SYNC: /* Wait for previous commands completion */
1977 target = desc->cmd_desc_sync.col->col_target;
1978 cmd_format_command(cmd, ITS_CMD_SYNC);
1979 cmd_format_target(cmd, target);
1980 break;
1981 case ITS_CMD_MAPD: /* Assign ITT to device */
1982 cmd_format_command(cmd, ITS_CMD_MAPD);
1983 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1984 /*
1985 * Size describes number of bits to encode interrupt IDs
1986 * supported by the device minus one.
1987 * When V (valid) bit is zero, this field should be written
1988 * as zero.
1989 */
1990 if (desc->cmd_desc_mapd.valid != 0) {
1991 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1992 size = MAX(1, size) - 1;
1993 } else
1994 size = 0;
1995
1996 cmd_format_size(cmd, size);
1997 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1998 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1999 break;
2000 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */
2001 target = desc->cmd_desc_mapc.col->col_target;
2002 cmd_format_command(cmd, ITS_CMD_MAPC);
2003 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
2004 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
2005 cmd_format_target(cmd, target);
2006 break;
2007 case ITS_CMD_MAPTI:
2008 target = desc->cmd_desc_mapvi.col->col_target;
2009 cmd_format_command(cmd, ITS_CMD_MAPTI);
2010 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
2011 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
2012 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
2013 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
2014 break;
2015 case ITS_CMD_MAPI:
2016 target = desc->cmd_desc_mapi.col->col_target;
2017 cmd_format_command(cmd, ITS_CMD_MAPI);
2018 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
2019 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
2020 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
2021 break;
2022 case ITS_CMD_INV:
2023 target = desc->cmd_desc_inv.col->col_target;
2024 cmd_format_command(cmd, ITS_CMD_INV);
2025 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
2026 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
2027 break;
2028 case ITS_CMD_INVALL:
2029 cmd_format_command(cmd, ITS_CMD_INVALL);
2030 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
2031 break;
2032 default:
2033 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
2034 }
2035
2036 return (target);
2037 }
2038
2039 static int
its_cmd_send(device_t dev,struct its_cmd_desc * desc)2040 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
2041 {
2042 struct gicv3_its_softc *sc;
2043 struct its_cmd *cmd, *cmd_sync, *cmd_write;
2044 struct its_col col_sync;
2045 struct its_cmd_desc desc_sync;
2046 uint64_t target, cwriter;
2047
2048 sc = device_get_softc(dev);
2049 mtx_lock_spin(&sc->sc_its_cmd_lock);
2050 cmd = its_cmd_alloc_locked(dev);
2051 if (cmd == NULL) {
2052 device_printf(dev, "could not allocate ITS command\n");
2053 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2054 return (EBUSY);
2055 }
2056
2057 target = its_cmd_prepare(cmd, desc);
2058 its_cmd_sync(sc, cmd);
2059
2060 if (target != ITS_TARGET_NONE) {
2061 cmd_sync = its_cmd_alloc_locked(dev);
2062 if (cmd_sync != NULL) {
2063 desc_sync.cmd_type = ITS_CMD_SYNC;
2064 col_sync.col_target = target;
2065 desc_sync.cmd_desc_sync.col = &col_sync;
2066 its_cmd_prepare(cmd_sync, &desc_sync);
2067 its_cmd_sync(sc, cmd_sync);
2068 }
2069 }
2070
2071 /* Update GITS_CWRITER */
2072 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
2073 gic_its_write_8(sc, GITS_CWRITER, cwriter);
2074 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2075 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2076
2077 its_cmd_wait_completion(dev, cmd, cmd_write);
2078
2079 return (0);
2080 }
2081
2082 /* Handlers to send commands */
2083 static void
its_cmd_movi(device_t dev,struct gicv3_its_irqsrc * girq)2084 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
2085 {
2086 struct gicv3_its_softc *sc;
2087 struct its_cmd_desc desc;
2088 struct its_col *col;
2089
2090 sc = device_get_softc(dev);
2091 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2092
2093 desc.cmd_type = ITS_CMD_MOVI;
2094 desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
2095 desc.cmd_desc_movi.col = col;
2096 desc.cmd_desc_movi.id = girq->gi_id;
2097
2098 its_cmd_send(dev, &desc);
2099 }
2100
2101 static void
its_cmd_mapc(device_t dev,struct its_col * col,uint8_t valid)2102 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
2103 {
2104 struct its_cmd_desc desc;
2105
2106 desc.cmd_type = ITS_CMD_MAPC;
2107 desc.cmd_desc_mapc.col = col;
2108 /*
2109 * Valid bit set - map the collection.
2110 * Valid bit cleared - unmap the collection.
2111 */
2112 desc.cmd_desc_mapc.valid = valid;
2113
2114 its_cmd_send(dev, &desc);
2115 }
2116
2117 static void
its_cmd_mapti(device_t dev,struct gicv3_its_irqsrc * girq)2118 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
2119 {
2120 struct gicv3_its_softc *sc;
2121 struct its_cmd_desc desc;
2122 struct its_col *col;
2123 u_int col_id;
2124
2125 sc = device_get_softc(dev);
2126
2127 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
2128 col = sc->sc_its_cols[col_id];
2129
2130 desc.cmd_type = ITS_CMD_MAPTI;
2131 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
2132 desc.cmd_desc_mapvi.col = col;
2133 /* The EventID sent to the device */
2134 desc.cmd_desc_mapvi.id = girq->gi_id;
2135 /* The physical interrupt presented to softeware */
2136 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
2137
2138 its_cmd_send(dev, &desc);
2139 }
2140
2141 static void
its_cmd_mapd(device_t dev,struct its_dev * its_dev,uint8_t valid)2142 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
2143 {
2144 struct its_cmd_desc desc;
2145
2146 desc.cmd_type = ITS_CMD_MAPD;
2147 desc.cmd_desc_mapd.its_dev = its_dev;
2148 desc.cmd_desc_mapd.valid = valid;
2149
2150 its_cmd_send(dev, &desc);
2151 }
2152
2153 static void
its_cmd_inv(device_t dev,struct its_dev * its_dev,struct gicv3_its_irqsrc * girq)2154 its_cmd_inv(device_t dev, struct its_dev *its_dev,
2155 struct gicv3_its_irqsrc *girq)
2156 {
2157 struct gicv3_its_softc *sc;
2158 struct its_cmd_desc desc;
2159 struct its_col *col;
2160
2161 sc = device_get_softc(dev);
2162 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2163
2164 desc.cmd_type = ITS_CMD_INV;
2165 /* The EventID sent to the device */
2166 desc.cmd_desc_inv.pid = girq->gi_id;
2167 desc.cmd_desc_inv.its_dev = its_dev;
2168 desc.cmd_desc_inv.col = col;
2169
2170 its_cmd_send(dev, &desc);
2171 }
2172
2173 static void
its_cmd_invall(device_t dev,struct its_col * col)2174 its_cmd_invall(device_t dev, struct its_col *col)
2175 {
2176 struct its_cmd_desc desc;
2177
2178 desc.cmd_type = ITS_CMD_INVALL;
2179 desc.cmd_desc_invall.col = col;
2180
2181 its_cmd_send(dev, &desc);
2182 }
2183
2184 #ifdef FDT
2185 static device_probe_t gicv3_its_fdt_probe;
2186 static device_attach_t gicv3_its_fdt_attach;
2187
2188 static device_method_t gicv3_its_fdt_methods[] = {
2189 /* Device interface */
2190 DEVMETHOD(device_probe, gicv3_its_fdt_probe),
2191 DEVMETHOD(device_attach, gicv3_its_fdt_attach),
2192
2193 /* End */
2194 DEVMETHOD_END
2195 };
2196
2197 #define its_baseclasses its_fdt_baseclasses
2198 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
2199 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2200 #undef its_baseclasses
2201
2202 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
2203 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2204
2205 static int
gicv3_its_fdt_probe(device_t dev)2206 gicv3_its_fdt_probe(device_t dev)
2207 {
2208
2209 if (!ofw_bus_status_okay(dev))
2210 return (ENXIO);
2211
2212 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
2213 return (ENXIO);
2214
2215 if (!gicv3_get_support_lpis(dev))
2216 return (ENXIO);
2217
2218 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2219 return (BUS_PROBE_DEFAULT);
2220 }
2221
2222 static int
gicv3_its_fdt_attach(device_t dev)2223 gicv3_its_fdt_attach(device_t dev)
2224 {
2225 struct gicv3_its_softc *sc;
2226 phandle_t xref;
2227 int err;
2228
2229 sc = device_get_softc(dev);
2230 sc->dev = dev;
2231 err = gicv3_its_attach(dev);
2232 if (err != 0)
2233 return (err);
2234
2235 /* Register this device as a interrupt controller */
2236 xref = OF_xref_from_node(ofw_bus_get_node(dev));
2237 sc->sc_pic = intr_pic_register(dev, xref);
2238 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2239 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2240 if (err != 0) {
2241 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2242 return (err);
2243 }
2244
2245 /* Register this device to handle MSI interrupts */
2246 err = intr_msi_register(dev, xref);
2247 if (err != 0) {
2248 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2249 return (err);
2250 }
2251
2252 return (0);
2253 }
2254 #endif
2255
2256 #ifdef DEV_ACPI
2257 static device_probe_t gicv3_its_acpi_probe;
2258 static device_attach_t gicv3_its_acpi_attach;
2259
2260 static device_method_t gicv3_its_acpi_methods[] = {
2261 /* Device interface */
2262 DEVMETHOD(device_probe, gicv3_its_acpi_probe),
2263 DEVMETHOD(device_attach, gicv3_its_acpi_attach),
2264
2265 /* End */
2266 DEVMETHOD_END
2267 };
2268
2269 #define its_baseclasses its_acpi_baseclasses
2270 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2271 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2272 #undef its_baseclasses
2273
2274 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2275 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2276
2277 static int
gicv3_its_acpi_probe(device_t dev)2278 gicv3_its_acpi_probe(device_t dev)
2279 {
2280
2281 if (gic_get_bus(dev) != GIC_BUS_ACPI)
2282 return (EINVAL);
2283
2284 if (gic_get_hw_rev(dev) < 3)
2285 return (EINVAL);
2286
2287 if (!gicv3_get_support_lpis(dev))
2288 return (ENXIO);
2289
2290 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2291 return (BUS_PROBE_DEFAULT);
2292 }
2293
2294 static int
gicv3_its_acpi_attach(device_t dev)2295 gicv3_its_acpi_attach(device_t dev)
2296 {
2297 struct gicv3_its_softc *sc;
2298 struct gic_v3_devinfo *di;
2299 int err;
2300
2301 sc = device_get_softc(dev);
2302 sc->dev = dev;
2303 err = gicv3_its_attach(dev);
2304 if (err != 0)
2305 return (err);
2306
2307 di = device_get_ivars(dev);
2308 sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2309 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2310 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2311 if (err != 0) {
2312 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2313 return (err);
2314 }
2315
2316 /* Register this device to handle MSI interrupts */
2317 err = intr_msi_register(dev, di->msi_xref);
2318 if (err != 0) {
2319 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2320 return (err);
2321 }
2322
2323 return (0);
2324 }
2325 #endif
2326