1 /*-
2 * Copyright (c) 2015, 2020 Ruslan Bukin <[email protected]>
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * This software was developed by Semihalf under
7 * the sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /* Generic ECAM PCIe driver */
32
33 #include <sys/cdefs.h>
34 #include "opt_platform.h"
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/malloc.h>
39 #include <sys/kernel.h>
40 #include <sys/rman.h>
41 #include <sys/module.h>
42 #include <sys/bus.h>
43 #include <sys/endian.h>
44
45 #include <dev/pci/pcivar.h>
46 #include <dev/pci/pcireg.h>
47 #include <dev/pci/pcib_private.h>
48 #include <dev/pci/pci_host_generic.h>
49
50 #include <machine/bus.h>
51 #include <machine/intr.h>
52
53 #include "pcib_if.h"
54
55 #if defined(VM_MEMATTR_DEVICE_NP)
56 #define PCI_UNMAPPED
57 #define PCI_RF_FLAGS RF_UNMAPPED
58 #else
59 #define PCI_RF_FLAGS 0
60 #endif
61
62
63 /* Forward prototypes */
64
65 static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot,
66 u_int func, u_int reg, int bytes);
67 static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot,
68 u_int func, u_int reg, uint32_t val, int bytes);
69 static int generic_pcie_maxslots(device_t dev);
70 static int generic_pcie_read_ivar(device_t dev, device_t child, int index,
71 uintptr_t *result);
72 static int generic_pcie_write_ivar(device_t dev, device_t child, int index,
73 uintptr_t value);
74
75 int
pci_host_generic_core_attach(device_t dev)76 pci_host_generic_core_attach(device_t dev)
77 {
78 #ifdef PCI_UNMAPPED
79 struct resource_map_request req;
80 struct resource_map map;
81 #endif
82 struct generic_pcie_core_softc *sc;
83 uint64_t phys_base;
84 uint64_t pci_base;
85 uint64_t size;
86 char buf[64];
87 int domain, error;
88 int rid, tuple;
89
90 sc = device_get_softc(dev);
91 sc->dev = dev;
92
93 /* Create the parent DMA tag to pass down the coherent flag */
94 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
95 1, 0, /* alignment, bounds */
96 BUS_SPACE_MAXADDR, /* lowaddr */
97 BUS_SPACE_MAXADDR, /* highaddr */
98 NULL, NULL, /* filter, filterarg */
99 BUS_SPACE_MAXSIZE, /* maxsize */
100 BUS_SPACE_UNRESTRICTED, /* nsegments */
101 BUS_SPACE_MAXSIZE, /* maxsegsize */
102 sc->coherent ? BUS_DMA_COHERENT : 0, /* flags */
103 NULL, NULL, /* lockfunc, lockarg */
104 &sc->dmat);
105 if (error != 0)
106 return (error);
107
108 /*
109 * Attempt to set the domain. If it's missing, or we are unable to
110 * set it then memory allocations may be placed in the wrong domain.
111 */
112 if (bus_get_domain(dev, &domain) == 0)
113 (void)bus_dma_tag_set_domain(sc->dmat, domain);
114
115 if ((sc->quirks & PCIE_CUSTOM_CONFIG_SPACE_QUIRK) == 0) {
116 rid = 0;
117 sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
118 PCI_RF_FLAGS | RF_ACTIVE);
119 if (sc->res == NULL) {
120 device_printf(dev, "could not allocate memory.\n");
121 error = ENXIO;
122 goto err_resource;
123 }
124 #ifdef PCI_UNMAPPED
125 resource_init_map_request(&req);
126 req.memattr = VM_MEMATTR_DEVICE_NP;
127 error = bus_map_resource(dev, SYS_RES_MEMORY, sc->res, &req,
128 &map);
129 if (error != 0) {
130 device_printf(dev, "could not map memory.\n");
131 return (error);
132 }
133 rman_set_mapping(sc->res, &map);
134 #endif
135 }
136
137 sc->has_pmem = false;
138 sc->pmem_rman.rm_type = RMAN_ARRAY;
139 snprintf(buf, sizeof(buf), "%s prefetch window",
140 device_get_nameunit(dev));
141 sc->pmem_rman.rm_descr = strdup(buf, M_DEVBUF);
142
143 sc->mem_rman.rm_type = RMAN_ARRAY;
144 snprintf(buf, sizeof(buf), "%s memory window",
145 device_get_nameunit(dev));
146 sc->mem_rman.rm_descr = strdup(buf, M_DEVBUF);
147
148 sc->io_rman.rm_type = RMAN_ARRAY;
149 snprintf(buf, sizeof(buf), "%s I/O port window",
150 device_get_nameunit(dev));
151 sc->io_rman.rm_descr = strdup(buf, M_DEVBUF);
152
153 /* Initialize rman and allocate memory regions */
154 error = rman_init(&sc->pmem_rman);
155 if (error) {
156 device_printf(dev, "rman_init() failed. error = %d\n", error);
157 goto err_pmem_rman;
158 }
159
160 error = rman_init(&sc->mem_rman);
161 if (error) {
162 device_printf(dev, "rman_init() failed. error = %d\n", error);
163 goto err_mem_rman;
164 }
165
166 error = rman_init(&sc->io_rman);
167 if (error) {
168 device_printf(dev, "rman_init() failed. error = %d\n", error);
169 goto err_io_rman;
170 }
171
172 for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) {
173 phys_base = sc->ranges[tuple].phys_base;
174 pci_base = sc->ranges[tuple].pci_base;
175 size = sc->ranges[tuple].size;
176 if (phys_base == 0 || size == 0)
177 continue; /* empty range element */
178 switch (FLAG_TYPE(sc->ranges[tuple].flags)) {
179 case FLAG_TYPE_PMEM:
180 sc->has_pmem = true;
181 error = rman_manage_region(&sc->pmem_rman,
182 pci_base, pci_base + size - 1);
183 break;
184 case FLAG_TYPE_MEM:
185 error = rman_manage_region(&sc->mem_rman,
186 pci_base, pci_base + size - 1);
187 break;
188 case FLAG_TYPE_IO:
189 error = rman_manage_region(&sc->io_rman,
190 pci_base, pci_base + size - 1);
191 break;
192 default:
193 continue;
194 }
195 if (error) {
196 device_printf(dev, "rman_manage_region() failed."
197 "error = %d\n", error);
198 goto err_rman_manage;
199 }
200 }
201
202 return (0);
203
204 err_rman_manage:
205 rman_fini(&sc->io_rman);
206 err_io_rman:
207 rman_fini(&sc->mem_rman);
208 err_mem_rman:
209 rman_fini(&sc->pmem_rman);
210 err_pmem_rman:
211 free(__DECONST(char *, sc->io_rman.rm_descr), M_DEVBUF);
212 free(__DECONST(char *, sc->mem_rman.rm_descr), M_DEVBUF);
213 free(__DECONST(char *, sc->pmem_rman.rm_descr), M_DEVBUF);
214 if (sc->res != NULL)
215 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->res);
216 err_resource:
217 bus_dma_tag_destroy(sc->dmat);
218 return (error);
219 }
220
221 int
pci_host_generic_core_detach(device_t dev)222 pci_host_generic_core_detach(device_t dev)
223 {
224 struct generic_pcie_core_softc *sc;
225 int error;
226
227 sc = device_get_softc(dev);
228
229 error = bus_generic_detach(dev);
230 if (error != 0)
231 return (error);
232
233 rman_fini(&sc->io_rman);
234 rman_fini(&sc->mem_rman);
235 rman_fini(&sc->pmem_rman);
236 free(__DECONST(char *, sc->io_rman.rm_descr), M_DEVBUF);
237 free(__DECONST(char *, sc->mem_rman.rm_descr), M_DEVBUF);
238 free(__DECONST(char *, sc->pmem_rman.rm_descr), M_DEVBUF);
239 if (sc->res != NULL)
240 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->res);
241 bus_dma_tag_destroy(sc->dmat);
242
243 return (0);
244 }
245
246 static uint32_t
generic_pcie_read_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,int bytes)247 generic_pcie_read_config(device_t dev, u_int bus, u_int slot,
248 u_int func, u_int reg, int bytes)
249 {
250 struct generic_pcie_core_softc *sc;
251 uint64_t offset;
252 uint32_t data;
253
254 sc = device_get_softc(dev);
255 if ((bus < sc->bus_start) || (bus > sc->bus_end))
256 return (~0U);
257 if ((slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) ||
258 (reg > PCIE_REGMAX))
259 return (~0U);
260 if ((sc->quirks & PCIE_ECAM_DESIGNWARE_QUIRK) && bus == 0 && slot > 0)
261 return (~0U);
262
263 offset = PCIE_ADDR_OFFSET(bus - sc->bus_start, slot, func, reg);
264
265 switch (bytes) {
266 case 1:
267 data = bus_read_1(sc->res, offset);
268 break;
269 case 2:
270 data = le16toh(bus_read_2(sc->res, offset));
271 break;
272 case 4:
273 data = le32toh(bus_read_4(sc->res, offset));
274 break;
275 default:
276 return (~0U);
277 }
278
279 return (data);
280 }
281
282 static void
generic_pcie_write_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,uint32_t val,int bytes)283 generic_pcie_write_config(device_t dev, u_int bus, u_int slot,
284 u_int func, u_int reg, uint32_t val, int bytes)
285 {
286 struct generic_pcie_core_softc *sc;
287 uint64_t offset;
288
289 sc = device_get_softc(dev);
290 if ((bus < sc->bus_start) || (bus > sc->bus_end))
291 return;
292 if ((slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) ||
293 (reg > PCIE_REGMAX))
294 return;
295
296 offset = PCIE_ADDR_OFFSET(bus - sc->bus_start, slot, func, reg);
297
298 switch (bytes) {
299 case 1:
300 bus_write_1(sc->res, offset, val);
301 break;
302 case 2:
303 bus_write_2(sc->res, offset, htole16(val));
304 break;
305 case 4:
306 bus_write_4(sc->res, offset, htole32(val));
307 break;
308 default:
309 return;
310 }
311 }
312
313 static int
generic_pcie_maxslots(device_t dev)314 generic_pcie_maxslots(device_t dev)
315 {
316
317 return (31); /* max slots per bus acc. to standard */
318 }
319
320 static int
generic_pcie_read_ivar(device_t dev,device_t child,int index,uintptr_t * result)321 generic_pcie_read_ivar(device_t dev, device_t child, int index,
322 uintptr_t *result)
323 {
324 struct generic_pcie_core_softc *sc;
325
326 sc = device_get_softc(dev);
327
328 if (index == PCIB_IVAR_BUS) {
329 *result = sc->bus_start;
330 return (0);
331 }
332
333 if (index == PCIB_IVAR_DOMAIN) {
334 *result = sc->ecam;
335 return (0);
336 }
337
338 if (bootverbose)
339 device_printf(dev, "ERROR: Unknown index %d.\n", index);
340 return (ENOENT);
341 }
342
343 static int
generic_pcie_write_ivar(device_t dev,device_t child,int index,uintptr_t value)344 generic_pcie_write_ivar(device_t dev, device_t child, int index,
345 uintptr_t value)
346 {
347
348 return (ENOENT);
349 }
350
351 static struct rman *
generic_pcie_rman(struct generic_pcie_core_softc * sc,int type,int flags)352 generic_pcie_rman(struct generic_pcie_core_softc *sc, int type, int flags)
353 {
354
355 switch (type) {
356 case SYS_RES_IOPORT:
357 return (&sc->io_rman);
358 case SYS_RES_MEMORY:
359 if (sc->has_pmem && (flags & RF_PREFETCHABLE) != 0)
360 return (&sc->pmem_rman);
361 return (&sc->mem_rman);
362 default:
363 break;
364 }
365
366 return (NULL);
367 }
368
369 int
pci_host_generic_core_release_resource(device_t dev,device_t child,int type,int rid,struct resource * res)370 pci_host_generic_core_release_resource(device_t dev, device_t child, int type,
371 int rid, struct resource *res)
372 {
373 struct generic_pcie_core_softc *sc;
374 struct rman *rm;
375 int error;
376
377 sc = device_get_softc(dev);
378
379 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
380 if (type == PCI_RES_BUS) {
381 return (pci_domain_release_bus(sc->ecam, child, rid, res));
382 }
383 #endif
384
385 rm = generic_pcie_rman(sc, type, rman_get_flags(res));
386 if (rm != NULL) {
387 KASSERT(rman_is_region_manager(res, rm), ("rman mismatch"));
388 if (rman_get_flags(res) & RF_ACTIVE) {
389 error = bus_deactivate_resource(child, type, rid, res);
390 if (error)
391 return (error);
392 }
393 return (rman_release_resource(res));
394 }
395
396 return (bus_generic_release_resource(dev, child, type, rid, res));
397 }
398
399 static int
generic_pcie_translate_resource_common(device_t dev,int type,rman_res_t start,rman_res_t end,rman_res_t * new_start,rman_res_t * new_end)400 generic_pcie_translate_resource_common(device_t dev, int type, rman_res_t start,
401 rman_res_t end, rman_res_t *new_start, rman_res_t *new_end)
402 {
403 struct generic_pcie_core_softc *sc;
404 uint64_t phys_base;
405 uint64_t pci_base;
406 uint64_t size;
407 int i, space;
408 bool found;
409
410 sc = device_get_softc(dev);
411 /* Translate the address from a PCI address to a physical address */
412 switch (type) {
413 case SYS_RES_IOPORT:
414 case SYS_RES_MEMORY:
415 found = false;
416 for (i = 0; i < MAX_RANGES_TUPLES; i++) {
417 pci_base = sc->ranges[i].pci_base;
418 phys_base = sc->ranges[i].phys_base;
419 size = sc->ranges[i].size;
420
421 if (start < pci_base || start >= pci_base + size)
422 continue;
423
424 switch (FLAG_TYPE(sc->ranges[i].flags)) {
425 case FLAG_TYPE_MEM:
426 case FLAG_TYPE_PMEM:
427 space = SYS_RES_MEMORY;
428 break;
429 case FLAG_TYPE_IO:
430 space = SYS_RES_IOPORT;
431 break;
432 default:
433 space = -1;
434 continue;
435 }
436
437 if (type == space) {
438 *new_start = start - pci_base + phys_base;
439 *new_end = end - pci_base + phys_base;
440 found = true;
441 break;
442 }
443 }
444 break;
445 default:
446 /* No translation for non-memory types */
447 *new_start = start;
448 *new_end = end;
449 found = true;
450 break;
451 }
452
453 return (found ? 0 : ENOENT);
454 }
455
456 static int
generic_pcie_translate_resource(device_t bus,int type,rman_res_t start,rman_res_t * newstart)457 generic_pcie_translate_resource(device_t bus, int type,
458 rman_res_t start, rman_res_t *newstart)
459 {
460 rman_res_t newend; /* unused */
461
462 return (generic_pcie_translate_resource_common(
463 bus, type, start, 0, newstart, &newend));
464 }
465
466 struct resource *
pci_host_generic_core_alloc_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)467 pci_host_generic_core_alloc_resource(device_t dev, device_t child, int type,
468 int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
469 {
470 struct generic_pcie_core_softc *sc;
471 struct resource *res;
472 struct rman *rm;
473
474 sc = device_get_softc(dev);
475
476 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
477 if (type == PCI_RES_BUS) {
478 return (pci_domain_alloc_bus(sc->ecam, child, rid, start, end,
479 count, flags));
480 }
481 #endif
482
483 rm = generic_pcie_rman(sc, type, flags);
484 if (rm == NULL)
485 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
486 type, rid, start, end, count, flags));
487
488 if (bootverbose) {
489 device_printf(dev,
490 "rman_reserve_resource: start=%#jx, end=%#jx, count=%#jx\n",
491 start, end, count);
492 }
493
494 res = rman_reserve_resource(rm, start, end, count, flags, child);
495 if (res == NULL)
496 goto fail;
497
498 rman_set_rid(res, *rid);
499
500 if (flags & RF_ACTIVE)
501 if (bus_activate_resource(child, type, *rid, res)) {
502 rman_release_resource(res);
503 goto fail;
504 }
505
506 return (res);
507
508 fail:
509 device_printf(dev, "%s FAIL: type=%d, rid=%d, "
510 "start=%016jx, end=%016jx, count=%016jx, flags=%x\n",
511 __func__, type, *rid, start, end, count, flags);
512
513 return (NULL);
514 }
515
516 static int
generic_pcie_activate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)517 generic_pcie_activate_resource(device_t dev, device_t child, int type,
518 int rid, struct resource *r)
519 {
520 rman_res_t start, end;
521 int res;
522
523 if ((res = rman_activate_resource(r)) != 0)
524 return (res);
525
526 start = rman_get_start(r);
527 end = rman_get_end(r);
528 res = generic_pcie_translate_resource_common(dev, type, start, end,
529 &start, &end);
530 if (res != 0) {
531 rman_deactivate_resource(r);
532 return (res);
533 }
534 rman_set_start(r, start);
535 rman_set_end(r, end);
536
537 return (BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type,
538 rid, r));
539 }
540
541 static int
generic_pcie_deactivate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)542 generic_pcie_deactivate_resource(device_t dev, device_t child, int type,
543 int rid, struct resource *r)
544 {
545 int res;
546
547 if ((res = rman_deactivate_resource(r)) != 0)
548 return (res);
549
550 switch (type) {
551 case SYS_RES_IOPORT:
552 case SYS_RES_MEMORY:
553 case SYS_RES_IRQ:
554 res = BUS_DEACTIVATE_RESOURCE(device_get_parent(dev), child,
555 type, rid, r);
556 break;
557 default:
558 break;
559 }
560
561 return (res);
562 }
563
564 static int
generic_pcie_adjust_resource(device_t dev,device_t child,int type,struct resource * res,rman_res_t start,rman_res_t end)565 generic_pcie_adjust_resource(device_t dev, device_t child, int type,
566 struct resource *res, rman_res_t start, rman_res_t end)
567 {
568 struct generic_pcie_core_softc *sc;
569 struct rman *rm;
570
571 sc = device_get_softc(dev);
572 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
573 if (type == PCI_RES_BUS)
574 return (pci_domain_adjust_bus(sc->ecam, child, res, start,
575 end));
576 #endif
577
578 rm = generic_pcie_rman(sc, type, rman_get_flags(res));
579 if (rm != NULL)
580 return (rman_adjust_resource(res, start, end));
581 return (bus_generic_adjust_resource(dev, child, type, res, start, end));
582 }
583
584 static bus_dma_tag_t
generic_pcie_get_dma_tag(device_t dev,device_t child)585 generic_pcie_get_dma_tag(device_t dev, device_t child)
586 {
587 struct generic_pcie_core_softc *sc;
588
589 sc = device_get_softc(dev);
590 return (sc->dmat);
591 }
592
593 static device_method_t generic_pcie_methods[] = {
594 DEVMETHOD(device_attach, pci_host_generic_core_attach),
595 DEVMETHOD(device_detach, pci_host_generic_core_detach),
596
597 DEVMETHOD(bus_read_ivar, generic_pcie_read_ivar),
598 DEVMETHOD(bus_write_ivar, generic_pcie_write_ivar),
599 DEVMETHOD(bus_alloc_resource, pci_host_generic_core_alloc_resource),
600 DEVMETHOD(bus_adjust_resource, generic_pcie_adjust_resource),
601 DEVMETHOD(bus_activate_resource, generic_pcie_activate_resource),
602 DEVMETHOD(bus_deactivate_resource, generic_pcie_deactivate_resource),
603 DEVMETHOD(bus_release_resource, pci_host_generic_core_release_resource),
604 DEVMETHOD(bus_translate_resource, generic_pcie_translate_resource),
605 DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
606 DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
607
608 DEVMETHOD(bus_get_dma_tag, generic_pcie_get_dma_tag),
609
610 /* pcib interface */
611 DEVMETHOD(pcib_maxslots, generic_pcie_maxslots),
612 DEVMETHOD(pcib_read_config, generic_pcie_read_config),
613 DEVMETHOD(pcib_write_config, generic_pcie_write_config),
614
615 DEVMETHOD_END
616 };
617
618 DEFINE_CLASS_0(pcib, generic_pcie_core_driver,
619 generic_pcie_methods, sizeof(struct generic_pcie_core_softc));
620