1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <[email protected]>
5 * Copyright (c) 2000, Michael Smith <[email protected]>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52
53 #include <vm/vm.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_extern.h>
56
57 #include <sys/bus.h>
58 #include <machine/bus.h>
59 #include <sys/rman.h>
60 #include <machine/resource.h>
61 #include <machine/stdarg.h>
62
63 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
64 #include <machine/intr_machdep.h>
65 #endif
66
67 #include <sys/pciio.h>
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 #include <dev/pci/pci_private.h>
71
72 #ifdef PCI_IOV
73 #include <sys/nv.h>
74 #include <dev/pci/pci_iov_private.h>
75 #endif
76
77 #include <dev/usb/controller/xhcireg.h>
78 #include <dev/usb/controller/ehcireg.h>
79 #include <dev/usb/controller/ohcireg.h>
80 #include <dev/usb/controller/uhcireg.h>
81
82 #include <dev/iommu/iommu.h>
83
84 #include "pcib_if.h"
85 #include "pci_if.h"
86
87 #define PCIR_IS_BIOS(cfg, reg) \
88 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
89 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
90
91 static int pci_has_quirk(uint32_t devid, int quirk);
92 static pci_addr_t pci_mapbase(uint64_t mapreg);
93 static const char *pci_maptype(uint64_t mapreg);
94 static int pci_maprange(uint64_t mapreg);
95 static pci_addr_t pci_rombase(uint64_t mapreg);
96 static int pci_romsize(uint64_t testval);
97 static void pci_fixancient(pcicfgregs *cfg);
98 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
99
100 static int pci_porten(device_t dev);
101 static int pci_memen(device_t dev);
102 static void pci_assign_interrupt(device_t bus, device_t dev,
103 int force_route);
104 static int pci_add_map(device_t bus, device_t dev, int reg,
105 struct resource_list *rl, int force, int prefetch);
106 static int pci_probe(device_t dev);
107 static void pci_load_vendor_data(void);
108 static int pci_describe_parse_line(char **ptr, int *vendor,
109 int *device, char **desc);
110 static char *pci_describe_device(device_t dev);
111 static int pci_modevent(module_t mod, int what, void *arg);
112 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
113 pcicfgregs *cfg);
114 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
115 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
116 int reg, uint32_t *data);
117 #if 0
118 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
119 int reg, uint32_t data);
120 #endif
121 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
122 static void pci_mask_msix(device_t dev, u_int index);
123 static void pci_unmask_msix(device_t dev, u_int index);
124 static int pci_msi_blacklisted(void);
125 static int pci_msix_blacklisted(void);
126 static void pci_resume_msi(device_t dev);
127 static void pci_resume_msix(device_t dev);
128 static int pci_remap_intr_method(device_t bus, device_t dev,
129 u_int irq);
130 static void pci_hint_device_unit(device_t acdev, device_t child,
131 const char *name, int *unitp);
132 static int pci_reset_post(device_t dev, device_t child);
133 static int pci_reset_prepare(device_t dev, device_t child);
134 static int pci_reset_child(device_t dev, device_t child,
135 int flags);
136
137 static int pci_get_id_method(device_t dev, device_t child,
138 enum pci_id_type type, uintptr_t *rid);
139 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
140 int b, int s, int f, uint16_t vid, uint16_t did);
141
142 static device_method_t pci_methods[] = {
143 /* Device interface */
144 DEVMETHOD(device_probe, pci_probe),
145 DEVMETHOD(device_attach, pci_attach),
146 DEVMETHOD(device_detach, pci_detach),
147 DEVMETHOD(device_shutdown, bus_generic_shutdown),
148 DEVMETHOD(device_suspend, bus_generic_suspend),
149 DEVMETHOD(device_resume, pci_resume),
150
151 /* Bus interface */
152 DEVMETHOD(bus_print_child, pci_print_child),
153 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
154 DEVMETHOD(bus_read_ivar, pci_read_ivar),
155 DEVMETHOD(bus_write_ivar, pci_write_ivar),
156 DEVMETHOD(bus_driver_added, pci_driver_added),
157 DEVMETHOD(bus_setup_intr, pci_setup_intr),
158 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
159 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
160 DEVMETHOD(bus_reset_post, pci_reset_post),
161 DEVMETHOD(bus_reset_child, pci_reset_child),
162
163 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
164 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
165 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
166 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
167 DEVMETHOD(bus_delete_resource, pci_delete_resource),
168 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
169 DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
170 DEVMETHOD(bus_release_resource, pci_release_resource),
171 DEVMETHOD(bus_activate_resource, pci_activate_resource),
172 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
173 DEVMETHOD(bus_child_deleted, pci_child_deleted),
174 DEVMETHOD(bus_child_detached, pci_child_detached),
175 DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
176 DEVMETHOD(bus_child_location, pci_child_location_method),
177 DEVMETHOD(bus_get_device_path, pci_get_device_path_method),
178 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
179 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
180 DEVMETHOD(bus_suspend_child, pci_suspend_child),
181 DEVMETHOD(bus_resume_child, pci_resume_child),
182 DEVMETHOD(bus_rescan, pci_rescan_method),
183
184 /* PCI interface */
185 DEVMETHOD(pci_read_config, pci_read_config_method),
186 DEVMETHOD(pci_write_config, pci_write_config_method),
187 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
188 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
189 DEVMETHOD(pci_enable_io, pci_enable_io_method),
190 DEVMETHOD(pci_disable_io, pci_disable_io_method),
191 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
192 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
193 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
194 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
195 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
196 DEVMETHOD(pci_find_cap, pci_find_cap_method),
197 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
198 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
199 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
200 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
201 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
202 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
203 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
204 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
205 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
206 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
207 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
208 DEVMETHOD(pci_release_msi, pci_release_msi_method),
209 DEVMETHOD(pci_msi_count, pci_msi_count_method),
210 DEVMETHOD(pci_msix_count, pci_msix_count_method),
211 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
212 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
213 DEVMETHOD(pci_get_id, pci_get_id_method),
214 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
215 DEVMETHOD(pci_child_added, pci_child_added_method),
216 #ifdef PCI_IOV
217 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
218 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
219 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
220 #endif
221
222 DEVMETHOD_END
223 };
224
225 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
226
227 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS);
228 MODULE_VERSION(pci, 1);
229
230 static char *pci_vendordata;
231 static size_t pci_vendordata_size;
232
233 struct pci_quirk {
234 uint32_t devid; /* Vendor/device of the card */
235 int type;
236 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
237 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
238 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
239 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
240 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
241 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
242 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
243 int arg1;
244 int arg2;
245 };
246
247 static const struct pci_quirk pci_quirks[] = {
248 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
249 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
250 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
251 /* As does the Serverworks OSB4 (the SMBus mapping register) */
252 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
253
254 /*
255 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
256 * or the CMIC-SL (AKA ServerWorks GC_LE).
257 */
258 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
259 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
260
261 /*
262 * MSI doesn't work on earlier Intel chipsets including
263 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
264 */
265 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
266 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
267 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
268 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
269 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
270 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
271 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272
273 /*
274 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
275 * bridge.
276 */
277 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
278
279 /*
280 * Some virtualization environments emulate an older chipset
281 * but support MSI just fine. QEMU uses the Intel 82440.
282 */
283 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
284
285 /*
286 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
287 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
288 * It prevents us from attaching hpet(4) when the bit is unset.
289 * Note this quirk only affects SB600 revision A13 and earlier.
290 * For SB600 A21 and later, firmware must set the bit to hide it.
291 * For SB700 and later, it is unused and hardcoded to zero.
292 */
293 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
294
295 /*
296 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
297 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
298 * of the command register is set.
299 */
300 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
301 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
302 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
303 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
304 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
305
306 /*
307 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
308 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
309 */
310 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
311 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
312 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
313 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
314 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
315 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
316
317 /*
318 * HPE Gen 10 VGA has a memory range that can't be allocated in the
319 * expected place.
320 */
321 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
322 { 0 }
323 };
324
325 /* map register information */
326 #define PCI_MAPMEM 0x01 /* memory map */
327 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
328 #define PCI_MAPPORT 0x04 /* port map */
329
330 struct devlist pci_devq;
331 uint32_t pci_generation;
332 uint32_t pci_numdevs = 0;
333 static int pcie_chipset, pcix_chipset;
334
335 /* sysctl vars */
336 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
337 "PCI bus tuning parameters");
338
339 static int pci_enable_io_modes = 1;
340 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
341 &pci_enable_io_modes, 1,
342 "Enable I/O and memory bits in the config register. Some BIOSes do not"
343 " enable these bits correctly. We'd like to do this all the time, but"
344 " there are some peripherals that this causes problems with.");
345
346 static int pci_do_realloc_bars = 1;
347 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
348 &pci_do_realloc_bars, 0,
349 "Attempt to allocate a new range for any BARs whose original "
350 "firmware-assigned ranges fail to allocate during the initial device scan.");
351
352 static int pci_do_power_nodriver = 0;
353 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
354 &pci_do_power_nodriver, 0,
355 "Place a function into D3 state when no driver attaches to it. 0 means"
356 " disable. 1 means conservatively place devices into D3 state. 2 means"
357 " aggressively place devices into D3 state. 3 means put absolutely"
358 " everything in D3 state.");
359
360 int pci_do_power_resume = 1;
361 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
362 &pci_do_power_resume, 1,
363 "Transition from D3 -> D0 on resume.");
364
365 int pci_do_power_suspend = 1;
366 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
367 &pci_do_power_suspend, 1,
368 "Transition from D0 -> D3 on suspend.");
369
370 static int pci_do_msi = 1;
371 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
372 "Enable support for MSI interrupts");
373
374 static int pci_do_msix = 1;
375 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
376 "Enable support for MSI-X interrupts");
377
378 static int pci_msix_rewrite_table = 0;
379 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
380 &pci_msix_rewrite_table, 0,
381 "Rewrite entire MSI-X table when updating MSI-X entries");
382
383 static int pci_honor_msi_blacklist = 1;
384 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
385 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
386
387 #if defined(__i386__) || defined(__amd64__)
388 static int pci_usb_takeover = 1;
389 #else
390 static int pci_usb_takeover = 0;
391 #endif
392 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
393 &pci_usb_takeover, 1,
394 "Enable early takeover of USB controllers. Disable this if you depend on"
395 " BIOS emulation of USB devices, that is you use USB devices (like"
396 " keyboard or mouse) but do not load USB drivers");
397
398 static int pci_clear_bars;
399 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
400 "Ignore firmware-assigned resources for BARs.");
401
402 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
403 static int pci_clear_buses;
404 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
405 "Ignore firmware-assigned bus numbers.");
406 #endif
407
408 static int pci_enable_ari = 1;
409 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
410 0, "Enable support for PCIe Alternative RID Interpretation");
411
412 int pci_enable_aspm = 1;
413 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
414 0, "Enable support for PCIe Active State Power Management");
415
416 static int pci_clear_aer_on_attach = 0;
417 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
418 &pci_clear_aer_on_attach, 0,
419 "Clear port and device AER state on driver attach");
420
421 static bool pci_enable_mps_tune = true;
422 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
423 &pci_enable_mps_tune, 1,
424 "Enable tuning of MPS(maximum payload size)." );
425
426 static int
pci_has_quirk(uint32_t devid,int quirk)427 pci_has_quirk(uint32_t devid, int quirk)
428 {
429 const struct pci_quirk *q;
430
431 for (q = &pci_quirks[0]; q->devid; q++) {
432 if (q->devid == devid && q->type == quirk)
433 return (1);
434 }
435 return (0);
436 }
437
438 /* Find a device_t by bus/slot/function in domain 0 */
439
440 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)441 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
442 {
443
444 return (pci_find_dbsf(0, bus, slot, func));
445 }
446
447 /* Find a device_t by domain/bus/slot/function */
448
449 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)450 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
451 {
452 struct pci_devinfo *dinfo = NULL;
453
454 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
455 if ((dinfo->cfg.domain == domain) &&
456 (dinfo->cfg.bus == bus) &&
457 (dinfo->cfg.slot == slot) &&
458 (dinfo->cfg.func == func)) {
459 break;
460 }
461 }
462
463 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
464 }
465
466 /* Find a device_t by vendor/device ID */
467
468 device_t
pci_find_device(uint16_t vendor,uint16_t device)469 pci_find_device(uint16_t vendor, uint16_t device)
470 {
471 struct pci_devinfo *dinfo;
472
473 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
474 if ((dinfo->cfg.vendor == vendor) &&
475 (dinfo->cfg.device == device)) {
476 return (dinfo->cfg.dev);
477 }
478 }
479
480 return (NULL);
481 }
482
483 device_t
pci_find_class(uint8_t class,uint8_t subclass)484 pci_find_class(uint8_t class, uint8_t subclass)
485 {
486 struct pci_devinfo *dinfo;
487
488 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
489 if (dinfo->cfg.baseclass == class &&
490 dinfo->cfg.subclass == subclass) {
491 return (dinfo->cfg.dev);
492 }
493 }
494
495 return (NULL);
496 }
497
498 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)499 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
500 {
501 struct pci_devinfo *dinfo;
502 bool found = false;
503
504 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
505 if (from != NULL && found == false) {
506 if (from != dinfo->cfg.dev)
507 continue;
508 found = true;
509 continue;
510 }
511 if (dinfo->cfg.baseclass == class &&
512 dinfo->cfg.subclass == subclass) {
513 return (dinfo->cfg.dev);
514 }
515 }
516
517 return (NULL);
518 }
519
520 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)521 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
522 {
523 va_list ap;
524 int retval;
525
526 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
527 cfg->func);
528 va_start(ap, fmt);
529 retval += vprintf(fmt, ap);
530 va_end(ap);
531 return (retval);
532 }
533
534 /* return base address of memory or port map */
535
536 static pci_addr_t
pci_mapbase(uint64_t mapreg)537 pci_mapbase(uint64_t mapreg)
538 {
539
540 if (PCI_BAR_MEM(mapreg))
541 return (mapreg & PCIM_BAR_MEM_BASE);
542 else
543 return (mapreg & PCIM_BAR_IO_BASE);
544 }
545
546 /* return map type of memory or port map */
547
548 static const char *
pci_maptype(uint64_t mapreg)549 pci_maptype(uint64_t mapreg)
550 {
551
552 if (PCI_BAR_IO(mapreg))
553 return ("I/O Port");
554 if (mapreg & PCIM_BAR_MEM_PREFETCH)
555 return ("Prefetchable Memory");
556 return ("Memory");
557 }
558
559 /* return log2 of map size decoded for memory or port map */
560
561 int
pci_mapsize(uint64_t testval)562 pci_mapsize(uint64_t testval)
563 {
564 int ln2size;
565
566 testval = pci_mapbase(testval);
567 ln2size = 0;
568 if (testval != 0) {
569 while ((testval & 1) == 0)
570 {
571 ln2size++;
572 testval >>= 1;
573 }
574 }
575 return (ln2size);
576 }
577
578 /* return base address of device ROM */
579
580 static pci_addr_t
pci_rombase(uint64_t mapreg)581 pci_rombase(uint64_t mapreg)
582 {
583
584 return (mapreg & PCIM_BIOS_ADDR_MASK);
585 }
586
587 /* return log2 of map size decided for device ROM */
588
589 static int
pci_romsize(uint64_t testval)590 pci_romsize(uint64_t testval)
591 {
592 int ln2size;
593
594 testval = pci_rombase(testval);
595 ln2size = 0;
596 if (testval != 0) {
597 while ((testval & 1) == 0)
598 {
599 ln2size++;
600 testval >>= 1;
601 }
602 }
603 return (ln2size);
604 }
605
606 /* return log2 of address range supported by map register */
607
608 static int
pci_maprange(uint64_t mapreg)609 pci_maprange(uint64_t mapreg)
610 {
611 int ln2range = 0;
612
613 if (PCI_BAR_IO(mapreg))
614 ln2range = 32;
615 else
616 switch (mapreg & PCIM_BAR_MEM_TYPE) {
617 case PCIM_BAR_MEM_32:
618 ln2range = 32;
619 break;
620 case PCIM_BAR_MEM_1MB:
621 ln2range = 20;
622 break;
623 case PCIM_BAR_MEM_64:
624 ln2range = 64;
625 break;
626 }
627 return (ln2range);
628 }
629
630 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
631
632 static void
pci_fixancient(pcicfgregs * cfg)633 pci_fixancient(pcicfgregs *cfg)
634 {
635 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
636 return;
637
638 /* PCI to PCI bridges use header type 1 */
639 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
640 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
641 }
642
643 /* extract header type specific config data */
644
645 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)646 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
647 {
648 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
649 switch (cfg->hdrtype & PCIM_HDRTYPE) {
650 case PCIM_HDRTYPE_NORMAL:
651 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
652 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
653 cfg->mingnt = REG(PCIR_MINGNT, 1);
654 cfg->maxlat = REG(PCIR_MAXLAT, 1);
655 cfg->nummaps = PCI_MAXMAPS_0;
656 break;
657 case PCIM_HDRTYPE_BRIDGE:
658 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
659 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
660 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
661 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
662 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
663 cfg->nummaps = PCI_MAXMAPS_1;
664 break;
665 case PCIM_HDRTYPE_CARDBUS:
666 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
667 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
668 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
669 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
670 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
671 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
672 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
673 cfg->nummaps = PCI_MAXMAPS_2;
674 break;
675 }
676 #undef REG
677 }
678
679 /* read configuration header into pcicfgregs structure */
680 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)681 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
682 {
683 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
684 uint16_t vid, did;
685
686 vid = REG(PCIR_VENDOR, 2);
687 if (vid == PCIV_INVALID)
688 return (NULL);
689
690 did = REG(PCIR_DEVICE, 2);
691
692 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
693 }
694
695 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)696 pci_alloc_devinfo_method(device_t dev)
697 {
698
699 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
700 M_WAITOK | M_ZERO));
701 }
702
703 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)704 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
705 uint16_t vid, uint16_t did)
706 {
707 struct pci_devinfo *devlist_entry;
708 pcicfgregs *cfg;
709
710 devlist_entry = PCI_ALLOC_DEVINFO(bus);
711
712 cfg = &devlist_entry->cfg;
713
714 cfg->domain = d;
715 cfg->bus = b;
716 cfg->slot = s;
717 cfg->func = f;
718 cfg->vendor = vid;
719 cfg->device = did;
720 cfg->cmdreg = REG(PCIR_COMMAND, 2);
721 cfg->statreg = REG(PCIR_STATUS, 2);
722 cfg->baseclass = REG(PCIR_CLASS, 1);
723 cfg->subclass = REG(PCIR_SUBCLASS, 1);
724 cfg->progif = REG(PCIR_PROGIF, 1);
725 cfg->revid = REG(PCIR_REVID, 1);
726 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
727 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
728 cfg->lattimer = REG(PCIR_LATTIMER, 1);
729 cfg->intpin = REG(PCIR_INTPIN, 1);
730 cfg->intline = REG(PCIR_INTLINE, 1);
731
732 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
733 cfg->hdrtype &= ~PCIM_MFDEV;
734 STAILQ_INIT(&cfg->maps);
735
736 cfg->iov = NULL;
737
738 pci_fixancient(cfg);
739 pci_hdrtypedata(pcib, b, s, f, cfg);
740
741 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
742 pci_read_cap(pcib, cfg);
743
744 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
745
746 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
747 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
748 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
749 devlist_entry->conf.pc_sel.pc_func = cfg->func;
750 devlist_entry->conf.pc_hdr = cfg->hdrtype;
751
752 devlist_entry->conf.pc_subvendor = cfg->subvendor;
753 devlist_entry->conf.pc_subdevice = cfg->subdevice;
754 devlist_entry->conf.pc_vendor = cfg->vendor;
755 devlist_entry->conf.pc_device = cfg->device;
756
757 devlist_entry->conf.pc_class = cfg->baseclass;
758 devlist_entry->conf.pc_subclass = cfg->subclass;
759 devlist_entry->conf.pc_progif = cfg->progif;
760 devlist_entry->conf.pc_revid = cfg->revid;
761
762 pci_numdevs++;
763 pci_generation++;
764
765 return (devlist_entry);
766 }
767 #undef REG
768
769 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)770 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
771 {
772 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
773 cfg->ea.ea_location + (n), w)
774 int num_ent;
775 int ptr;
776 int a, b;
777 uint32_t val;
778 int ent_size;
779 uint32_t dw[4];
780 uint64_t base, max_offset;
781 struct pci_ea_entry *eae;
782
783 if (cfg->ea.ea_location == 0)
784 return;
785
786 STAILQ_INIT(&cfg->ea.ea_entries);
787
788 /* Determine the number of entries */
789 num_ent = REG(PCIR_EA_NUM_ENT, 2);
790 num_ent &= PCIM_EA_NUM_ENT_MASK;
791
792 /* Find the first entry to care of */
793 ptr = PCIR_EA_FIRST_ENT;
794
795 /* Skip DWORD 2 for type 1 functions */
796 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
797 ptr += 4;
798
799 for (a = 0; a < num_ent; a++) {
800 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
801 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
802
803 /* Read a number of dwords in the entry */
804 val = REG(ptr, 4);
805 ptr += 4;
806 ent_size = (val & PCIM_EA_ES);
807
808 for (b = 0; b < ent_size; b++) {
809 dw[b] = REG(ptr, 4);
810 ptr += 4;
811 }
812
813 eae->eae_flags = val;
814 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
815
816 base = dw[0] & PCIM_EA_FIELD_MASK;
817 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
818 b = 2;
819 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
820 base |= (uint64_t)dw[b] << 32UL;
821 b++;
822 }
823 if (((dw[1] & PCIM_EA_IS_64) != 0)
824 && (b < ent_size)) {
825 max_offset |= (uint64_t)dw[b] << 32UL;
826 b++;
827 }
828
829 eae->eae_base = base;
830 eae->eae_max_offset = max_offset;
831
832 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
833
834 if (bootverbose) {
835 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
836 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
837 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
838 }
839 }
840 }
841 #undef REG
842
843 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)844 pci_read_cap(device_t pcib, pcicfgregs *cfg)
845 {
846 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
847 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
848 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
849 uint64_t addr;
850 #endif
851 uint32_t val;
852 int ptr, nextptr, ptrptr;
853
854 switch (cfg->hdrtype & PCIM_HDRTYPE) {
855 case PCIM_HDRTYPE_NORMAL:
856 case PCIM_HDRTYPE_BRIDGE:
857 ptrptr = PCIR_CAP_PTR;
858 break;
859 case PCIM_HDRTYPE_CARDBUS:
860 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
861 break;
862 default:
863 return; /* no extended capabilities support */
864 }
865 nextptr = REG(ptrptr, 1); /* sanity check? */
866
867 /*
868 * Read capability entries.
869 */
870 while (nextptr != 0) {
871 /* Sanity check */
872 if (nextptr > 255) {
873 printf("illegal PCI extended capability offset %d\n",
874 nextptr);
875 return;
876 }
877 /* Find the next entry */
878 ptr = nextptr;
879 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
880
881 /* Process this entry */
882 switch (REG(ptr + PCICAP_ID, 1)) {
883 case PCIY_PMG: /* PCI power management */
884 if (cfg->pp.pp_cap == 0) {
885 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
886 cfg->pp.pp_status = ptr + PCIR_POWER_STATUS;
887 cfg->pp.pp_bse = ptr + PCIR_POWER_BSE;
888 if ((nextptr - ptr) > PCIR_POWER_DATA)
889 cfg->pp.pp_data = ptr + PCIR_POWER_DATA;
890 }
891 break;
892 case PCIY_HT: /* HyperTransport */
893 /* Determine HT-specific capability type. */
894 val = REG(ptr + PCIR_HT_COMMAND, 2);
895
896 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
897 cfg->ht.ht_slave = ptr;
898
899 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
900 switch (val & PCIM_HTCMD_CAP_MASK) {
901 case PCIM_HTCAP_MSI_MAPPING:
902 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
903 /* Sanity check the mapping window. */
904 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
905 4);
906 addr <<= 32;
907 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
908 4);
909 if (addr != MSI_INTEL_ADDR_BASE)
910 device_printf(pcib,
911 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
912 cfg->domain, cfg->bus,
913 cfg->slot, cfg->func,
914 (long long)addr);
915 } else
916 addr = MSI_INTEL_ADDR_BASE;
917
918 cfg->ht.ht_msimap = ptr;
919 cfg->ht.ht_msictrl = val;
920 cfg->ht.ht_msiaddr = addr;
921 break;
922 }
923 #endif
924 break;
925 case PCIY_MSI: /* PCI MSI */
926 cfg->msi.msi_location = ptr;
927 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
928 cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl &
929 PCIM_MSICTRL_MMC_MASK)>>1);
930 break;
931 case PCIY_MSIX: /* PCI MSI-X */
932 cfg->msix.msix_location = ptr;
933 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
934 cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl &
935 PCIM_MSIXCTRL_TABLE_SIZE) + 1;
936 val = REG(ptr + PCIR_MSIX_TABLE, 4);
937 cfg->msix.msix_table_bar = PCIR_BAR(val &
938 PCIM_MSIX_BIR_MASK);
939 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
940 val = REG(ptr + PCIR_MSIX_PBA, 4);
941 cfg->msix.msix_pba_bar = PCIR_BAR(val &
942 PCIM_MSIX_BIR_MASK);
943 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
944 break;
945 case PCIY_VPD: /* PCI Vital Product Data */
946 cfg->vpd.vpd_reg = ptr;
947 break;
948 case PCIY_SUBVENDOR:
949 /* Should always be true. */
950 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
951 PCIM_HDRTYPE_BRIDGE) {
952 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
953 cfg->subvendor = val & 0xffff;
954 cfg->subdevice = val >> 16;
955 }
956 break;
957 case PCIY_PCIX: /* PCI-X */
958 /*
959 * Assume we have a PCI-X chipset if we have
960 * at least one PCI-PCI bridge with a PCI-X
961 * capability. Note that some systems with
962 * PCI-express or HT chipsets might match on
963 * this check as well.
964 */
965 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
966 PCIM_HDRTYPE_BRIDGE)
967 pcix_chipset = 1;
968 cfg->pcix.pcix_location = ptr;
969 break;
970 case PCIY_EXPRESS: /* PCI-express */
971 /*
972 * Assume we have a PCI-express chipset if we have
973 * at least one PCI-express device.
974 */
975 pcie_chipset = 1;
976 cfg->pcie.pcie_location = ptr;
977 val = REG(ptr + PCIER_FLAGS, 2);
978 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
979 break;
980 case PCIY_EA: /* Enhanced Allocation */
981 cfg->ea.ea_location = ptr;
982 pci_ea_fill_info(pcib, cfg);
983 break;
984 default:
985 break;
986 }
987 }
988
989 #if defined(__powerpc__)
990 /*
991 * Enable the MSI mapping window for all HyperTransport
992 * slaves. PCI-PCI bridges have their windows enabled via
993 * PCIB_MAP_MSI().
994 */
995 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
996 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
997 device_printf(pcib,
998 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
999 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1000 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1001 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1002 2);
1003 }
1004 #endif
1005 /* REG and WREG use carry through to next functions */
1006 }
1007
1008 /*
1009 * PCI Vital Product Data
1010 */
1011
1012 #define PCI_VPD_TIMEOUT 1000000
1013
1014 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1015 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1016 {
1017 int count = PCI_VPD_TIMEOUT;
1018
1019 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1020
1021 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1022
1023 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1024 if (--count < 0)
1025 return (ENXIO);
1026 DELAY(1); /* limit looping */
1027 }
1028 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1029
1030 return (0);
1031 }
1032
1033 #if 0
1034 static int
1035 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1036 {
1037 int count = PCI_VPD_TIMEOUT;
1038
1039 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1040
1041 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1042 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1043 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1044 if (--count < 0)
1045 return (ENXIO);
1046 DELAY(1); /* limit looping */
1047 }
1048
1049 return (0);
1050 }
1051 #endif
1052
1053 #undef PCI_VPD_TIMEOUT
1054
1055 struct vpd_readstate {
1056 device_t pcib;
1057 pcicfgregs *cfg;
1058 uint32_t val;
1059 int bytesinval;
1060 int off;
1061 uint8_t cksum;
1062 };
1063
1064 /* return 0 and one byte in *data if no read error, -1 else */
1065 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1066 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1067 {
1068 uint32_t reg;
1069 uint8_t byte;
1070
1071 if (vrs->bytesinval == 0) {
1072 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1073 return (-1);
1074 vrs->val = le32toh(reg);
1075 vrs->off += 4;
1076 byte = vrs->val & 0xff;
1077 vrs->bytesinval = 3;
1078 } else {
1079 vrs->val = vrs->val >> 8;
1080 byte = vrs->val & 0xff;
1081 vrs->bytesinval--;
1082 }
1083
1084 vrs->cksum += byte;
1085 *data = byte;
1086 return (0);
1087 }
1088
1089 /* return 0 on match, -1 and "unget" byte on no match */
1090 static int
vpd_expectbyte(struct vpd_readstate * vrs,uint8_t expected)1091 vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected)
1092 {
1093 uint8_t data;
1094
1095 if (vpd_nextbyte(vrs, &data) != 0)
1096 return (-1);
1097
1098 if (data == expected)
1099 return (0);
1100
1101 vrs->cksum -= data;
1102 vrs->val = (vrs->val << 8) + data;
1103 vrs->bytesinval++;
1104 return (-1);
1105 }
1106
1107 /* return size if tag matches, -1 on no match, -2 on read error */
1108 static int
vpd_read_tag_size(struct vpd_readstate * vrs,uint8_t vpd_tag)1109 vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag)
1110 {
1111 uint8_t byte1, byte2;
1112
1113 if (vpd_expectbyte(vrs, vpd_tag) != 0)
1114 return (-1);
1115
1116 if ((vpd_tag & 0x80) == 0)
1117 return (vpd_tag & 0x07);
1118
1119 if (vpd_nextbyte(vrs, &byte1) != 0)
1120 return (-2);
1121 if (vpd_nextbyte(vrs, &byte2) != 0)
1122 return (-2);
1123
1124 return ((byte2 << 8) + byte1);
1125 }
1126
1127 /* (re)allocate buffer in multiples of 8 elements */
1128 static void*
alloc_buffer(void * buffer,size_t element_size,int needed)1129 alloc_buffer(void* buffer, size_t element_size, int needed)
1130 {
1131 int alloc, new_alloc;
1132
1133 alloc = roundup2(needed, 8);
1134 new_alloc = roundup2(needed + 1, 8);
1135 if (alloc != new_alloc) {
1136 buffer = reallocf(buffer,
1137 new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO);
1138 }
1139
1140 return (buffer);
1141 }
1142
1143 /* read VPD keyword and return element size, return -1 on read error */
1144 static int
vpd_read_elem_head(struct vpd_readstate * vrs,char keyword[2])1145 vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2])
1146 {
1147 uint8_t data;
1148
1149 if (vpd_nextbyte(vrs, &keyword[0]) != 0)
1150 return (-1);
1151 if (vpd_nextbyte(vrs, &keyword[1]) != 0)
1152 return (-1);
1153 if (vpd_nextbyte(vrs, &data) != 0)
1154 return (-1);
1155
1156 return (data);
1157 }
1158
1159 /* read VPD data element of given size into allocated buffer */
1160 static char *
vpd_read_value(struct vpd_readstate * vrs,int size)1161 vpd_read_value(struct vpd_readstate *vrs, int size)
1162 {
1163 int i;
1164 char char1;
1165 char *value;
1166
1167 value = malloc(size + 1, M_DEVBUF, M_WAITOK);
1168 for (i = 0; i < size; i++) {
1169 if (vpd_nextbyte(vrs, &char1) != 0) {
1170 free(value, M_DEVBUF);
1171 return (NULL);
1172 }
1173 value[i] = char1;
1174 }
1175 value[size] = '\0';
1176
1177 return (value);
1178 }
1179
1180 /* read VPD into *keyword and *value, return length of data element */
1181 static int
vpd_read_elem_data(struct vpd_readstate * vrs,char keyword[2],char ** value,int maxlen)1182 vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen)
1183 {
1184 int len;
1185
1186 len = vpd_read_elem_head(vrs, keyword);
1187 if (len > maxlen)
1188 return (-1);
1189 *value = vpd_read_value(vrs, len);
1190
1191 return (len);
1192 }
1193
1194 /* subtract all data following first byte from checksum of RV element */
1195 static void
vpd_fixup_cksum(struct vpd_readstate * vrs,char * rvstring,int len)1196 vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len)
1197 {
1198 int i;
1199 uint8_t fixup;
1200
1201 fixup = 0;
1202 for (i = 1; i < len; i++)
1203 fixup += rvstring[i];
1204 vrs->cksum -= fixup;
1205 }
1206
1207 /* fetch one read-only element and return size of heading + data */
1208 static size_t
next_vpd_ro_elem(struct vpd_readstate * vrs,int maxsize)1209 next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize)
1210 {
1211 struct pcicfg_vpd *vpd;
1212 pcicfgregs *cfg;
1213 struct vpd_readonly *vpd_ros;
1214 int len;
1215
1216 cfg = vrs->cfg;
1217 vpd = &cfg->vpd;
1218
1219 if (maxsize < 3)
1220 return (-1);
1221 vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt);
1222 vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt];
1223 maxsize -= 3;
1224 len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize);
1225 if (vpd_ros->value == NULL)
1226 return (-1);
1227 vpd_ros->len = len;
1228 if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') {
1229 vpd_fixup_cksum(vrs, vpd_ros->value, len);
1230 if (vrs->cksum != 0) {
1231 pci_printf(cfg,
1232 "invalid VPD checksum %#hhx\n", vrs->cksum);
1233 return (-1);
1234 }
1235 }
1236 vpd->vpd_rocnt++;
1237
1238 return (len + 3);
1239 }
1240
1241 /* fetch one writable element and return size of heading + data */
1242 static size_t
next_vpd_rw_elem(struct vpd_readstate * vrs,int maxsize)1243 next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize)
1244 {
1245 struct pcicfg_vpd *vpd;
1246 pcicfgregs *cfg;
1247 struct vpd_write *vpd_w;
1248 int len;
1249
1250 cfg = vrs->cfg;
1251 vpd = &cfg->vpd;
1252
1253 if (maxsize < 3)
1254 return (-1);
1255 vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt);
1256 if (vpd->vpd_w == NULL) {
1257 pci_printf(cfg, "out of memory");
1258 return (-1);
1259 }
1260 vpd_w = &vpd->vpd_w[vpd->vpd_wcnt];
1261 maxsize -= 3;
1262 vpd_w->start = vrs->off + 3 - vrs->bytesinval;
1263 len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize);
1264 if (vpd_w->value == NULL)
1265 return (-1);
1266 vpd_w->len = len;
1267 vpd->vpd_wcnt++;
1268
1269 return (len + 3);
1270 }
1271
1272 /* free all memory allocated for VPD data */
1273 static void
vpd_free(struct pcicfg_vpd * vpd)1274 vpd_free(struct pcicfg_vpd *vpd)
1275 {
1276 int i;
1277
1278 free(vpd->vpd_ident, M_DEVBUF);
1279 for (i = 0; i < vpd->vpd_rocnt; i++)
1280 free(vpd->vpd_ros[i].value, M_DEVBUF);
1281 free(vpd->vpd_ros, M_DEVBUF);
1282 vpd->vpd_rocnt = 0;
1283 for (i = 0; i < vpd->vpd_wcnt; i++)
1284 free(vpd->vpd_w[i].value, M_DEVBUF);
1285 free(vpd->vpd_w, M_DEVBUF);
1286 vpd->vpd_wcnt = 0;
1287 }
1288
1289 #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */
1290 #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */
1291 #define VPD_TAG_RO (0x10 | 0x80) /* large tag */
1292 #define VPD_TAG_RW (0x11 | 0x80) /* large tag */
1293
1294 static int
pci_parse_vpd(device_t pcib,pcicfgregs * cfg)1295 pci_parse_vpd(device_t pcib, pcicfgregs *cfg)
1296 {
1297 struct vpd_readstate vrs;
1298 int cksumvalid;
1299 int size, elem_size;
1300
1301 /* init vpd reader */
1302 vrs.bytesinval = 0;
1303 vrs.off = 0;
1304 vrs.pcib = pcib;
1305 vrs.cfg = cfg;
1306 vrs.cksum = 0;
1307
1308 /* read VPD ident element - mandatory */
1309 size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT);
1310 if (size <= 0) {
1311 pci_printf(cfg, "no VPD ident found\n");
1312 return (0);
1313 }
1314 cfg->vpd.vpd_ident = vpd_read_value(&vrs, size);
1315 if (cfg->vpd.vpd_ident == NULL) {
1316 pci_printf(cfg, "error accessing VPD ident data\n");
1317 return (0);
1318 }
1319
1320 /* read VPD RO elements - mandatory */
1321 size = vpd_read_tag_size(&vrs, VPD_TAG_RO);
1322 if (size <= 0) {
1323 pci_printf(cfg, "no read-only VPD data found\n");
1324 return (0);
1325 }
1326 while (size > 0) {
1327 elem_size = next_vpd_ro_elem(&vrs, size);
1328 if (elem_size < 0) {
1329 pci_printf(cfg, "error accessing read-only VPD data\n");
1330 return (-1);
1331 }
1332 size -= elem_size;
1333 }
1334 cksumvalid = (vrs.cksum == 0);
1335 if (!cksumvalid)
1336 return (-1);
1337
1338 /* read VPD RW elements - optional */
1339 size = vpd_read_tag_size(&vrs, VPD_TAG_RW);
1340 if (size == -2)
1341 return (-1);
1342 while (size > 0) {
1343 elem_size = next_vpd_rw_elem(&vrs, size);
1344 if (elem_size < 0) {
1345 pci_printf(cfg, "error accessing writeable VPD data\n");
1346 return (-1);
1347 }
1348 size -= elem_size;
1349 }
1350
1351 /* read empty END tag - mandatory */
1352 size = vpd_read_tag_size(&vrs, VPD_TAG_END);
1353 if (size != 0) {
1354 pci_printf(cfg, "No valid VPD end tag found\n");
1355 }
1356 return (0);
1357 }
1358
1359 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1360 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1361 {
1362 int status;
1363
1364 status = pci_parse_vpd(pcib, cfg);
1365 if (status < 0)
1366 vpd_free(&cfg->vpd);
1367 cfg->vpd.vpd_cached = 1;
1368 #undef REG
1369 #undef WREG
1370 }
1371
1372 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1373 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1374 {
1375 struct pci_devinfo *dinfo = device_get_ivars(child);
1376 pcicfgregs *cfg = &dinfo->cfg;
1377
1378 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1379 pci_read_vpd(device_get_parent(dev), cfg);
1380
1381 *identptr = cfg->vpd.vpd_ident;
1382
1383 if (*identptr == NULL)
1384 return (ENXIO);
1385
1386 return (0);
1387 }
1388
1389 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1390 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1391 const char **vptr)
1392 {
1393 struct pci_devinfo *dinfo = device_get_ivars(child);
1394 pcicfgregs *cfg = &dinfo->cfg;
1395 int i;
1396
1397 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1398 pci_read_vpd(device_get_parent(dev), cfg);
1399
1400 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1401 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1402 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1403 *vptr = cfg->vpd.vpd_ros[i].value;
1404 return (0);
1405 }
1406
1407 *vptr = NULL;
1408 return (ENXIO);
1409 }
1410
1411 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1412 pci_fetch_vpd_list(device_t dev)
1413 {
1414 struct pci_devinfo *dinfo = device_get_ivars(dev);
1415 pcicfgregs *cfg = &dinfo->cfg;
1416
1417 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1418 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1419 return (&cfg->vpd);
1420 }
1421
1422 /*
1423 * Find the requested HyperTransport capability and return the offset
1424 * in configuration space via the pointer provided. The function
1425 * returns 0 on success and an error code otherwise.
1426 */
1427 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1428 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1429 {
1430 int ptr, error;
1431 uint16_t val;
1432
1433 error = pci_find_cap(child, PCIY_HT, &ptr);
1434 if (error)
1435 return (error);
1436
1437 /*
1438 * Traverse the capabilities list checking each HT capability
1439 * to see if it matches the requested HT capability.
1440 */
1441 for (;;) {
1442 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1443 if (capability == PCIM_HTCAP_SLAVE ||
1444 capability == PCIM_HTCAP_HOST)
1445 val &= 0xe000;
1446 else
1447 val &= PCIM_HTCMD_CAP_MASK;
1448 if (val == capability) {
1449 if (capreg != NULL)
1450 *capreg = ptr;
1451 return (0);
1452 }
1453
1454 /* Skip to the next HT capability. */
1455 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1456 break;
1457 }
1458
1459 return (ENOENT);
1460 }
1461
1462 /*
1463 * Find the next requested HyperTransport capability after start and return
1464 * the offset in configuration space via the pointer provided. The function
1465 * returns 0 on success and an error code otherwise.
1466 */
1467 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1468 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1469 int start, int *capreg)
1470 {
1471 int ptr;
1472 uint16_t val;
1473
1474 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1475 ("start capability is not HyperTransport capability"));
1476 ptr = start;
1477
1478 /*
1479 * Traverse the capabilities list checking each HT capability
1480 * to see if it matches the requested HT capability.
1481 */
1482 for (;;) {
1483 /* Skip to the next HT capability. */
1484 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1485 break;
1486
1487 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1488 if (capability == PCIM_HTCAP_SLAVE ||
1489 capability == PCIM_HTCAP_HOST)
1490 val &= 0xe000;
1491 else
1492 val &= PCIM_HTCMD_CAP_MASK;
1493 if (val == capability) {
1494 if (capreg != NULL)
1495 *capreg = ptr;
1496 return (0);
1497 }
1498 }
1499
1500 return (ENOENT);
1501 }
1502
1503 /*
1504 * Find the requested capability and return the offset in
1505 * configuration space via the pointer provided. The function returns
1506 * 0 on success and an error code otherwise.
1507 */
1508 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1509 pci_find_cap_method(device_t dev, device_t child, int capability,
1510 int *capreg)
1511 {
1512 struct pci_devinfo *dinfo = device_get_ivars(child);
1513 pcicfgregs *cfg = &dinfo->cfg;
1514 uint32_t status;
1515 uint8_t ptr;
1516
1517 /*
1518 * Check the CAP_LIST bit of the PCI status register first.
1519 */
1520 status = pci_read_config(child, PCIR_STATUS, 2);
1521 if (!(status & PCIM_STATUS_CAPPRESENT))
1522 return (ENXIO);
1523
1524 /*
1525 * Determine the start pointer of the capabilities list.
1526 */
1527 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1528 case PCIM_HDRTYPE_NORMAL:
1529 case PCIM_HDRTYPE_BRIDGE:
1530 ptr = PCIR_CAP_PTR;
1531 break;
1532 case PCIM_HDRTYPE_CARDBUS:
1533 ptr = PCIR_CAP_PTR_2;
1534 break;
1535 default:
1536 /* XXX: panic? */
1537 return (ENXIO); /* no extended capabilities support */
1538 }
1539 ptr = pci_read_config(child, ptr, 1);
1540
1541 /*
1542 * Traverse the capabilities list.
1543 */
1544 while (ptr != 0) {
1545 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1546 if (capreg != NULL)
1547 *capreg = ptr;
1548 return (0);
1549 }
1550 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1551 }
1552
1553 return (ENOENT);
1554 }
1555
1556 /*
1557 * Find the next requested capability after start and return the offset in
1558 * configuration space via the pointer provided. The function returns
1559 * 0 on success and an error code otherwise.
1560 */
1561 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1562 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1563 int start, int *capreg)
1564 {
1565 uint8_t ptr;
1566
1567 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1568 ("start capability is not expected capability"));
1569
1570 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1571 while (ptr != 0) {
1572 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1573 if (capreg != NULL)
1574 *capreg = ptr;
1575 return (0);
1576 }
1577 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1578 }
1579
1580 return (ENOENT);
1581 }
1582
1583 /*
1584 * Find the requested extended capability and return the offset in
1585 * configuration space via the pointer provided. The function returns
1586 * 0 on success and an error code otherwise.
1587 */
1588 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1589 pci_find_extcap_method(device_t dev, device_t child, int capability,
1590 int *capreg)
1591 {
1592 struct pci_devinfo *dinfo = device_get_ivars(child);
1593 pcicfgregs *cfg = &dinfo->cfg;
1594 uint32_t ecap;
1595 uint16_t ptr;
1596
1597 /* Only supported for PCI-express devices. */
1598 if (cfg->pcie.pcie_location == 0)
1599 return (ENXIO);
1600
1601 ptr = PCIR_EXTCAP;
1602 ecap = pci_read_config(child, ptr, 4);
1603 if (ecap == 0xffffffff || ecap == 0)
1604 return (ENOENT);
1605 for (;;) {
1606 if (PCI_EXTCAP_ID(ecap) == capability) {
1607 if (capreg != NULL)
1608 *capreg = ptr;
1609 return (0);
1610 }
1611 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1612 if (ptr == 0)
1613 break;
1614 ecap = pci_read_config(child, ptr, 4);
1615 }
1616
1617 return (ENOENT);
1618 }
1619
1620 /*
1621 * Find the next requested extended capability after start and return the
1622 * offset in configuration space via the pointer provided. The function
1623 * returns 0 on success and an error code otherwise.
1624 */
1625 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1626 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1627 int start, int *capreg)
1628 {
1629 struct pci_devinfo *dinfo = device_get_ivars(child);
1630 pcicfgregs *cfg = &dinfo->cfg;
1631 uint32_t ecap;
1632 uint16_t ptr;
1633
1634 /* Only supported for PCI-express devices. */
1635 if (cfg->pcie.pcie_location == 0)
1636 return (ENXIO);
1637
1638 ecap = pci_read_config(child, start, 4);
1639 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1640 ("start extended capability is not expected capability"));
1641 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1642 while (ptr != 0) {
1643 ecap = pci_read_config(child, ptr, 4);
1644 if (PCI_EXTCAP_ID(ecap) == capability) {
1645 if (capreg != NULL)
1646 *capreg = ptr;
1647 return (0);
1648 }
1649 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1650 }
1651
1652 return (ENOENT);
1653 }
1654
1655 /*
1656 * Support for MSI-X message interrupts.
1657 */
1658 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1659 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1660 {
1661 struct pci_devinfo *dinfo = device_get_ivars(dev);
1662 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1663 uint32_t offset;
1664
1665 KASSERT(msix->msix_table_len > index, ("bogus index"));
1666 offset = msix->msix_table_offset + index * 16;
1667 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1668 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1669 bus_write_4(msix->msix_table_res, offset + 8, data);
1670 }
1671
1672 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1673 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1674 uint64_t address, uint32_t data)
1675 {
1676
1677 if (pci_msix_rewrite_table) {
1678 struct pci_devinfo *dinfo = device_get_ivars(child);
1679 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1680
1681 /*
1682 * Some VM hosts require MSIX to be disabled in the
1683 * control register before updating the MSIX table
1684 * entries are allowed. It is not enough to only
1685 * disable MSIX while updating a single entry. MSIX
1686 * must be disabled while updating all entries in the
1687 * table.
1688 */
1689 pci_write_config(child,
1690 msix->msix_location + PCIR_MSIX_CTRL,
1691 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1692 pci_resume_msix(child);
1693 } else
1694 pci_write_msix_entry(child, index, address, data);
1695
1696 /* Enable MSI -> HT mapping. */
1697 pci_ht_map_msi(child, address);
1698 }
1699
1700 void
pci_mask_msix(device_t dev,u_int index)1701 pci_mask_msix(device_t dev, u_int index)
1702 {
1703 struct pci_devinfo *dinfo = device_get_ivars(dev);
1704 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1705 uint32_t offset, val;
1706
1707 KASSERT(msix->msix_msgnum > index, ("bogus index"));
1708 offset = msix->msix_table_offset + index * 16 + 12;
1709 val = bus_read_4(msix->msix_table_res, offset);
1710 val |= PCIM_MSIX_VCTRL_MASK;
1711
1712 /*
1713 * Some devices (e.g. Samsung PM961) do not support reads of this
1714 * register, so always write the new value.
1715 */
1716 bus_write_4(msix->msix_table_res, offset, val);
1717 }
1718
1719 void
pci_unmask_msix(device_t dev,u_int index)1720 pci_unmask_msix(device_t dev, u_int index)
1721 {
1722 struct pci_devinfo *dinfo = device_get_ivars(dev);
1723 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1724 uint32_t offset, val;
1725
1726 KASSERT(msix->msix_table_len > index, ("bogus index"));
1727 offset = msix->msix_table_offset + index * 16 + 12;
1728 val = bus_read_4(msix->msix_table_res, offset);
1729 val &= ~PCIM_MSIX_VCTRL_MASK;
1730
1731 /*
1732 * Some devices (e.g. Samsung PM961) do not support reads of this
1733 * register, so always write the new value.
1734 */
1735 bus_write_4(msix->msix_table_res, offset, val);
1736 }
1737
1738 int
pci_pending_msix(device_t dev,u_int index)1739 pci_pending_msix(device_t dev, u_int index)
1740 {
1741 struct pci_devinfo *dinfo = device_get_ivars(dev);
1742 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1743 uint32_t offset, bit;
1744
1745 KASSERT(msix->msix_table_len > index, ("bogus index"));
1746 offset = msix->msix_pba_offset + (index / 32) * 4;
1747 bit = 1 << index % 32;
1748 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1749 }
1750
1751 /*
1752 * Restore MSI-X registers and table during resume. If MSI-X is
1753 * enabled then walk the virtual table to restore the actual MSI-X
1754 * table.
1755 */
1756 static void
pci_resume_msix(device_t dev)1757 pci_resume_msix(device_t dev)
1758 {
1759 struct pci_devinfo *dinfo = device_get_ivars(dev);
1760 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1761 struct msix_table_entry *mte;
1762 struct msix_vector *mv;
1763 int i;
1764
1765 if (msix->msix_alloc > 0) {
1766 /* First, mask all vectors. */
1767 for (i = 0; i < msix->msix_msgnum; i++)
1768 pci_mask_msix(dev, i);
1769
1770 /* Second, program any messages with at least one handler. */
1771 for (i = 0; i < msix->msix_table_len; i++) {
1772 mte = &msix->msix_table[i];
1773 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1774 continue;
1775 mv = &msix->msix_vectors[mte->mte_vector - 1];
1776 pci_write_msix_entry(dev, i, mv->mv_address,
1777 mv->mv_data);
1778 pci_unmask_msix(dev, i);
1779 }
1780 }
1781 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1782 msix->msix_ctrl, 2);
1783 }
1784
1785 /*
1786 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1787 * returned in *count. After this function returns, each message will be
1788 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1789 */
1790 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1791 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1792 {
1793 struct pci_devinfo *dinfo = device_get_ivars(child);
1794 pcicfgregs *cfg = &dinfo->cfg;
1795 struct resource_list_entry *rle;
1796 int actual, error, i, irq, max;
1797
1798 /* Don't let count == 0 get us into trouble. */
1799 if (*count == 0)
1800 return (EINVAL);
1801
1802 /* If rid 0 is allocated, then fail. */
1803 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1804 if (rle != NULL && rle->res != NULL)
1805 return (ENXIO);
1806
1807 /* Already have allocated messages? */
1808 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1809 return (ENXIO);
1810
1811 /* If MSI-X is blacklisted for this system, fail. */
1812 if (pci_msix_blacklisted())
1813 return (ENXIO);
1814
1815 /* MSI-X capability present? */
1816 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1817 return (ENODEV);
1818
1819 /* Make sure the appropriate BARs are mapped. */
1820 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1821 cfg->msix.msix_table_bar);
1822 if (rle == NULL || rle->res == NULL ||
1823 !(rman_get_flags(rle->res) & RF_ACTIVE))
1824 return (ENXIO);
1825 cfg->msix.msix_table_res = rle->res;
1826 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1827 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1828 cfg->msix.msix_pba_bar);
1829 if (rle == NULL || rle->res == NULL ||
1830 !(rman_get_flags(rle->res) & RF_ACTIVE))
1831 return (ENXIO);
1832 }
1833 cfg->msix.msix_pba_res = rle->res;
1834
1835 if (bootverbose)
1836 device_printf(child,
1837 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1838 *count, cfg->msix.msix_msgnum);
1839 max = min(*count, cfg->msix.msix_msgnum);
1840 for (i = 0; i < max; i++) {
1841 /* Allocate a message. */
1842 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1843 if (error) {
1844 if (i == 0)
1845 return (error);
1846 break;
1847 }
1848 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1849 irq, 1);
1850 }
1851 actual = i;
1852
1853 if (bootverbose) {
1854 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1855 if (actual == 1)
1856 device_printf(child, "using IRQ %ju for MSI-X\n",
1857 rle->start);
1858 else {
1859 int run;
1860
1861 /*
1862 * Be fancy and try to print contiguous runs of
1863 * IRQ values as ranges. 'irq' is the previous IRQ.
1864 * 'run' is true if we are in a range.
1865 */
1866 device_printf(child, "using IRQs %ju", rle->start);
1867 irq = rle->start;
1868 run = 0;
1869 for (i = 1; i < actual; i++) {
1870 rle = resource_list_find(&dinfo->resources,
1871 SYS_RES_IRQ, i + 1);
1872
1873 /* Still in a run? */
1874 if (rle->start == irq + 1) {
1875 run = 1;
1876 irq++;
1877 continue;
1878 }
1879
1880 /* Finish previous range. */
1881 if (run) {
1882 printf("-%d", irq);
1883 run = 0;
1884 }
1885
1886 /* Start new range. */
1887 printf(",%ju", rle->start);
1888 irq = rle->start;
1889 }
1890
1891 /* Unfinished range? */
1892 if (run)
1893 printf("-%d", irq);
1894 printf(" for MSI-X\n");
1895 }
1896 }
1897
1898 /* Mask all vectors. */
1899 for (i = 0; i < cfg->msix.msix_msgnum; i++)
1900 pci_mask_msix(child, i);
1901
1902 /* Allocate and initialize vector data and virtual table. */
1903 cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
1904 M_DEVBUF, M_WAITOK | M_ZERO);
1905 cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
1906 M_DEVBUF, M_WAITOK | M_ZERO);
1907 for (i = 0; i < actual; i++) {
1908 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1909 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1910 cfg->msix.msix_table[i].mte_vector = i + 1;
1911 }
1912
1913 /* Update control register to enable MSI-X. */
1914 cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1915 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1916 cfg->msix.msix_ctrl, 2);
1917
1918 /* Update counts of alloc'd messages. */
1919 cfg->msix.msix_alloc = actual;
1920 cfg->msix.msix_table_len = actual;
1921 *count = actual;
1922 return (0);
1923 }
1924
1925 /*
1926 * By default, pci_alloc_msix() will assign the allocated IRQ
1927 * resources consecutively to the first N messages in the MSI-X table.
1928 * However, device drivers may want to use different layouts if they
1929 * either receive fewer messages than they asked for, or they wish to
1930 * populate the MSI-X table sparsely. This method allows the driver
1931 * to specify what layout it wants. It must be called after a
1932 * successful pci_alloc_msix() but before any of the associated
1933 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1934 *
1935 * The 'vectors' array contains 'count' message vectors. The array
1936 * maps directly to the MSI-X table in that index 0 in the array
1937 * specifies the vector for the first message in the MSI-X table, etc.
1938 * The vector value in each array index can either be 0 to indicate
1939 * that no vector should be assigned to a message slot, or it can be a
1940 * number from 1 to N (where N is the count returned from a
1941 * succcessful call to pci_alloc_msix()) to indicate which message
1942 * vector (IRQ) to be used for the corresponding message.
1943 *
1944 * On successful return, each message with a non-zero vector will have
1945 * an associated SYS_RES_IRQ whose rid is equal to the array index +
1946 * 1. Additionally, if any of the IRQs allocated via the previous
1947 * call to pci_alloc_msix() are not used in the mapping, those IRQs
1948 * will be freed back to the system automatically.
1949 *
1950 * For example, suppose a driver has a MSI-X table with 6 messages and
1951 * asks for 6 messages, but pci_alloc_msix() only returns a count of
1952 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
1953 * C. After the call to pci_alloc_msix(), the device will be setup to
1954 * have an MSI-X table of ABC--- (where - means no vector assigned).
1955 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
1956 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
1957 * be freed back to the system. This device will also have valid
1958 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
1959 *
1960 * In any case, the SYS_RES_IRQ rid X will always map to the message
1961 * at MSI-X table index X - 1 and will only be valid if a vector is
1962 * assigned to that table entry.
1963 */
1964 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)1965 pci_remap_msix_method(device_t dev, device_t child, int count,
1966 const u_int *vectors)
1967 {
1968 struct pci_devinfo *dinfo = device_get_ivars(child);
1969 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1970 struct resource_list_entry *rle;
1971 int i, irq, j, *used;
1972
1973 /*
1974 * Have to have at least one message in the table but the
1975 * table can't be bigger than the actual MSI-X table in the
1976 * device.
1977 */
1978 if (count == 0 || count > msix->msix_msgnum)
1979 return (EINVAL);
1980
1981 /* Sanity check the vectors. */
1982 for (i = 0; i < count; i++)
1983 if (vectors[i] > msix->msix_alloc)
1984 return (EINVAL);
1985
1986 /*
1987 * Make sure there aren't any holes in the vectors to be used.
1988 * It's a big pain to support it, and it doesn't really make
1989 * sense anyway. Also, at least one vector must be used.
1990 */
1991 used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK |
1992 M_ZERO);
1993 for (i = 0; i < count; i++)
1994 if (vectors[i] != 0)
1995 used[vectors[i] - 1] = 1;
1996 for (i = 0; i < msix->msix_alloc - 1; i++)
1997 if (used[i] == 0 && used[i + 1] == 1) {
1998 free(used, M_DEVBUF);
1999 return (EINVAL);
2000 }
2001 if (used[0] != 1) {
2002 free(used, M_DEVBUF);
2003 return (EINVAL);
2004 }
2005
2006 /* Make sure none of the resources are allocated. */
2007 for (i = 0; i < msix->msix_table_len; i++) {
2008 if (msix->msix_table[i].mte_vector == 0)
2009 continue;
2010 if (msix->msix_table[i].mte_handlers > 0) {
2011 free(used, M_DEVBUF);
2012 return (EBUSY);
2013 }
2014 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2015 KASSERT(rle != NULL, ("missing resource"));
2016 if (rle->res != NULL) {
2017 free(used, M_DEVBUF);
2018 return (EBUSY);
2019 }
2020 }
2021
2022 /* Free the existing resource list entries. */
2023 for (i = 0; i < msix->msix_table_len; i++) {
2024 if (msix->msix_table[i].mte_vector == 0)
2025 continue;
2026 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2027 }
2028
2029 /*
2030 * Build the new virtual table keeping track of which vectors are
2031 * used.
2032 */
2033 free(msix->msix_table, M_DEVBUF);
2034 msix->msix_table = malloc(sizeof(struct msix_table_entry) * count,
2035 M_DEVBUF, M_WAITOK | M_ZERO);
2036 for (i = 0; i < count; i++)
2037 msix->msix_table[i].mte_vector = vectors[i];
2038 msix->msix_table_len = count;
2039
2040 /* Free any unused IRQs and resize the vectors array if necessary. */
2041 j = msix->msix_alloc - 1;
2042 if (used[j] == 0) {
2043 struct msix_vector *vec;
2044
2045 while (used[j] == 0) {
2046 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2047 msix->msix_vectors[j].mv_irq);
2048 j--;
2049 }
2050 vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF,
2051 M_WAITOK);
2052 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2053 (j + 1));
2054 free(msix->msix_vectors, M_DEVBUF);
2055 msix->msix_vectors = vec;
2056 msix->msix_alloc = j + 1;
2057 }
2058 free(used, M_DEVBUF);
2059
2060 /* Map the IRQs onto the rids. */
2061 for (i = 0; i < count; i++) {
2062 if (vectors[i] == 0)
2063 continue;
2064 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2065 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2066 irq, 1);
2067 }
2068
2069 if (bootverbose) {
2070 device_printf(child, "Remapped MSI-X IRQs as: ");
2071 for (i = 0; i < count; i++) {
2072 if (i != 0)
2073 printf(", ");
2074 if (vectors[i] == 0)
2075 printf("---");
2076 else
2077 printf("%d",
2078 msix->msix_vectors[vectors[i] - 1].mv_irq);
2079 }
2080 printf("\n");
2081 }
2082
2083 return (0);
2084 }
2085
2086 static int
pci_release_msix(device_t dev,device_t child)2087 pci_release_msix(device_t dev, device_t child)
2088 {
2089 struct pci_devinfo *dinfo = device_get_ivars(child);
2090 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2091 struct resource_list_entry *rle;
2092 int i;
2093
2094 /* Do we have any messages to release? */
2095 if (msix->msix_alloc == 0)
2096 return (ENODEV);
2097
2098 /* Make sure none of the resources are allocated. */
2099 for (i = 0; i < msix->msix_table_len; i++) {
2100 if (msix->msix_table[i].mte_vector == 0)
2101 continue;
2102 if (msix->msix_table[i].mte_handlers > 0)
2103 return (EBUSY);
2104 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2105 KASSERT(rle != NULL, ("missing resource"));
2106 if (rle->res != NULL)
2107 return (EBUSY);
2108 }
2109
2110 /* Update control register to disable MSI-X. */
2111 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2112 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2113 msix->msix_ctrl, 2);
2114
2115 /* Free the resource list entries. */
2116 for (i = 0; i < msix->msix_table_len; i++) {
2117 if (msix->msix_table[i].mte_vector == 0)
2118 continue;
2119 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2120 }
2121 free(msix->msix_table, M_DEVBUF);
2122 msix->msix_table_len = 0;
2123
2124 /* Release the IRQs. */
2125 for (i = 0; i < msix->msix_alloc; i++)
2126 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2127 msix->msix_vectors[i].mv_irq);
2128 free(msix->msix_vectors, M_DEVBUF);
2129 msix->msix_alloc = 0;
2130 return (0);
2131 }
2132
2133 /*
2134 * Return the max supported MSI-X messages this device supports.
2135 * Basically, assuming the MD code can alloc messages, this function
2136 * should return the maximum value that pci_alloc_msix() can return.
2137 * Thus, it is subject to the tunables, etc.
2138 */
2139 int
pci_msix_count_method(device_t dev,device_t child)2140 pci_msix_count_method(device_t dev, device_t child)
2141 {
2142 struct pci_devinfo *dinfo = device_get_ivars(child);
2143 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2144
2145 if (pci_do_msix && msix->msix_location != 0)
2146 return (msix->msix_msgnum);
2147 return (0);
2148 }
2149
2150 int
pci_msix_pba_bar_method(device_t dev,device_t child)2151 pci_msix_pba_bar_method(device_t dev, device_t child)
2152 {
2153 struct pci_devinfo *dinfo = device_get_ivars(child);
2154 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2155
2156 if (pci_do_msix && msix->msix_location != 0)
2157 return (msix->msix_pba_bar);
2158 return (-1);
2159 }
2160
2161 int
pci_msix_table_bar_method(device_t dev,device_t child)2162 pci_msix_table_bar_method(device_t dev, device_t child)
2163 {
2164 struct pci_devinfo *dinfo = device_get_ivars(child);
2165 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2166
2167 if (pci_do_msix && msix->msix_location != 0)
2168 return (msix->msix_table_bar);
2169 return (-1);
2170 }
2171
2172 /*
2173 * HyperTransport MSI mapping control
2174 */
2175 void
pci_ht_map_msi(device_t dev,uint64_t addr)2176 pci_ht_map_msi(device_t dev, uint64_t addr)
2177 {
2178 struct pci_devinfo *dinfo = device_get_ivars(dev);
2179 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2180
2181 if (!ht->ht_msimap)
2182 return;
2183
2184 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2185 ht->ht_msiaddr >> 20 == addr >> 20) {
2186 /* Enable MSI -> HT mapping. */
2187 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2188 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2189 ht->ht_msictrl, 2);
2190 }
2191
2192 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2193 /* Disable MSI -> HT mapping. */
2194 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2195 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2196 ht->ht_msictrl, 2);
2197 }
2198 }
2199
2200 int
pci_get_relaxed_ordering_enabled(device_t dev)2201 pci_get_relaxed_ordering_enabled(device_t dev)
2202 {
2203 struct pci_devinfo *dinfo = device_get_ivars(dev);
2204 int cap;
2205 uint16_t val;
2206
2207 cap = dinfo->cfg.pcie.pcie_location;
2208 if (cap == 0)
2209 return (0);
2210 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2211 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2212 return (val != 0);
2213 }
2214
2215 int
pci_get_max_payload(device_t dev)2216 pci_get_max_payload(device_t dev)
2217 {
2218 struct pci_devinfo *dinfo = device_get_ivars(dev);
2219 int cap;
2220 uint16_t val;
2221
2222 cap = dinfo->cfg.pcie.pcie_location;
2223 if (cap == 0)
2224 return (0);
2225 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2226 val &= PCIEM_CTL_MAX_PAYLOAD;
2227 val >>= 5;
2228 return (1 << (val + 7));
2229 }
2230
2231 int
pci_get_max_read_req(device_t dev)2232 pci_get_max_read_req(device_t dev)
2233 {
2234 struct pci_devinfo *dinfo = device_get_ivars(dev);
2235 int cap;
2236 uint16_t val;
2237
2238 cap = dinfo->cfg.pcie.pcie_location;
2239 if (cap == 0)
2240 return (0);
2241 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2242 val &= PCIEM_CTL_MAX_READ_REQUEST;
2243 val >>= 12;
2244 return (1 << (val + 7));
2245 }
2246
2247 int
pci_set_max_read_req(device_t dev,int size)2248 pci_set_max_read_req(device_t dev, int size)
2249 {
2250 struct pci_devinfo *dinfo = device_get_ivars(dev);
2251 int cap;
2252 uint16_t val;
2253
2254 cap = dinfo->cfg.pcie.pcie_location;
2255 if (cap == 0)
2256 return (0);
2257 if (size < 128)
2258 size = 128;
2259 if (size > 4096)
2260 size = 4096;
2261 size = (1 << (fls(size) - 1));
2262 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2263 val &= ~PCIEM_CTL_MAX_READ_REQUEST;
2264 val |= (fls(size) - 8) << 12;
2265 pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2);
2266 return (size);
2267 }
2268
2269 uint32_t
pcie_read_config(device_t dev,int reg,int width)2270 pcie_read_config(device_t dev, int reg, int width)
2271 {
2272 struct pci_devinfo *dinfo = device_get_ivars(dev);
2273 int cap;
2274
2275 cap = dinfo->cfg.pcie.pcie_location;
2276 if (cap == 0) {
2277 if (width == 2)
2278 return (0xffff);
2279 return (0xffffffff);
2280 }
2281
2282 return (pci_read_config(dev, cap + reg, width));
2283 }
2284
2285 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2286 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2287 {
2288 struct pci_devinfo *dinfo = device_get_ivars(dev);
2289 int cap;
2290
2291 cap = dinfo->cfg.pcie.pcie_location;
2292 if (cap == 0)
2293 return;
2294 pci_write_config(dev, cap + reg, value, width);
2295 }
2296
2297 /*
2298 * Adjusts a PCI-e capability register by clearing the bits in mask
2299 * and setting the bits in (value & mask). Bits not set in mask are
2300 * not adjusted.
2301 *
2302 * Returns the old value on success or all ones on failure.
2303 */
2304 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2305 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2306 int width)
2307 {
2308 struct pci_devinfo *dinfo = device_get_ivars(dev);
2309 uint32_t old, new;
2310 int cap;
2311
2312 cap = dinfo->cfg.pcie.pcie_location;
2313 if (cap == 0) {
2314 if (width == 2)
2315 return (0xffff);
2316 return (0xffffffff);
2317 }
2318
2319 old = pci_read_config(dev, cap + reg, width);
2320 new = old & ~mask;
2321 new |= (value & mask);
2322 pci_write_config(dev, cap + reg, new, width);
2323 return (old);
2324 }
2325
2326 /*
2327 * Support for MSI message signalled interrupts.
2328 */
2329 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2330 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2331 uint16_t data)
2332 {
2333 struct pci_devinfo *dinfo = device_get_ivars(child);
2334 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2335
2336 /* Write data and address values. */
2337 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2338 address & 0xffffffff, 4);
2339 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2340 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2341 address >> 32, 4);
2342 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2343 data, 2);
2344 } else
2345 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2346 2);
2347
2348 /* Enable MSI in the control register. */
2349 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2350 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2351 msi->msi_ctrl, 2);
2352
2353 /* Enable MSI -> HT mapping. */
2354 pci_ht_map_msi(child, address);
2355 }
2356
2357 void
pci_disable_msi_method(device_t dev,device_t child)2358 pci_disable_msi_method(device_t dev, device_t child)
2359 {
2360 struct pci_devinfo *dinfo = device_get_ivars(child);
2361 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2362
2363 /* Disable MSI -> HT mapping. */
2364 pci_ht_map_msi(child, 0);
2365
2366 /* Disable MSI in the control register. */
2367 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2368 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2369 msi->msi_ctrl, 2);
2370 }
2371
2372 /*
2373 * Restore MSI registers during resume. If MSI is enabled then
2374 * restore the data and address registers in addition to the control
2375 * register.
2376 */
2377 static void
pci_resume_msi(device_t dev)2378 pci_resume_msi(device_t dev)
2379 {
2380 struct pci_devinfo *dinfo = device_get_ivars(dev);
2381 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2382 uint64_t address;
2383 uint16_t data;
2384
2385 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2386 address = msi->msi_addr;
2387 data = msi->msi_data;
2388 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2389 address & 0xffffffff, 4);
2390 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2391 pci_write_config(dev, msi->msi_location +
2392 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2393 pci_write_config(dev, msi->msi_location +
2394 PCIR_MSI_DATA_64BIT, data, 2);
2395 } else
2396 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2397 data, 2);
2398 }
2399 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2400 2);
2401 }
2402
2403 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2404 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2405 {
2406 struct pci_devinfo *dinfo = device_get_ivars(dev);
2407 pcicfgregs *cfg = &dinfo->cfg;
2408 struct resource_list_entry *rle;
2409 struct msix_table_entry *mte;
2410 struct msix_vector *mv;
2411 uint64_t addr;
2412 uint32_t data;
2413 int error, i, j;
2414
2415 /*
2416 * Handle MSI first. We try to find this IRQ among our list
2417 * of MSI IRQs. If we find it, we request updated address and
2418 * data registers and apply the results.
2419 */
2420 if (cfg->msi.msi_alloc > 0) {
2421 /* If we don't have any active handlers, nothing to do. */
2422 if (cfg->msi.msi_handlers == 0)
2423 return (0);
2424 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2425 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2426 i + 1);
2427 if (rle->start == irq) {
2428 error = PCIB_MAP_MSI(device_get_parent(bus),
2429 dev, irq, &addr, &data);
2430 if (error)
2431 return (error);
2432 pci_disable_msi(dev);
2433 dinfo->cfg.msi.msi_addr = addr;
2434 dinfo->cfg.msi.msi_data = data;
2435 pci_enable_msi(dev, addr, data);
2436 return (0);
2437 }
2438 }
2439 return (ENOENT);
2440 }
2441
2442 /*
2443 * For MSI-X, we check to see if we have this IRQ. If we do,
2444 * we request the updated mapping info. If that works, we go
2445 * through all the slots that use this IRQ and update them.
2446 */
2447 if (cfg->msix.msix_alloc > 0) {
2448 bool found = false;
2449
2450 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2451 mv = &cfg->msix.msix_vectors[i];
2452 if (mv->mv_irq == irq) {
2453 error = PCIB_MAP_MSI(device_get_parent(bus),
2454 dev, irq, &addr, &data);
2455 if (error)
2456 return (error);
2457 mv->mv_address = addr;
2458 mv->mv_data = data;
2459 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2460 mte = &cfg->msix.msix_table[j];
2461 if (mte->mte_vector != i + 1)
2462 continue;
2463 if (mte->mte_handlers == 0)
2464 continue;
2465 pci_mask_msix(dev, j);
2466 pci_enable_msix(dev, j, addr, data);
2467 pci_unmask_msix(dev, j);
2468 }
2469 found = true;
2470 }
2471 }
2472 return (found ? 0 : ENOENT);
2473 }
2474
2475 return (ENOENT);
2476 }
2477
2478 /*
2479 * Returns true if the specified device is blacklisted because MSI
2480 * doesn't work.
2481 */
2482 int
pci_msi_device_blacklisted(device_t dev)2483 pci_msi_device_blacklisted(device_t dev)
2484 {
2485
2486 if (!pci_honor_msi_blacklist)
2487 return (0);
2488
2489 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2490 }
2491
2492 /*
2493 * Determine if MSI is blacklisted globally on this system. Currently,
2494 * we just check for blacklisted chipsets as represented by the
2495 * host-PCI bridge at device 0:0:0. In the future, it may become
2496 * necessary to check other system attributes, such as the kenv values
2497 * that give the motherboard manufacturer and model number.
2498 */
2499 static int
pci_msi_blacklisted(void)2500 pci_msi_blacklisted(void)
2501 {
2502 device_t dev;
2503
2504 if (!pci_honor_msi_blacklist)
2505 return (0);
2506
2507 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2508 if (!(pcie_chipset || pcix_chipset)) {
2509 if (vm_guest != VM_GUEST_NO) {
2510 /*
2511 * Whitelist older chipsets in virtual
2512 * machines known to support MSI.
2513 */
2514 dev = pci_find_bsf(0, 0, 0);
2515 if (dev != NULL)
2516 return (!pci_has_quirk(pci_get_devid(dev),
2517 PCI_QUIRK_ENABLE_MSI_VM));
2518 }
2519 return (1);
2520 }
2521
2522 dev = pci_find_bsf(0, 0, 0);
2523 if (dev != NULL)
2524 return (pci_msi_device_blacklisted(dev));
2525 return (0);
2526 }
2527
2528 /*
2529 * Returns true if the specified device is blacklisted because MSI-X
2530 * doesn't work. Note that this assumes that if MSI doesn't work,
2531 * MSI-X doesn't either.
2532 */
2533 int
pci_msix_device_blacklisted(device_t dev)2534 pci_msix_device_blacklisted(device_t dev)
2535 {
2536
2537 if (!pci_honor_msi_blacklist)
2538 return (0);
2539
2540 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2541 return (1);
2542
2543 return (pci_msi_device_blacklisted(dev));
2544 }
2545
2546 /*
2547 * Determine if MSI-X is blacklisted globally on this system. If MSI
2548 * is blacklisted, assume that MSI-X is as well. Check for additional
2549 * chipsets where MSI works but MSI-X does not.
2550 */
2551 static int
pci_msix_blacklisted(void)2552 pci_msix_blacklisted(void)
2553 {
2554 device_t dev;
2555
2556 if (!pci_honor_msi_blacklist)
2557 return (0);
2558
2559 dev = pci_find_bsf(0, 0, 0);
2560 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2561 PCI_QUIRK_DISABLE_MSIX))
2562 return (1);
2563
2564 return (pci_msi_blacklisted());
2565 }
2566
2567 /*
2568 * Attempt to allocate *count MSI messages. The actual number allocated is
2569 * returned in *count. After this function returns, each message will be
2570 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2571 */
2572 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2573 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2574 {
2575 struct pci_devinfo *dinfo = device_get_ivars(child);
2576 pcicfgregs *cfg = &dinfo->cfg;
2577 struct resource_list_entry *rle;
2578 int actual, error, i, irqs[32];
2579 uint16_t ctrl;
2580
2581 /* Don't let count == 0 get us into trouble. */
2582 if (*count == 0)
2583 return (EINVAL);
2584
2585 /* If rid 0 is allocated, then fail. */
2586 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2587 if (rle != NULL && rle->res != NULL)
2588 return (ENXIO);
2589
2590 /* Already have allocated messages? */
2591 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2592 return (ENXIO);
2593
2594 /* If MSI is blacklisted for this system, fail. */
2595 if (pci_msi_blacklisted())
2596 return (ENXIO);
2597
2598 /* MSI capability present? */
2599 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2600 return (ENODEV);
2601
2602 if (bootverbose)
2603 device_printf(child,
2604 "attempting to allocate %d MSI vectors (%d supported)\n",
2605 *count, cfg->msi.msi_msgnum);
2606
2607 /* Don't ask for more than the device supports. */
2608 actual = min(*count, cfg->msi.msi_msgnum);
2609
2610 /* Don't ask for more than 32 messages. */
2611 actual = min(actual, 32);
2612
2613 /* MSI requires power of 2 number of messages. */
2614 if (!powerof2(actual))
2615 return (EINVAL);
2616
2617 for (;;) {
2618 /* Try to allocate N messages. */
2619 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2620 actual, irqs);
2621 if (error == 0)
2622 break;
2623 if (actual == 1)
2624 return (error);
2625
2626 /* Try N / 2. */
2627 actual >>= 1;
2628 }
2629
2630 /*
2631 * We now have N actual messages mapped onto SYS_RES_IRQ
2632 * resources in the irqs[] array, so add new resources
2633 * starting at rid 1.
2634 */
2635 for (i = 0; i < actual; i++)
2636 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2637 irqs[i], irqs[i], 1);
2638
2639 if (bootverbose) {
2640 if (actual == 1)
2641 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2642 else {
2643 int run;
2644
2645 /*
2646 * Be fancy and try to print contiguous runs
2647 * of IRQ values as ranges. 'run' is true if
2648 * we are in a range.
2649 */
2650 device_printf(child, "using IRQs %d", irqs[0]);
2651 run = 0;
2652 for (i = 1; i < actual; i++) {
2653 /* Still in a run? */
2654 if (irqs[i] == irqs[i - 1] + 1) {
2655 run = 1;
2656 continue;
2657 }
2658
2659 /* Finish previous range. */
2660 if (run) {
2661 printf("-%d", irqs[i - 1]);
2662 run = 0;
2663 }
2664
2665 /* Start new range. */
2666 printf(",%d", irqs[i]);
2667 }
2668
2669 /* Unfinished range? */
2670 if (run)
2671 printf("-%d", irqs[actual - 1]);
2672 printf(" for MSI\n");
2673 }
2674 }
2675
2676 /* Update control register with actual count. */
2677 ctrl = cfg->msi.msi_ctrl;
2678 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2679 ctrl |= (ffs(actual) - 1) << 4;
2680 cfg->msi.msi_ctrl = ctrl;
2681 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2682
2683 /* Update counts of alloc'd messages. */
2684 cfg->msi.msi_alloc = actual;
2685 cfg->msi.msi_handlers = 0;
2686 *count = actual;
2687 return (0);
2688 }
2689
2690 /* Release the MSI messages associated with this device. */
2691 int
pci_release_msi_method(device_t dev,device_t child)2692 pci_release_msi_method(device_t dev, device_t child)
2693 {
2694 struct pci_devinfo *dinfo = device_get_ivars(child);
2695 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2696 struct resource_list_entry *rle;
2697 int error, i, irqs[32];
2698
2699 /* Try MSI-X first. */
2700 error = pci_release_msix(dev, child);
2701 if (error != ENODEV)
2702 return (error);
2703
2704 /* Do we have any messages to release? */
2705 if (msi->msi_alloc == 0)
2706 return (ENODEV);
2707 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2708
2709 /* Make sure none of the resources are allocated. */
2710 if (msi->msi_handlers > 0)
2711 return (EBUSY);
2712 for (i = 0; i < msi->msi_alloc; i++) {
2713 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2714 KASSERT(rle != NULL, ("missing MSI resource"));
2715 if (rle->res != NULL)
2716 return (EBUSY);
2717 irqs[i] = rle->start;
2718 }
2719
2720 /* Update control register with 0 count. */
2721 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2722 ("%s: MSI still enabled", __func__));
2723 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2724 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2725 msi->msi_ctrl, 2);
2726
2727 /* Release the messages. */
2728 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2729 for (i = 0; i < msi->msi_alloc; i++)
2730 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2731
2732 /* Update alloc count. */
2733 msi->msi_alloc = 0;
2734 msi->msi_addr = 0;
2735 msi->msi_data = 0;
2736 return (0);
2737 }
2738
2739 /*
2740 * Return the max supported MSI messages this device supports.
2741 * Basically, assuming the MD code can alloc messages, this function
2742 * should return the maximum value that pci_alloc_msi() can return.
2743 * Thus, it is subject to the tunables, etc.
2744 */
2745 int
pci_msi_count_method(device_t dev,device_t child)2746 pci_msi_count_method(device_t dev, device_t child)
2747 {
2748 struct pci_devinfo *dinfo = device_get_ivars(child);
2749 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2750
2751 if (pci_do_msi && msi->msi_location != 0)
2752 return (msi->msi_msgnum);
2753 return (0);
2754 }
2755
2756 /* free pcicfgregs structure and all depending data structures */
2757
2758 int
pci_freecfg(struct pci_devinfo * dinfo)2759 pci_freecfg(struct pci_devinfo *dinfo)
2760 {
2761 struct devlist *devlist_head;
2762 struct pci_map *pm, *next;
2763
2764 devlist_head = &pci_devq;
2765
2766 if (dinfo->cfg.vpd.vpd_reg)
2767 vpd_free(&dinfo->cfg.vpd);
2768
2769 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2770 free(pm, M_DEVBUF);
2771 }
2772 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2773 free(dinfo, M_DEVBUF);
2774
2775 /* increment the generation count */
2776 pci_generation++;
2777
2778 /* we're losing one device */
2779 pci_numdevs--;
2780 return (0);
2781 }
2782
2783 /*
2784 * PCI power manangement
2785 */
2786 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2787 pci_set_powerstate_method(device_t dev, device_t child, int state)
2788 {
2789 struct pci_devinfo *dinfo = device_get_ivars(child);
2790 pcicfgregs *cfg = &dinfo->cfg;
2791 uint16_t status;
2792 int oldstate, highest, delay;
2793
2794 if (cfg->pp.pp_cap == 0)
2795 return (EOPNOTSUPP);
2796
2797 /*
2798 * Optimize a no state change request away. While it would be OK to
2799 * write to the hardware in theory, some devices have shown odd
2800 * behavior when going from D3 -> D3.
2801 */
2802 oldstate = pci_get_powerstate(child);
2803 if (oldstate == state)
2804 return (0);
2805
2806 /*
2807 * The PCI power management specification states that after a state
2808 * transition between PCI power states, system software must
2809 * guarantee a minimal delay before the function accesses the device.
2810 * Compute the worst case delay that we need to guarantee before we
2811 * access the device. Many devices will be responsive much more
2812 * quickly than this delay, but there are some that don't respond
2813 * instantly to state changes. Transitions to/from D3 state require
2814 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2815 * is done below with DELAY rather than a sleeper function because
2816 * this function can be called from contexts where we cannot sleep.
2817 */
2818 highest = (oldstate > state) ? oldstate : state;
2819 if (highest == PCI_POWERSTATE_D3)
2820 delay = 10000;
2821 else if (highest == PCI_POWERSTATE_D2)
2822 delay = 200;
2823 else
2824 delay = 0;
2825 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2)
2826 & ~PCIM_PSTAT_DMASK;
2827 switch (state) {
2828 case PCI_POWERSTATE_D0:
2829 status |= PCIM_PSTAT_D0;
2830 break;
2831 case PCI_POWERSTATE_D1:
2832 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2833 return (EOPNOTSUPP);
2834 status |= PCIM_PSTAT_D1;
2835 break;
2836 case PCI_POWERSTATE_D2:
2837 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2838 return (EOPNOTSUPP);
2839 status |= PCIM_PSTAT_D2;
2840 break;
2841 case PCI_POWERSTATE_D3:
2842 status |= PCIM_PSTAT_D3;
2843 break;
2844 default:
2845 return (EINVAL);
2846 }
2847
2848 if (bootverbose)
2849 pci_printf(cfg, "Transition from D%d to D%d\n", oldstate,
2850 state);
2851
2852 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2);
2853 if (delay)
2854 DELAY(delay);
2855 return (0);
2856 }
2857
2858 int
pci_get_powerstate_method(device_t dev,device_t child)2859 pci_get_powerstate_method(device_t dev, device_t child)
2860 {
2861 struct pci_devinfo *dinfo = device_get_ivars(child);
2862 pcicfgregs *cfg = &dinfo->cfg;
2863 uint16_t status;
2864 int result;
2865
2866 if (cfg->pp.pp_cap != 0) {
2867 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2);
2868 switch (status & PCIM_PSTAT_DMASK) {
2869 case PCIM_PSTAT_D0:
2870 result = PCI_POWERSTATE_D0;
2871 break;
2872 case PCIM_PSTAT_D1:
2873 result = PCI_POWERSTATE_D1;
2874 break;
2875 case PCIM_PSTAT_D2:
2876 result = PCI_POWERSTATE_D2;
2877 break;
2878 case PCIM_PSTAT_D3:
2879 result = PCI_POWERSTATE_D3;
2880 break;
2881 default:
2882 result = PCI_POWERSTATE_UNKNOWN;
2883 break;
2884 }
2885 } else {
2886 /* No support, device is always at D0 */
2887 result = PCI_POWERSTATE_D0;
2888 }
2889 return (result);
2890 }
2891
2892 /*
2893 * Some convenience functions for PCI device drivers.
2894 */
2895
2896 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)2897 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
2898 {
2899 uint16_t command;
2900
2901 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2902 command |= bit;
2903 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2904 }
2905
2906 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)2907 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
2908 {
2909 uint16_t command;
2910
2911 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2912 command &= ~bit;
2913 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2914 }
2915
2916 int
pci_enable_busmaster_method(device_t dev,device_t child)2917 pci_enable_busmaster_method(device_t dev, device_t child)
2918 {
2919 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2920 return (0);
2921 }
2922
2923 int
pci_disable_busmaster_method(device_t dev,device_t child)2924 pci_disable_busmaster_method(device_t dev, device_t child)
2925 {
2926 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2927 return (0);
2928 }
2929
2930 int
pci_enable_io_method(device_t dev,device_t child,int space)2931 pci_enable_io_method(device_t dev, device_t child, int space)
2932 {
2933 uint16_t bit;
2934
2935 switch(space) {
2936 case SYS_RES_IOPORT:
2937 bit = PCIM_CMD_PORTEN;
2938 break;
2939 case SYS_RES_MEMORY:
2940 bit = PCIM_CMD_MEMEN;
2941 break;
2942 default:
2943 return (EINVAL);
2944 }
2945 pci_set_command_bit(dev, child, bit);
2946 return (0);
2947 }
2948
2949 int
pci_disable_io_method(device_t dev,device_t child,int space)2950 pci_disable_io_method(device_t dev, device_t child, int space)
2951 {
2952 uint16_t bit;
2953
2954 switch(space) {
2955 case SYS_RES_IOPORT:
2956 bit = PCIM_CMD_PORTEN;
2957 break;
2958 case SYS_RES_MEMORY:
2959 bit = PCIM_CMD_MEMEN;
2960 break;
2961 default:
2962 return (EINVAL);
2963 }
2964 pci_clear_command_bit(dev, child, bit);
2965 return (0);
2966 }
2967
2968 /*
2969 * New style pci driver. Parent device is either a pci-host-bridge or a
2970 * pci-pci-bridge. Both kinds are represented by instances of pcib.
2971 */
2972
2973 void
pci_print_verbose(struct pci_devinfo * dinfo)2974 pci_print_verbose(struct pci_devinfo *dinfo)
2975 {
2976
2977 if (bootverbose) {
2978 pcicfgregs *cfg = &dinfo->cfg;
2979
2980 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
2981 cfg->vendor, cfg->device, cfg->revid);
2982 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
2983 cfg->domain, cfg->bus, cfg->slot, cfg->func);
2984 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
2985 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
2986 cfg->mfdev);
2987 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
2988 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
2989 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
2990 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
2991 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
2992 if (cfg->intpin > 0)
2993 printf("\tintpin=%c, irq=%d\n",
2994 cfg->intpin +'a' -1, cfg->intline);
2995 if (cfg->pp.pp_cap) {
2996 uint16_t status;
2997
2998 status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2);
2999 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3000 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3001 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3002 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3003 status & PCIM_PSTAT_DMASK);
3004 }
3005 if (cfg->msi.msi_location) {
3006 int ctrl;
3007
3008 ctrl = cfg->msi.msi_ctrl;
3009 printf("\tMSI supports %d message%s%s%s\n",
3010 cfg->msi.msi_msgnum,
3011 (cfg->msi.msi_msgnum == 1) ? "" : "s",
3012 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3013 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3014 }
3015 if (cfg->msix.msix_location) {
3016 printf("\tMSI-X supports %d message%s ",
3017 cfg->msix.msix_msgnum,
3018 (cfg->msix.msix_msgnum == 1) ? "" : "s");
3019 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3020 printf("in map 0x%x\n",
3021 cfg->msix.msix_table_bar);
3022 else
3023 printf("in maps 0x%x and 0x%x\n",
3024 cfg->msix.msix_table_bar,
3025 cfg->msix.msix_pba_bar);
3026 }
3027 }
3028 }
3029
3030 static int
pci_porten(device_t dev)3031 pci_porten(device_t dev)
3032 {
3033 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3034 }
3035
3036 static int
pci_memen(device_t dev)3037 pci_memen(device_t dev)
3038 {
3039 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3040 }
3041
3042 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3043 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3044 int *bar64)
3045 {
3046 struct pci_devinfo *dinfo;
3047 pci_addr_t map, testval;
3048 int ln2range;
3049 uint16_t cmd;
3050
3051 /*
3052 * The device ROM BAR is special. It is always a 32-bit
3053 * memory BAR. Bit 0 is special and should not be set when
3054 * sizing the BAR.
3055 */
3056 dinfo = device_get_ivars(dev);
3057 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3058 map = pci_read_config(dev, reg, 4);
3059 pci_write_config(dev, reg, 0xfffffffe, 4);
3060 testval = pci_read_config(dev, reg, 4);
3061 pci_write_config(dev, reg, map, 4);
3062 *mapp = map;
3063 *testvalp = testval;
3064 if (bar64 != NULL)
3065 *bar64 = 0;
3066 return;
3067 }
3068
3069 map = pci_read_config(dev, reg, 4);
3070 ln2range = pci_maprange(map);
3071 if (ln2range == 64)
3072 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3073
3074 /*
3075 * Disable decoding via the command register before
3076 * determining the BAR's length since we will be placing it in
3077 * a weird state.
3078 */
3079 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3080 pci_write_config(dev, PCIR_COMMAND,
3081 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3082
3083 /*
3084 * Determine the BAR's length by writing all 1's. The bottom
3085 * log_2(size) bits of the BAR will stick as 0 when we read
3086 * the value back.
3087 *
3088 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3089 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3090 * and combines the result into a 64-bit value." (section 6.2.5.1)
3091 *
3092 * Writes to both registers must be performed before attempting to
3093 * read back the size value.
3094 */
3095 testval = 0;
3096 pci_write_config(dev, reg, 0xffffffff, 4);
3097 if (ln2range == 64) {
3098 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3099 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3100 }
3101 testval |= pci_read_config(dev, reg, 4);
3102
3103 /*
3104 * Restore the original value of the BAR. We may have reprogrammed
3105 * the BAR of the low-level console device and when booting verbose,
3106 * we need the console device addressable.
3107 */
3108 pci_write_config(dev, reg, map, 4);
3109 if (ln2range == 64)
3110 pci_write_config(dev, reg + 4, map >> 32, 4);
3111 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3112
3113 *mapp = map;
3114 *testvalp = testval;
3115 if (bar64 != NULL)
3116 *bar64 = (ln2range == 64);
3117 }
3118
3119 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3120 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3121 {
3122 struct pci_devinfo *dinfo;
3123 int ln2range;
3124
3125 /* The device ROM BAR is always a 32-bit memory BAR. */
3126 dinfo = device_get_ivars(dev);
3127 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3128 ln2range = 32;
3129 else
3130 ln2range = pci_maprange(pm->pm_value);
3131 pci_write_config(dev, pm->pm_reg, base, 4);
3132 if (ln2range == 64)
3133 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3134 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3135 if (ln2range == 64)
3136 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3137 pm->pm_reg + 4, 4) << 32;
3138 }
3139
3140 struct pci_map *
pci_find_bar(device_t dev,int reg)3141 pci_find_bar(device_t dev, int reg)
3142 {
3143 struct pci_devinfo *dinfo;
3144 struct pci_map *pm;
3145
3146 dinfo = device_get_ivars(dev);
3147 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3148 if (pm->pm_reg == reg)
3149 return (pm);
3150 }
3151 return (NULL);
3152 }
3153
3154 struct pci_map *
pci_first_bar(device_t dev)3155 pci_first_bar(device_t dev)
3156 {
3157 struct pci_devinfo *dinfo;
3158
3159 dinfo = device_get_ivars(dev);
3160 return (STAILQ_FIRST(&dinfo->cfg.maps));
3161 }
3162
3163 struct pci_map *
pci_next_bar(struct pci_map * pm)3164 pci_next_bar(struct pci_map *pm)
3165 {
3166 return (STAILQ_NEXT(pm, pm_link));
3167 }
3168
3169 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3170 pci_bar_enabled(device_t dev, struct pci_map *pm)
3171 {
3172 struct pci_devinfo *dinfo;
3173 uint16_t cmd;
3174
3175 dinfo = device_get_ivars(dev);
3176 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3177 !(pm->pm_value & PCIM_BIOS_ENABLE))
3178 return (0);
3179 #ifdef PCI_IOV
3180 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3181 struct pcicfg_iov *iov;
3182
3183 iov = dinfo->cfg.iov;
3184 cmd = pci_read_config(iov->iov_pf,
3185 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3186 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3187 }
3188 #endif
3189 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3190 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3191 return ((cmd & PCIM_CMD_MEMEN) != 0);
3192 else
3193 return ((cmd & PCIM_CMD_PORTEN) != 0);
3194 }
3195
3196 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3197 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3198 {
3199 struct pci_devinfo *dinfo;
3200 struct pci_map *pm, *prev;
3201
3202 dinfo = device_get_ivars(dev);
3203 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3204 pm->pm_reg = reg;
3205 pm->pm_value = value;
3206 pm->pm_size = size;
3207 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3208 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3209 reg));
3210 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3211 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3212 break;
3213 }
3214 if (prev != NULL)
3215 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3216 else
3217 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3218 return (pm);
3219 }
3220
3221 static void
pci_restore_bars(device_t dev)3222 pci_restore_bars(device_t dev)
3223 {
3224 struct pci_devinfo *dinfo;
3225 struct pci_map *pm;
3226 int ln2range;
3227
3228 dinfo = device_get_ivars(dev);
3229 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3230 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3231 ln2range = 32;
3232 else
3233 ln2range = pci_maprange(pm->pm_value);
3234 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3235 if (ln2range == 64)
3236 pci_write_config(dev, pm->pm_reg + 4,
3237 pm->pm_value >> 32, 4);
3238 }
3239 }
3240
3241 /*
3242 * Add a resource based on a pci map register. Return 1 if the map
3243 * register is a 32bit map register or 2 if it is a 64bit register.
3244 */
3245 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3246 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3247 int force, int prefetch)
3248 {
3249 struct pci_map *pm;
3250 pci_addr_t base, map, testval;
3251 pci_addr_t start, end, count;
3252 int barlen, basezero, flags, maprange, mapsize, type;
3253 uint16_t cmd;
3254 struct resource *res;
3255
3256 /*
3257 * The BAR may already exist if the device is a CardBus card
3258 * whose CIS is stored in this BAR.
3259 */
3260 pm = pci_find_bar(dev, reg);
3261 if (pm != NULL) {
3262 maprange = pci_maprange(pm->pm_value);
3263 barlen = maprange == 64 ? 2 : 1;
3264 return (barlen);
3265 }
3266
3267 pci_read_bar(dev, reg, &map, &testval, NULL);
3268 if (PCI_BAR_MEM(map)) {
3269 type = SYS_RES_MEMORY;
3270 if (map & PCIM_BAR_MEM_PREFETCH)
3271 prefetch = 1;
3272 } else
3273 type = SYS_RES_IOPORT;
3274 mapsize = pci_mapsize(testval);
3275 base = pci_mapbase(map);
3276 #ifdef __PCI_BAR_ZERO_VALID
3277 basezero = 0;
3278 #else
3279 basezero = base == 0;
3280 #endif
3281 maprange = pci_maprange(map);
3282 barlen = maprange == 64 ? 2 : 1;
3283
3284 /*
3285 * For I/O registers, if bottom bit is set, and the next bit up
3286 * isn't clear, we know we have a BAR that doesn't conform to the
3287 * spec, so ignore it. Also, sanity check the size of the data
3288 * areas to the type of memory involved. Memory must be at least
3289 * 16 bytes in size, while I/O ranges must be at least 4.
3290 */
3291 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3292 return (barlen);
3293 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3294 (type == SYS_RES_IOPORT && mapsize < 2))
3295 return (barlen);
3296
3297 /* Save a record of this BAR. */
3298 pm = pci_add_bar(dev, reg, map, mapsize);
3299 if (bootverbose) {
3300 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3301 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3302 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3303 printf(", port disabled\n");
3304 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3305 printf(", memory disabled\n");
3306 else
3307 printf(", enabled\n");
3308 }
3309
3310 /*
3311 * If base is 0, then we have problems if this architecture does
3312 * not allow that. It is best to ignore such entries for the
3313 * moment. These will be allocated later if the driver specifically
3314 * requests them. However, some removable buses look better when
3315 * all resources are allocated, so allow '0' to be overridden.
3316 *
3317 * Similarly treat maps whose values is the same as the test value
3318 * read back. These maps have had all f's written to them by the
3319 * BIOS in an attempt to disable the resources.
3320 */
3321 if (!force && (basezero || map == testval))
3322 return (barlen);
3323 if ((u_long)base != base) {
3324 device_printf(bus,
3325 "pci%d:%d:%d:%d bar %#x too many address bits",
3326 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3327 pci_get_function(dev), reg);
3328 return (barlen);
3329 }
3330
3331 /*
3332 * This code theoretically does the right thing, but has
3333 * undesirable side effects in some cases where peripherals
3334 * respond oddly to having these bits enabled. Let the user
3335 * be able to turn them off (since pci_enable_io_modes is 1 by
3336 * default).
3337 */
3338 if (pci_enable_io_modes) {
3339 /* Turn on resources that have been left off by a lazy BIOS */
3340 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3341 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3342 cmd |= PCIM_CMD_PORTEN;
3343 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3344 }
3345 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3346 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3347 cmd |= PCIM_CMD_MEMEN;
3348 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3349 }
3350 } else {
3351 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3352 return (barlen);
3353 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3354 return (barlen);
3355 }
3356
3357 count = (pci_addr_t)1 << mapsize;
3358 flags = RF_ALIGNMENT_LOG2(mapsize);
3359 if (prefetch)
3360 flags |= RF_PREFETCHABLE;
3361 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3362 start = 0; /* Let the parent decide. */
3363 end = ~0;
3364 } else {
3365 start = base;
3366 end = base + count - 1;
3367 }
3368 resource_list_add(rl, type, reg, start, end, count);
3369
3370 /*
3371 * Try to allocate the resource for this BAR from our parent
3372 * so that this resource range is already reserved. The
3373 * driver for this device will later inherit this resource in
3374 * pci_alloc_resource().
3375 */
3376 res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count,
3377 flags);
3378 if ((pci_do_realloc_bars
3379 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3380 && res == NULL && (start != 0 || end != ~0)) {
3381 /*
3382 * If the allocation fails, try to allocate a resource for
3383 * this BAR using any available range. The firmware felt
3384 * it was important enough to assign a resource, so don't
3385 * disable decoding if we can help it.
3386 */
3387 resource_list_delete(rl, type, reg);
3388 resource_list_add(rl, type, reg, 0, ~0, count);
3389 res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0,
3390 count, flags);
3391 }
3392 if (res == NULL) {
3393 /*
3394 * If the allocation fails, delete the resource list entry
3395 * and disable decoding for this device.
3396 *
3397 * If the driver requests this resource in the future,
3398 * pci_reserve_map() will try to allocate a fresh
3399 * resource range.
3400 */
3401 resource_list_delete(rl, type, reg);
3402 pci_disable_io(dev, type);
3403 if (bootverbose)
3404 device_printf(bus,
3405 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3406 pci_get_domain(dev), pci_get_bus(dev),
3407 pci_get_slot(dev), pci_get_function(dev), reg);
3408 } else {
3409 start = rman_get_start(res);
3410 pci_write_bar(dev, pm, start);
3411 }
3412 return (barlen);
3413 }
3414
3415 /*
3416 * For ATA devices we need to decide early what addressing mode to use.
3417 * Legacy demands that the primary and secondary ATA ports sits on the
3418 * same addresses that old ISA hardware did. This dictates that we use
3419 * those addresses and ignore the BAR's if we cannot set PCI native
3420 * addressing mode.
3421 */
3422 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3423 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3424 uint32_t prefetchmask)
3425 {
3426 int rid, type, progif;
3427 #if 0
3428 /* if this device supports PCI native addressing use it */
3429 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3430 if ((progif & 0x8a) == 0x8a) {
3431 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3432 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3433 printf("Trying ATA native PCI addressing mode\n");
3434 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3435 }
3436 }
3437 #endif
3438 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3439 type = SYS_RES_IOPORT;
3440 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3441 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3442 prefetchmask & (1 << 0));
3443 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3444 prefetchmask & (1 << 1));
3445 } else {
3446 rid = PCIR_BAR(0);
3447 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3448 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
3449 0x1f7, 8, 0);
3450 rid = PCIR_BAR(1);
3451 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3452 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
3453 0x3f6, 1, 0);
3454 }
3455 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3456 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3457 prefetchmask & (1 << 2));
3458 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3459 prefetchmask & (1 << 3));
3460 } else {
3461 rid = PCIR_BAR(2);
3462 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3463 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
3464 0x177, 8, 0);
3465 rid = PCIR_BAR(3);
3466 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3467 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
3468 0x376, 1, 0);
3469 }
3470 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3471 prefetchmask & (1 << 4));
3472 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3473 prefetchmask & (1 << 5));
3474 }
3475
3476 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3477 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3478 {
3479 struct pci_devinfo *dinfo = device_get_ivars(dev);
3480 pcicfgregs *cfg = &dinfo->cfg;
3481 char tunable_name[64];
3482 int irq;
3483
3484 /* Has to have an intpin to have an interrupt. */
3485 if (cfg->intpin == 0)
3486 return;
3487
3488 /* Let the user override the IRQ with a tunable. */
3489 irq = PCI_INVALID_IRQ;
3490 snprintf(tunable_name, sizeof(tunable_name),
3491 "hw.pci%d.%d.%d.INT%c.irq",
3492 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3493 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3494 irq = PCI_INVALID_IRQ;
3495
3496 /*
3497 * If we didn't get an IRQ via the tunable, then we either use the
3498 * IRQ value in the intline register or we ask the bus to route an
3499 * interrupt for us. If force_route is true, then we only use the
3500 * value in the intline register if the bus was unable to assign an
3501 * IRQ.
3502 */
3503 if (!PCI_INTERRUPT_VALID(irq)) {
3504 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3505 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3506 if (!PCI_INTERRUPT_VALID(irq))
3507 irq = cfg->intline;
3508 }
3509
3510 /* If after all that we don't have an IRQ, just bail. */
3511 if (!PCI_INTERRUPT_VALID(irq))
3512 return;
3513
3514 /* Update the config register if it changed. */
3515 if (irq != cfg->intline) {
3516 cfg->intline = irq;
3517 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3518 }
3519
3520 /* Add this IRQ as rid 0 interrupt resource. */
3521 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3522 }
3523
3524 /* Perform early OHCI takeover from SMM. */
3525 static void
ohci_early_takeover(device_t self)3526 ohci_early_takeover(device_t self)
3527 {
3528 struct resource *res;
3529 uint32_t ctl;
3530 int rid;
3531 int i;
3532
3533 rid = PCIR_BAR(0);
3534 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3535 if (res == NULL)
3536 return;
3537
3538 ctl = bus_read_4(res, OHCI_CONTROL);
3539 if (ctl & OHCI_IR) {
3540 if (bootverbose)
3541 printf("ohci early: "
3542 "SMM active, request owner change\n");
3543 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3544 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3545 DELAY(1000);
3546 ctl = bus_read_4(res, OHCI_CONTROL);
3547 }
3548 if (ctl & OHCI_IR) {
3549 if (bootverbose)
3550 printf("ohci early: "
3551 "SMM does not respond, resetting\n");
3552 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3553 }
3554 /* Disable interrupts */
3555 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3556 }
3557
3558 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3559 }
3560
3561 /* Perform early UHCI takeover from SMM. */
3562 static void
uhci_early_takeover(device_t self)3563 uhci_early_takeover(device_t self)
3564 {
3565 struct resource *res;
3566 int rid;
3567
3568 /*
3569 * Set the PIRQD enable bit and switch off all the others. We don't
3570 * want legacy support to interfere with us XXX Does this also mean
3571 * that the BIOS won't touch the keyboard anymore if it is connected
3572 * to the ports of the root hub?
3573 */
3574 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3575
3576 /* Disable interrupts */
3577 rid = PCI_UHCI_BASE_REG;
3578 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3579 if (res != NULL) {
3580 bus_write_2(res, UHCI_INTR, 0);
3581 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3582 }
3583 }
3584
3585 /* Perform early EHCI takeover from SMM. */
3586 static void
ehci_early_takeover(device_t self)3587 ehci_early_takeover(device_t self)
3588 {
3589 struct resource *res;
3590 uint32_t cparams;
3591 uint32_t eec;
3592 uint8_t eecp;
3593 uint8_t bios_sem;
3594 uint8_t offs;
3595 int rid;
3596 int i;
3597
3598 rid = PCIR_BAR(0);
3599 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3600 if (res == NULL)
3601 return;
3602
3603 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3604
3605 /* Synchronise with the BIOS if it owns the controller. */
3606 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3607 eecp = EHCI_EECP_NEXT(eec)) {
3608 eec = pci_read_config(self, eecp, 4);
3609 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3610 continue;
3611 }
3612 bios_sem = pci_read_config(self, eecp +
3613 EHCI_LEGSUP_BIOS_SEM, 1);
3614 if (bios_sem == 0) {
3615 continue;
3616 }
3617 if (bootverbose)
3618 printf("ehci early: "
3619 "SMM active, request owner change\n");
3620
3621 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3622
3623 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3624 DELAY(1000);
3625 bios_sem = pci_read_config(self, eecp +
3626 EHCI_LEGSUP_BIOS_SEM, 1);
3627 }
3628
3629 if (bios_sem != 0) {
3630 if (bootverbose)
3631 printf("ehci early: "
3632 "SMM does not respond\n");
3633 }
3634 /* Disable interrupts */
3635 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3636 bus_write_4(res, offs + EHCI_USBINTR, 0);
3637 }
3638 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3639 }
3640
3641 /* Perform early XHCI takeover from SMM. */
3642 static void
xhci_early_takeover(device_t self)3643 xhci_early_takeover(device_t self)
3644 {
3645 struct resource *res;
3646 uint32_t cparams;
3647 uint32_t eec;
3648 uint8_t eecp;
3649 uint8_t bios_sem;
3650 uint8_t offs;
3651 int rid;
3652 int i;
3653
3654 rid = PCIR_BAR(0);
3655 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3656 if (res == NULL)
3657 return;
3658
3659 cparams = bus_read_4(res, XHCI_HCSPARAMS0);
3660
3661 eec = -1;
3662
3663 /* Synchronise with the BIOS if it owns the controller. */
3664 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3665 eecp += XHCI_XECP_NEXT(eec) << 2) {
3666 eec = bus_read_4(res, eecp);
3667
3668 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3669 continue;
3670
3671 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3672 if (bios_sem == 0)
3673 continue;
3674
3675 if (bootverbose)
3676 printf("xhci early: "
3677 "SMM active, request owner change\n");
3678
3679 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3680
3681 /* wait a maximum of 5 second */
3682
3683 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3684 DELAY(1000);
3685 bios_sem = bus_read_1(res, eecp +
3686 XHCI_XECP_BIOS_SEM);
3687 }
3688
3689 if (bios_sem != 0) {
3690 if (bootverbose)
3691 printf("xhci early: "
3692 "SMM does not respond\n");
3693 }
3694
3695 /* Disable interrupts */
3696 offs = bus_read_1(res, XHCI_CAPLENGTH);
3697 bus_write_4(res, offs + XHCI_USBCMD, 0);
3698 bus_read_4(res, offs + XHCI_USBSTS);
3699 }
3700 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3701 }
3702
3703 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
3704 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3705 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3706 struct resource_list *rl)
3707 {
3708 struct resource *res;
3709 char *cp;
3710 rman_res_t start, end, count;
3711 int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3712
3713 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3714 case PCIM_HDRTYPE_BRIDGE:
3715 sec_reg = PCIR_SECBUS_1;
3716 sub_reg = PCIR_SUBBUS_1;
3717 break;
3718 case PCIM_HDRTYPE_CARDBUS:
3719 sec_reg = PCIR_SECBUS_2;
3720 sub_reg = PCIR_SUBBUS_2;
3721 break;
3722 default:
3723 return;
3724 }
3725
3726 /*
3727 * If the existing bus range is valid, attempt to reserve it
3728 * from our parent. If this fails for any reason, clear the
3729 * secbus and subbus registers.
3730 *
3731 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3732 * This would at least preserve the existing sec_bus if it is
3733 * valid.
3734 */
3735 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3736 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3737
3738 /* Quirk handling. */
3739 switch (pci_get_devid(dev)) {
3740 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3741 sup_bus = pci_read_config(dev, 0x41, 1);
3742 if (sup_bus != 0xff) {
3743 sec_bus = sup_bus + 1;
3744 sub_bus = sup_bus + 1;
3745 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3746 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3747 }
3748 break;
3749
3750 case 0x00dd10de:
3751 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3752 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3753 break;
3754 if (strncmp(cp, "Compal", 6) != 0) {
3755 freeenv(cp);
3756 break;
3757 }
3758 freeenv(cp);
3759 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3760 break;
3761 if (strncmp(cp, "08A0", 4) != 0) {
3762 freeenv(cp);
3763 break;
3764 }
3765 freeenv(cp);
3766 if (sub_bus < 0xa) {
3767 sub_bus = 0xa;
3768 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3769 }
3770 break;
3771 }
3772
3773 if (bootverbose)
3774 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3775 if (sec_bus > 0 && sub_bus >= sec_bus) {
3776 start = sec_bus;
3777 end = sub_bus;
3778 count = end - start + 1;
3779
3780 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3781
3782 /*
3783 * If requested, clear secondary bus registers in
3784 * bridge devices to force a complete renumbering
3785 * rather than reserving the existing range. However,
3786 * preserve the existing size.
3787 */
3788 if (pci_clear_buses)
3789 goto clear;
3790
3791 rid = 0;
3792 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid,
3793 start, end, count, 0);
3794 if (res != NULL)
3795 return;
3796
3797 if (bootverbose)
3798 device_printf(bus,
3799 "pci%d:%d:%d:%d secbus failed to allocate\n",
3800 pci_get_domain(dev), pci_get_bus(dev),
3801 pci_get_slot(dev), pci_get_function(dev));
3802 }
3803
3804 clear:
3805 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3806 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3807 }
3808
3809 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3810 pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start,
3811 rman_res_t end, rman_res_t count, u_int flags)
3812 {
3813 struct pci_devinfo *dinfo;
3814 pcicfgregs *cfg;
3815 struct resource_list *rl;
3816 struct resource *res;
3817 int sec_reg, sub_reg;
3818
3819 dinfo = device_get_ivars(child);
3820 cfg = &dinfo->cfg;
3821 rl = &dinfo->resources;
3822 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3823 case PCIM_HDRTYPE_BRIDGE:
3824 sec_reg = PCIR_SECBUS_1;
3825 sub_reg = PCIR_SUBBUS_1;
3826 break;
3827 case PCIM_HDRTYPE_CARDBUS:
3828 sec_reg = PCIR_SECBUS_2;
3829 sub_reg = PCIR_SUBBUS_2;
3830 break;
3831 default:
3832 return (NULL);
3833 }
3834
3835 if (*rid != 0)
3836 return (NULL);
3837
3838 if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL)
3839 resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count);
3840 if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) {
3841 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3842 start, end, count, flags & ~RF_ACTIVE);
3843 if (res == NULL) {
3844 resource_list_delete(rl, PCI_RES_BUS, *rid);
3845 device_printf(child, "allocating %ju bus%s failed\n",
3846 count, count == 1 ? "" : "es");
3847 return (NULL);
3848 }
3849 if (bootverbose)
3850 device_printf(child,
3851 "Lazy allocation of %ju bus%s at %ju\n", count,
3852 count == 1 ? "" : "es", rman_get_start(res));
3853 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3854 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3855 }
3856 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3857 end, count, flags));
3858 }
3859 #endif
3860
3861 static int
pci_ea_bei_to_rid(device_t dev,int bei)3862 pci_ea_bei_to_rid(device_t dev, int bei)
3863 {
3864 #ifdef PCI_IOV
3865 struct pci_devinfo *dinfo;
3866 int iov_pos;
3867 struct pcicfg_iov *iov;
3868
3869 dinfo = device_get_ivars(dev);
3870 iov = dinfo->cfg.iov;
3871 if (iov != NULL)
3872 iov_pos = iov->iov_pos;
3873 else
3874 iov_pos = 0;
3875 #endif
3876
3877 /* Check if matches BAR */
3878 if ((bei >= PCIM_EA_BEI_BAR_0) &&
3879 (bei <= PCIM_EA_BEI_BAR_5))
3880 return (PCIR_BAR(bei));
3881
3882 /* Check ROM */
3883 if (bei == PCIM_EA_BEI_ROM)
3884 return (PCIR_BIOS);
3885
3886 #ifdef PCI_IOV
3887 /* Check if matches VF_BAR */
3888 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
3889 (bei <= PCIM_EA_BEI_VF_BAR_5))
3890 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
3891 iov_pos);
3892 #endif
3893
3894 return (-1);
3895 }
3896
3897 int
pci_ea_is_enabled(device_t dev,int rid)3898 pci_ea_is_enabled(device_t dev, int rid)
3899 {
3900 struct pci_ea_entry *ea;
3901 struct pci_devinfo *dinfo;
3902
3903 dinfo = device_get_ivars(dev);
3904
3905 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3906 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
3907 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
3908 }
3909
3910 return (0);
3911 }
3912
3913 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)3914 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
3915 {
3916 struct pci_ea_entry *ea;
3917 struct pci_devinfo *dinfo;
3918 pci_addr_t start, end, count;
3919 struct resource_list *rl;
3920 int type, flags, rid;
3921 struct resource *res;
3922 uint32_t tmp;
3923 #ifdef PCI_IOV
3924 struct pcicfg_iov *iov;
3925 #endif
3926
3927 dinfo = device_get_ivars(dev);
3928 rl = &dinfo->resources;
3929 flags = 0;
3930
3931 #ifdef PCI_IOV
3932 iov = dinfo->cfg.iov;
3933 #endif
3934
3935 if (dinfo->cfg.ea.ea_location == 0)
3936 return;
3937
3938 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3939 /*
3940 * TODO: Ignore EA-BAR if is not enabled.
3941 * Currently the EA implementation supports
3942 * only situation, where EA structure contains
3943 * predefined entries. In case they are not enabled
3944 * leave them unallocated and proceed with
3945 * a legacy-BAR mechanism.
3946 */
3947 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
3948 continue;
3949
3950 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
3951 case PCIM_EA_P_MEM_PREFETCH:
3952 case PCIM_EA_P_VF_MEM_PREFETCH:
3953 flags = RF_PREFETCHABLE;
3954 /* FALLTHROUGH */
3955 case PCIM_EA_P_VF_MEM:
3956 case PCIM_EA_P_MEM:
3957 type = SYS_RES_MEMORY;
3958 break;
3959 case PCIM_EA_P_IO:
3960 type = SYS_RES_IOPORT;
3961 break;
3962 default:
3963 continue;
3964 }
3965
3966 if (alloc_iov != 0) {
3967 #ifdef PCI_IOV
3968 /* Allocating IOV, confirm BEI matches */
3969 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
3970 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
3971 continue;
3972 #else
3973 continue;
3974 #endif
3975 } else {
3976 /* Allocating BAR, confirm BEI matches */
3977 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
3978 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
3979 (ea->eae_bei != PCIM_EA_BEI_ROM))
3980 continue;
3981 }
3982
3983 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
3984 if (rid < 0)
3985 continue;
3986
3987 /* Skip resources already allocated by EA */
3988 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
3989 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
3990 continue;
3991
3992 start = ea->eae_base;
3993 count = ea->eae_max_offset + 1;
3994 #ifdef PCI_IOV
3995 if (iov != NULL)
3996 count = count * iov->iov_num_vfs;
3997 #endif
3998 end = start + count - 1;
3999 if (count == 0)
4000 continue;
4001
4002 resource_list_add(rl, type, rid, start, end, count);
4003 res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count,
4004 flags);
4005 if (res == NULL) {
4006 resource_list_delete(rl, type, rid);
4007
4008 /*
4009 * Failed to allocate using EA, disable entry.
4010 * Another attempt to allocation will be performed
4011 * further, but this time using legacy BAR registers
4012 */
4013 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4014 tmp &= ~PCIM_EA_ENABLE;
4015 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4016
4017 /*
4018 * Disabling entry might fail in case it is hardwired.
4019 * Read flags again to match current status.
4020 */
4021 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4022
4023 continue;
4024 }
4025
4026 /* As per specification, fill BAR with zeros */
4027 pci_write_config(dev, rid, 0, 4);
4028 }
4029 }
4030
4031 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4032 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4033 {
4034 struct pci_devinfo *dinfo;
4035 pcicfgregs *cfg;
4036 struct resource_list *rl;
4037 const struct pci_quirk *q;
4038 uint32_t devid;
4039 int i;
4040
4041 dinfo = device_get_ivars(dev);
4042 cfg = &dinfo->cfg;
4043 rl = &dinfo->resources;
4044 devid = (cfg->device << 16) | cfg->vendor;
4045
4046 /* Allocate resources using Enhanced Allocation */
4047 pci_add_resources_ea(bus, dev, 0);
4048
4049 /* ATA devices needs special map treatment */
4050 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4051 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4052 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4053 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4054 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4055 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4056 else
4057 for (i = 0; i < cfg->nummaps;) {
4058 /* Skip resources already managed by EA */
4059 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4060 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4061 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4062 i++;
4063 continue;
4064 }
4065
4066 /*
4067 * Skip quirked resources.
4068 */
4069 for (q = &pci_quirks[0]; q->devid != 0; q++)
4070 if (q->devid == devid &&
4071 q->type == PCI_QUIRK_UNMAP_REG &&
4072 q->arg1 == PCIR_BAR(i))
4073 break;
4074 if (q->devid != 0) {
4075 i++;
4076 continue;
4077 }
4078 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4079 prefetchmask & (1 << i));
4080 }
4081
4082 /*
4083 * Add additional, quirked resources.
4084 */
4085 for (q = &pci_quirks[0]; q->devid != 0; q++)
4086 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4087 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4088
4089 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
4090 #ifdef __PCI_REROUTE_INTERRUPT
4091 /*
4092 * Try to re-route interrupts. Sometimes the BIOS or
4093 * firmware may leave bogus values in these registers.
4094 * If the re-route fails, then just stick with what we
4095 * have.
4096 */
4097 pci_assign_interrupt(bus, dev, 1);
4098 #else
4099 pci_assign_interrupt(bus, dev, 0);
4100 #endif
4101 }
4102
4103 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4104 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4105 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4106 xhci_early_takeover(dev);
4107 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4108 ehci_early_takeover(dev);
4109 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4110 ohci_early_takeover(dev);
4111 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4112 uhci_early_takeover(dev);
4113 }
4114
4115 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
4116 /*
4117 * Reserve resources for secondary bus ranges behind bridge
4118 * devices.
4119 */
4120 pci_reserve_secbus(bus, dev, cfg, rl);
4121 #endif
4122 }
4123
4124 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4125 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4126 int slot, int func)
4127 {
4128 struct pci_devinfo *dinfo;
4129
4130 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4131 if (dinfo != NULL)
4132 pci_add_child(dev, dinfo);
4133
4134 return (dinfo);
4135 }
4136
4137 void
pci_add_children(device_t dev,int domain,int busno)4138 pci_add_children(device_t dev, int domain, int busno)
4139 {
4140 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4141 device_t pcib = device_get_parent(dev);
4142 struct pci_devinfo *dinfo;
4143 int maxslots;
4144 int s, f, pcifunchigh;
4145 uint8_t hdrtype;
4146 int first_func;
4147
4148 /*
4149 * Try to detect a device at slot 0, function 0. If it exists, try to
4150 * enable ARI. We must enable ARI before detecting the rest of the
4151 * functions on this bus as ARI changes the set of slots and functions
4152 * that are legal on this bus.
4153 */
4154 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4155 if (dinfo != NULL && pci_enable_ari)
4156 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4157
4158 /*
4159 * Start looking for new devices on slot 0 at function 1 because we
4160 * just identified the device at slot 0, function 0.
4161 */
4162 first_func = 1;
4163
4164 maxslots = PCIB_MAXSLOTS(pcib);
4165 for (s = 0; s <= maxslots; s++, first_func = 0) {
4166 pcifunchigh = 0;
4167 f = 0;
4168 DELAY(1);
4169
4170 /* If function 0 is not present, skip to the next slot. */
4171 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4172 continue;
4173 hdrtype = REG(PCIR_HDRTYPE, 1);
4174 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4175 continue;
4176 if (hdrtype & PCIM_MFDEV)
4177 pcifunchigh = PCIB_MAXFUNCS(pcib);
4178 for (f = first_func; f <= pcifunchigh; f++)
4179 pci_identify_function(pcib, dev, domain, busno, s, f);
4180 }
4181 #undef REG
4182 }
4183
4184 int
pci_rescan_method(device_t dev)4185 pci_rescan_method(device_t dev)
4186 {
4187 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4188 device_t pcib = device_get_parent(dev);
4189 device_t child, *devlist, *unchanged;
4190 int devcount, error, i, j, maxslots, oldcount;
4191 int busno, domain, s, f, pcifunchigh;
4192 uint8_t hdrtype;
4193
4194 /* No need to check for ARI on a rescan. */
4195 error = device_get_children(dev, &devlist, &devcount);
4196 if (error)
4197 return (error);
4198 if (devcount != 0) {
4199 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4200 M_NOWAIT | M_ZERO);
4201 if (unchanged == NULL) {
4202 free(devlist, M_TEMP);
4203 return (ENOMEM);
4204 }
4205 } else
4206 unchanged = NULL;
4207
4208 domain = pcib_get_domain(dev);
4209 busno = pcib_get_bus(dev);
4210 maxslots = PCIB_MAXSLOTS(pcib);
4211 for (s = 0; s <= maxslots; s++) {
4212 /* If function 0 is not present, skip to the next slot. */
4213 f = 0;
4214 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4215 continue;
4216 pcifunchigh = 0;
4217 hdrtype = REG(PCIR_HDRTYPE, 1);
4218 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4219 continue;
4220 if (hdrtype & PCIM_MFDEV)
4221 pcifunchigh = PCIB_MAXFUNCS(pcib);
4222 for (f = 0; f <= pcifunchigh; f++) {
4223 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4224 continue;
4225
4226 /*
4227 * Found a valid function. Check if a
4228 * device_t for this device already exists.
4229 */
4230 for (i = 0; i < devcount; i++) {
4231 child = devlist[i];
4232 if (child == NULL)
4233 continue;
4234 if (pci_get_slot(child) == s &&
4235 pci_get_function(child) == f) {
4236 unchanged[i] = child;
4237 goto next_func;
4238 }
4239 }
4240
4241 pci_identify_function(pcib, dev, domain, busno, s, f);
4242 next_func:;
4243 }
4244 }
4245
4246 /* Remove devices that are no longer present. */
4247 for (i = 0; i < devcount; i++) {
4248 if (unchanged[i] != NULL)
4249 continue;
4250 device_delete_child(dev, devlist[i]);
4251 }
4252
4253 free(devlist, M_TEMP);
4254 oldcount = devcount;
4255
4256 /* Try to attach the devices just added. */
4257 error = device_get_children(dev, &devlist, &devcount);
4258 if (error) {
4259 free(unchanged, M_TEMP);
4260 return (error);
4261 }
4262
4263 for (i = 0; i < devcount; i++) {
4264 for (j = 0; j < oldcount; j++) {
4265 if (devlist[i] == unchanged[j])
4266 goto next_device;
4267 }
4268
4269 device_probe_and_attach(devlist[i]);
4270 next_device:;
4271 }
4272
4273 free(unchanged, M_TEMP);
4274 free(devlist, M_TEMP);
4275 return (0);
4276 #undef REG
4277 }
4278
4279 #ifdef PCI_IOV
4280 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4281 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4282 uint16_t did)
4283 {
4284 struct pci_devinfo *vf_dinfo;
4285 device_t pcib;
4286 int busno, slot, func;
4287
4288 pcib = device_get_parent(bus);
4289
4290 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4291
4292 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4293 slot, func, vid, did);
4294
4295 vf_dinfo->cfg.flags |= PCICFG_VF;
4296 pci_add_child(bus, vf_dinfo);
4297
4298 return (vf_dinfo->cfg.dev);
4299 }
4300
4301 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4302 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4303 uint16_t vid, uint16_t did)
4304 {
4305
4306 return (pci_add_iov_child(bus, pf, rid, vid, did));
4307 }
4308 #endif
4309
4310 /*
4311 * For PCIe device set Max_Payload_Size to match PCIe root's.
4312 */
4313 static void
pcie_setup_mps(device_t dev)4314 pcie_setup_mps(device_t dev)
4315 {
4316 struct pci_devinfo *dinfo = device_get_ivars(dev);
4317 device_t root;
4318 uint16_t rmps, mmps, mps;
4319
4320 if (dinfo->cfg.pcie.pcie_location == 0)
4321 return;
4322 root = pci_find_pcie_root_port(dev);
4323 if (root == NULL)
4324 return;
4325 /* Check whether the MPS is already configured. */
4326 rmps = pcie_read_config(root, PCIER_DEVICE_CTL, 2) &
4327 PCIEM_CTL_MAX_PAYLOAD;
4328 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4329 PCIEM_CTL_MAX_PAYLOAD;
4330 if (mps == rmps)
4331 return;
4332 /* Check whether the device is capable of the root's MPS. */
4333 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4334 PCIEM_CAP_MAX_PAYLOAD) << 5;
4335 if (rmps > mmps) {
4336 /*
4337 * The device is unable to handle root's MPS. Limit root.
4338 * XXX: We should traverse through all the tree, applying
4339 * it to all the devices.
4340 */
4341 pcie_adjust_config(root, PCIER_DEVICE_CTL,
4342 PCIEM_CTL_MAX_PAYLOAD, mmps, 2);
4343 } else {
4344 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
4345 PCIEM_CTL_MAX_PAYLOAD, rmps, 2);
4346 }
4347 }
4348
4349 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4350 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4351 {
4352 int aer;
4353 uint32_t r;
4354 uint16_t r2;
4355
4356 if (dinfo->cfg.pcie.pcie_location != 0 &&
4357 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4358 r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4359 PCIER_ROOT_CTL, 2);
4360 r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR |
4361 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL);
4362 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4363 PCIER_ROOT_CTL, r2, 2);
4364 }
4365 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4366 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4367 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4368 if (r != 0 && bootverbose) {
4369 pci_printf(&dinfo->cfg,
4370 "clearing AER UC 0x%08x -> 0x%08x\n",
4371 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4372 4));
4373 }
4374
4375 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4376 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4377 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4378 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4379 PCIM_AER_UC_POISONED_TLP |
4380 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4381 PCIM_AER_UC_COMPLETION_TIMEOUT |
4382 PCIM_AER_UC_COMPLETER_ABORT |
4383 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4384 PCIM_AER_UC_RECEIVER_OVERFLOW |
4385 PCIM_AER_UC_MALFORMED_TLP |
4386 PCIM_AER_UC_ECRC_ERROR |
4387 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4388 PCIM_AER_UC_ACS_VIOLATION |
4389 PCIM_AER_UC_INTERNAL_ERROR |
4390 PCIM_AER_UC_MC_BLOCKED_TLP |
4391 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4392 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4393 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4394
4395 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4396 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4397 if (r != 0 && bootverbose) {
4398 pci_printf(&dinfo->cfg,
4399 "clearing AER COR 0x%08x -> 0x%08x\n",
4400 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4401 4));
4402 }
4403
4404 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4405 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4406 PCIM_AER_COR_BAD_TLP |
4407 PCIM_AER_COR_BAD_DLLP |
4408 PCIM_AER_COR_REPLAY_ROLLOVER |
4409 PCIM_AER_COR_REPLAY_TIMEOUT |
4410 PCIM_AER_COR_ADVISORY_NF_ERROR |
4411 PCIM_AER_COR_INTERNAL_ERROR |
4412 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4413 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4414
4415 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4416 PCIER_DEVICE_CTL, 2);
4417 r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4418 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4419 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4420 PCIER_DEVICE_CTL, r, 2);
4421 }
4422 }
4423
4424 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4425 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4426 {
4427 device_t dev;
4428
4429 dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1);
4430 device_set_ivars(dev, dinfo);
4431 resource_list_init(&dinfo->resources);
4432 pci_cfg_save(dev, dinfo, 0);
4433 pci_cfg_restore(dev, dinfo);
4434 pci_print_verbose(dinfo);
4435 pci_add_resources(bus, dev, 0, 0);
4436 if (pci_enable_mps_tune)
4437 pcie_setup_mps(dev);
4438 pci_child_added(dinfo->cfg.dev);
4439
4440 if (pci_clear_aer_on_attach)
4441 pci_add_child_clear_aer(dev, dinfo);
4442
4443 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4444 }
4445
4446 void
pci_child_added_method(device_t dev,device_t child)4447 pci_child_added_method(device_t dev, device_t child)
4448 {
4449
4450 }
4451
4452 static int
pci_probe(device_t dev)4453 pci_probe(device_t dev)
4454 {
4455
4456 device_set_desc(dev, "PCI bus");
4457
4458 /* Allow other subclasses to override this driver. */
4459 return (BUS_PROBE_GENERIC);
4460 }
4461
4462 int
pci_attach_common(device_t dev)4463 pci_attach_common(device_t dev)
4464 {
4465 struct pci_softc *sc;
4466 int busno, domain;
4467 #ifdef PCI_RES_BUS
4468 int rid;
4469 #endif
4470
4471 sc = device_get_softc(dev);
4472 domain = pcib_get_domain(dev);
4473 busno = pcib_get_bus(dev);
4474 #ifdef PCI_RES_BUS
4475 rid = 0;
4476 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4477 1, 0);
4478 if (sc->sc_bus == NULL) {
4479 device_printf(dev, "failed to allocate bus number\n");
4480 return (ENXIO);
4481 }
4482 #endif
4483 if (bootverbose)
4484 device_printf(dev, "domain=%d, physical bus=%d\n",
4485 domain, busno);
4486 sc->sc_dma_tag = bus_get_dma_tag(dev);
4487 return (0);
4488 }
4489
4490 int
pci_attach(device_t dev)4491 pci_attach(device_t dev)
4492 {
4493 int busno, domain, error;
4494
4495 error = pci_attach_common(dev);
4496 if (error)
4497 return (error);
4498
4499 /*
4500 * Since there can be multiple independently numbered PCI
4501 * buses on systems with multiple PCI domains, we can't use
4502 * the unit number to decide which bus we are probing. We ask
4503 * the parent pcib what our domain and bus numbers are.
4504 */
4505 domain = pcib_get_domain(dev);
4506 busno = pcib_get_bus(dev);
4507 pci_add_children(dev, domain, busno);
4508 return (bus_generic_attach(dev));
4509 }
4510
4511 int
pci_detach(device_t dev)4512 pci_detach(device_t dev)
4513 {
4514 #ifdef PCI_RES_BUS
4515 struct pci_softc *sc;
4516 #endif
4517 int error;
4518
4519 error = bus_generic_detach(dev);
4520 if (error)
4521 return (error);
4522 #ifdef PCI_RES_BUS
4523 sc = device_get_softc(dev);
4524 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
4525 if (error)
4526 return (error);
4527 #endif
4528 return (device_delete_children(dev));
4529 }
4530
4531 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)4532 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
4533 {
4534 int line, unit;
4535 const char *at;
4536 char me1[24], me2[32];
4537 uint8_t b, s, f;
4538 uint32_t d;
4539 device_location_cache_t *cache;
4540
4541 d = pci_get_domain(child);
4542 b = pci_get_bus(child);
4543 s = pci_get_slot(child);
4544 f = pci_get_function(child);
4545 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
4546 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
4547 line = 0;
4548 cache = dev_wired_cache_init();
4549 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
4550 resource_string_value(name, unit, "at", &at);
4551 if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) {
4552 *unitp = unit;
4553 break;
4554 }
4555 if (dev_wired_cache_match(cache, child, at)) {
4556 *unitp = unit;
4557 break;
4558 }
4559 }
4560 dev_wired_cache_fini(cache);
4561 }
4562
4563 static void
pci_set_power_child(device_t dev,device_t child,int state)4564 pci_set_power_child(device_t dev, device_t child, int state)
4565 {
4566 device_t pcib;
4567 int dstate;
4568
4569 /*
4570 * Set the device to the given state. If the firmware suggests
4571 * a different power state, use it instead. If power management
4572 * is not present, the firmware is responsible for managing
4573 * device power. Skip children who aren't attached since they
4574 * are handled separately.
4575 */
4576 pcib = device_get_parent(dev);
4577 dstate = state;
4578 if (device_is_attached(child) &&
4579 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
4580 pci_set_powerstate(child, dstate);
4581 }
4582
4583 int
pci_suspend_child(device_t dev,device_t child)4584 pci_suspend_child(device_t dev, device_t child)
4585 {
4586 struct pci_devinfo *dinfo;
4587 struct resource_list_entry *rle;
4588 int error;
4589
4590 dinfo = device_get_ivars(child);
4591
4592 /*
4593 * Save the PCI configuration space for the child and set the
4594 * device in the appropriate power state for this sleep state.
4595 */
4596 pci_cfg_save(child, dinfo, 0);
4597
4598 /* Suspend devices before potentially powering them down. */
4599 error = bus_generic_suspend_child(dev, child);
4600
4601 if (error)
4602 return (error);
4603
4604 if (pci_do_power_suspend) {
4605 /*
4606 * Make sure this device's interrupt handler is not invoked
4607 * in the case the device uses a shared interrupt that can
4608 * be raised by some other device.
4609 * This is applicable only to regular (legacy) PCI interrupts
4610 * as MSI/MSI-X interrupts are never shared.
4611 */
4612 rle = resource_list_find(&dinfo->resources,
4613 SYS_RES_IRQ, 0);
4614 if (rle != NULL && rle->res != NULL)
4615 (void)bus_suspend_intr(child, rle->res);
4616 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
4617 }
4618
4619 return (0);
4620 }
4621
4622 int
pci_resume_child(device_t dev,device_t child)4623 pci_resume_child(device_t dev, device_t child)
4624 {
4625 struct pci_devinfo *dinfo;
4626 struct resource_list_entry *rle;
4627
4628 if (pci_do_power_resume)
4629 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
4630
4631 dinfo = device_get_ivars(child);
4632 pci_cfg_restore(child, dinfo);
4633 if (!device_is_attached(child))
4634 pci_cfg_save(child, dinfo, 1);
4635
4636 bus_generic_resume_child(dev, child);
4637
4638 /*
4639 * Allow interrupts only after fully resuming the driver and hardware.
4640 */
4641 if (pci_do_power_suspend) {
4642 /* See pci_suspend_child for details. */
4643 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
4644 if (rle != NULL && rle->res != NULL)
4645 (void)bus_resume_intr(child, rle->res);
4646 }
4647
4648 return (0);
4649 }
4650
4651 int
pci_resume(device_t dev)4652 pci_resume(device_t dev)
4653 {
4654 device_t child, *devlist;
4655 int error, i, numdevs;
4656
4657 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
4658 return (error);
4659
4660 /*
4661 * Resume critical devices first, then everything else later.
4662 */
4663 for (i = 0; i < numdevs; i++) {
4664 child = devlist[i];
4665 switch (pci_get_class(child)) {
4666 case PCIC_DISPLAY:
4667 case PCIC_MEMORY:
4668 case PCIC_BRIDGE:
4669 case PCIC_BASEPERIPH:
4670 BUS_RESUME_CHILD(dev, child);
4671 break;
4672 }
4673 }
4674 for (i = 0; i < numdevs; i++) {
4675 child = devlist[i];
4676 switch (pci_get_class(child)) {
4677 case PCIC_DISPLAY:
4678 case PCIC_MEMORY:
4679 case PCIC_BRIDGE:
4680 case PCIC_BASEPERIPH:
4681 break;
4682 default:
4683 BUS_RESUME_CHILD(dev, child);
4684 }
4685 }
4686 free(devlist, M_TEMP);
4687 return (0);
4688 }
4689
4690 static void
pci_load_vendor_data(void)4691 pci_load_vendor_data(void)
4692 {
4693 caddr_t data;
4694 void *ptr;
4695 size_t sz;
4696
4697 data = preload_search_by_type("pci_vendor_data");
4698 if (data != NULL) {
4699 ptr = preload_fetch_addr(data);
4700 sz = preload_fetch_size(data);
4701 if (ptr != NULL && sz != 0) {
4702 pci_vendordata = ptr;
4703 pci_vendordata_size = sz;
4704 /* terminate the database */
4705 pci_vendordata[pci_vendordata_size] = '\n';
4706 }
4707 }
4708 }
4709
4710 void
pci_driver_added(device_t dev,driver_t * driver)4711 pci_driver_added(device_t dev, driver_t *driver)
4712 {
4713 int numdevs;
4714 device_t *devlist;
4715 device_t child;
4716 struct pci_devinfo *dinfo;
4717 int i;
4718
4719 if (bootverbose)
4720 device_printf(dev, "driver added\n");
4721 DEVICE_IDENTIFY(driver, dev);
4722 if (device_get_children(dev, &devlist, &numdevs) != 0)
4723 return;
4724 for (i = 0; i < numdevs; i++) {
4725 child = devlist[i];
4726 if (device_get_state(child) != DS_NOTPRESENT)
4727 continue;
4728 dinfo = device_get_ivars(child);
4729 pci_print_verbose(dinfo);
4730 if (bootverbose)
4731 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
4732 pci_cfg_restore(child, dinfo);
4733 if (device_probe_and_attach(child) != 0)
4734 pci_child_detached(dev, child);
4735 }
4736 free(devlist, M_TEMP);
4737 }
4738
4739 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)4740 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
4741 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
4742 {
4743 struct pci_devinfo *dinfo;
4744 struct msix_table_entry *mte;
4745 struct msix_vector *mv;
4746 uint64_t addr;
4747 uint32_t data;
4748 void *cookie;
4749 int error, rid;
4750
4751 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
4752 arg, &cookie);
4753 if (error)
4754 return (error);
4755
4756 /* If this is not a direct child, just bail out. */
4757 if (device_get_parent(child) != dev) {
4758 *cookiep = cookie;
4759 return(0);
4760 }
4761
4762 rid = rman_get_rid(irq);
4763 if (rid == 0) {
4764 /* Make sure that INTx is enabled */
4765 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4766 } else {
4767 /*
4768 * Check to see if the interrupt is MSI or MSI-X.
4769 * Ask our parent to map the MSI and give
4770 * us the address and data register values.
4771 * If we fail for some reason, teardown the
4772 * interrupt handler.
4773 */
4774 dinfo = device_get_ivars(child);
4775 if (dinfo->cfg.msi.msi_alloc > 0) {
4776 if (dinfo->cfg.msi.msi_addr == 0) {
4777 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
4778 ("MSI has handlers, but vectors not mapped"));
4779 error = PCIB_MAP_MSI(device_get_parent(dev),
4780 child, rman_get_start(irq), &addr, &data);
4781 if (error)
4782 goto bad;
4783 dinfo->cfg.msi.msi_addr = addr;
4784 dinfo->cfg.msi.msi_data = data;
4785 }
4786 if (dinfo->cfg.msi.msi_handlers == 0)
4787 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
4788 dinfo->cfg.msi.msi_data);
4789 dinfo->cfg.msi.msi_handlers++;
4790 } else {
4791 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4792 ("No MSI or MSI-X interrupts allocated"));
4793 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4794 ("MSI-X index too high"));
4795 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4796 KASSERT(mte->mte_vector != 0, ("no message vector"));
4797 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
4798 KASSERT(mv->mv_irq == rman_get_start(irq),
4799 ("IRQ mismatch"));
4800 if (mv->mv_address == 0) {
4801 KASSERT(mte->mte_handlers == 0,
4802 ("MSI-X table entry has handlers, but vector not mapped"));
4803 error = PCIB_MAP_MSI(device_get_parent(dev),
4804 child, rman_get_start(irq), &addr, &data);
4805 if (error)
4806 goto bad;
4807 mv->mv_address = addr;
4808 mv->mv_data = data;
4809 }
4810
4811 /*
4812 * The MSIX table entry must be made valid by
4813 * incrementing the mte_handlers before
4814 * calling pci_enable_msix() and
4815 * pci_resume_msix(). Else the MSIX rewrite
4816 * table quirk will not work as expected.
4817 */
4818 mte->mte_handlers++;
4819 if (mte->mte_handlers == 1) {
4820 pci_enable_msix(child, rid - 1, mv->mv_address,
4821 mv->mv_data);
4822 pci_unmask_msix(child, rid - 1);
4823 }
4824 }
4825
4826 /*
4827 * Make sure that INTx is disabled if we are using MSI/MSI-X,
4828 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
4829 * in which case we "enable" INTx so MSI/MSI-X actually works.
4830 */
4831 if (!pci_has_quirk(pci_get_devid(child),
4832 PCI_QUIRK_MSI_INTX_BUG))
4833 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4834 else
4835 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4836 bad:
4837 if (error) {
4838 (void)bus_generic_teardown_intr(dev, child, irq,
4839 cookie);
4840 return (error);
4841 }
4842 }
4843 *cookiep = cookie;
4844 return (0);
4845 }
4846
4847 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)4848 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
4849 void *cookie)
4850 {
4851 struct msix_table_entry *mte;
4852 struct resource_list_entry *rle;
4853 struct pci_devinfo *dinfo;
4854 int error, rid;
4855
4856 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
4857 return (EINVAL);
4858
4859 /* If this isn't a direct child, just bail out */
4860 if (device_get_parent(child) != dev)
4861 return(bus_generic_teardown_intr(dev, child, irq, cookie));
4862
4863 rid = rman_get_rid(irq);
4864 if (rid == 0) {
4865 /* Mask INTx */
4866 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4867 } else {
4868 /*
4869 * Check to see if the interrupt is MSI or MSI-X. If so,
4870 * decrement the appropriate handlers count and mask the
4871 * MSI-X message, or disable MSI messages if the count
4872 * drops to 0.
4873 */
4874 dinfo = device_get_ivars(child);
4875 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
4876 if (rle->res != irq)
4877 return (EINVAL);
4878 if (dinfo->cfg.msi.msi_alloc > 0) {
4879 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
4880 ("MSI-X index too high"));
4881 if (dinfo->cfg.msi.msi_handlers == 0)
4882 return (EINVAL);
4883 dinfo->cfg.msi.msi_handlers--;
4884 if (dinfo->cfg.msi.msi_handlers == 0)
4885 pci_disable_msi(child);
4886 } else {
4887 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4888 ("No MSI or MSI-X interrupts allocated"));
4889 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4890 ("MSI-X index too high"));
4891 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4892 if (mte->mte_handlers == 0)
4893 return (EINVAL);
4894 mte->mte_handlers--;
4895 if (mte->mte_handlers == 0)
4896 pci_mask_msix(child, rid - 1);
4897 }
4898 }
4899 error = bus_generic_teardown_intr(dev, child, irq, cookie);
4900 if (rid > 0)
4901 KASSERT(error == 0,
4902 ("%s: generic teardown failed for MSI/MSI-X", __func__));
4903 return (error);
4904 }
4905
4906 int
pci_print_child(device_t dev,device_t child)4907 pci_print_child(device_t dev, device_t child)
4908 {
4909 struct pci_devinfo *dinfo;
4910 struct resource_list *rl;
4911 int retval = 0;
4912
4913 dinfo = device_get_ivars(child);
4914 rl = &dinfo->resources;
4915
4916 retval += bus_print_child_header(dev, child);
4917
4918 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
4919 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
4920 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
4921 if (device_get_flags(dev))
4922 retval += printf(" flags %#x", device_get_flags(dev));
4923
4924 retval += printf(" at device %d.%d", pci_get_slot(child),
4925 pci_get_function(child));
4926
4927 retval += bus_print_child_domain(dev, child);
4928 retval += bus_print_child_footer(dev, child);
4929
4930 return (retval);
4931 }
4932
4933 static const struct
4934 {
4935 int class;
4936 int subclass;
4937 int report; /* 0 = bootverbose, 1 = always */
4938 const char *desc;
4939 } pci_nomatch_tab[] = {
4940 {PCIC_OLD, -1, 1, "old"},
4941 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
4942 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
4943 {PCIC_STORAGE, -1, 1, "mass storage"},
4944 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
4945 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
4946 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
4947 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
4948 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
4949 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
4950 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
4951 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
4952 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
4953 {PCIC_NETWORK, -1, 1, "network"},
4954 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
4955 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
4956 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
4957 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
4958 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
4959 {PCIC_DISPLAY, -1, 1, "display"},
4960 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
4961 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
4962 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
4963 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
4964 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
4965 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
4966 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
4967 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
4968 {PCIC_MEMORY, -1, 1, "memory"},
4969 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
4970 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
4971 {PCIC_BRIDGE, -1, 1, "bridge"},
4972 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
4973 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
4974 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
4975 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
4976 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
4977 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
4978 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
4979 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
4980 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
4981 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
4982 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
4983 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
4984 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
4985 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
4986 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
4987 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
4988 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
4989 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
4990 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
4991 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
4992 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
4993 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
4994 {PCIC_INPUTDEV, -1, 1, "input device"},
4995 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
4996 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
4997 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
4998 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
4999 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5000 {PCIC_DOCKING, -1, 1, "docking station"},
5001 {PCIC_PROCESSOR, -1, 1, "processor"},
5002 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5003 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5004 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5005 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5006 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5007 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5008 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5009 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5010 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5011 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5012 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5013 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5014 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5015 {PCIC_SATCOM, -1, 1, "satellite communication"},
5016 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5017 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5018 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5019 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5020 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5021 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5022 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5023 {PCIC_DASP, -1, 0, "dasp"},
5024 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5025 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5026 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5027 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5028 {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"},
5029 {0, 0, 0, NULL}
5030 };
5031
5032 void
pci_probe_nomatch(device_t dev,device_t child)5033 pci_probe_nomatch(device_t dev, device_t child)
5034 {
5035 int i, report;
5036 const char *cp, *scp;
5037 char *device;
5038
5039 /*
5040 * Look for a listing for this device in a loaded device database.
5041 */
5042 report = 1;
5043 if ((device = pci_describe_device(child)) != NULL) {
5044 device_printf(dev, "<%s>", device);
5045 free(device, M_DEVBUF);
5046 } else {
5047 /*
5048 * Scan the class/subclass descriptions for a general
5049 * description.
5050 */
5051 cp = "unknown";
5052 scp = NULL;
5053 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5054 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5055 if (pci_nomatch_tab[i].subclass == -1) {
5056 cp = pci_nomatch_tab[i].desc;
5057 report = pci_nomatch_tab[i].report;
5058 } else if (pci_nomatch_tab[i].subclass ==
5059 pci_get_subclass(child)) {
5060 scp = pci_nomatch_tab[i].desc;
5061 report = pci_nomatch_tab[i].report;
5062 }
5063 }
5064 }
5065 if (report || bootverbose) {
5066 device_printf(dev, "<%s%s%s>",
5067 cp ? cp : "",
5068 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5069 scp ? scp : "");
5070 }
5071 }
5072 if (report || bootverbose) {
5073 printf(" at device %d.%d (no driver attached)\n",
5074 pci_get_slot(child), pci_get_function(child));
5075 }
5076 pci_cfg_save(child, device_get_ivars(child), 1);
5077 }
5078
5079 void
pci_child_detached(device_t dev,device_t child)5080 pci_child_detached(device_t dev, device_t child)
5081 {
5082 struct pci_devinfo *dinfo;
5083 struct resource_list *rl;
5084
5085 dinfo = device_get_ivars(child);
5086 rl = &dinfo->resources;
5087
5088 /*
5089 * Have to deallocate IRQs before releasing any MSI messages and
5090 * have to release MSI messages before deallocating any memory
5091 * BARs.
5092 */
5093 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5094 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5095 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5096 if (dinfo->cfg.msi.msi_alloc != 0)
5097 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5098 "vectors\n", dinfo->cfg.msi.msi_alloc);
5099 else
5100 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5101 "vectors\n", dinfo->cfg.msix.msix_alloc);
5102 (void)pci_release_msi(child);
5103 }
5104 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5105 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5106 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5107 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5108 #ifdef PCI_RES_BUS
5109 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5110 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5111 #endif
5112
5113 pci_cfg_save(child, dinfo, 1);
5114 }
5115
5116 /*
5117 * Parse the PCI device database, if loaded, and return a pointer to a
5118 * description of the device.
5119 *
5120 * The database is flat text formatted as follows:
5121 *
5122 * Any line not in a valid format is ignored.
5123 * Lines are terminated with newline '\n' characters.
5124 *
5125 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5126 * the vendor name.
5127 *
5128 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5129 * - devices cannot be listed without a corresponding VENDOR line.
5130 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5131 * another TAB, then the device name.
5132 */
5133
5134 /*
5135 * Assuming (ptr) points to the beginning of a line in the database,
5136 * return the vendor or device and description of the next entry.
5137 * The value of (vendor) or (device) inappropriate for the entry type
5138 * is set to -1. Returns nonzero at the end of the database.
5139 *
5140 * Note that this is slightly unrobust in the face of corrupt data;
5141 * we attempt to safeguard against this by spamming the end of the
5142 * database with a newline when we initialise.
5143 */
5144 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5145 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5146 {
5147 char *cp = *ptr;
5148 int left;
5149
5150 *device = -1;
5151 *vendor = -1;
5152 **desc = '\0';
5153 for (;;) {
5154 left = pci_vendordata_size - (cp - pci_vendordata);
5155 if (left <= 0) {
5156 *ptr = cp;
5157 return(1);
5158 }
5159
5160 /* vendor entry? */
5161 if (*cp != '\t' &&
5162 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5163 break;
5164 /* device entry? */
5165 if (*cp == '\t' &&
5166 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5167 break;
5168
5169 /* skip to next line */
5170 while (*cp != '\n' && left > 0) {
5171 cp++;
5172 left--;
5173 }
5174 if (*cp == '\n') {
5175 cp++;
5176 left--;
5177 }
5178 }
5179 /* skip to next line */
5180 while (*cp != '\n' && left > 0) {
5181 cp++;
5182 left--;
5183 }
5184 if (*cp == '\n' && left > 0)
5185 cp++;
5186 *ptr = cp;
5187 return(0);
5188 }
5189
5190 static char *
pci_describe_device(device_t dev)5191 pci_describe_device(device_t dev)
5192 {
5193 int vendor, device;
5194 char *desc, *vp, *dp, *line;
5195
5196 desc = vp = dp = NULL;
5197
5198 /*
5199 * If we have no vendor data, we can't do anything.
5200 */
5201 if (pci_vendordata == NULL)
5202 goto out;
5203
5204 /*
5205 * Scan the vendor data looking for this device
5206 */
5207 line = pci_vendordata;
5208 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5209 goto out;
5210 for (;;) {
5211 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5212 goto out;
5213 if (vendor == pci_get_vendor(dev))
5214 break;
5215 }
5216 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5217 goto out;
5218 for (;;) {
5219 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5220 *dp = 0;
5221 break;
5222 }
5223 if (vendor != -1) {
5224 *dp = 0;
5225 break;
5226 }
5227 if (device == pci_get_device(dev))
5228 break;
5229 }
5230 if (dp[0] == '\0')
5231 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5232 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5233 NULL)
5234 sprintf(desc, "%s, %s", vp, dp);
5235 out:
5236 if (vp != NULL)
5237 free(vp, M_DEVBUF);
5238 if (dp != NULL)
5239 free(dp, M_DEVBUF);
5240 return(desc);
5241 }
5242
5243 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5244 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5245 {
5246 struct pci_devinfo *dinfo;
5247 pcicfgregs *cfg;
5248
5249 dinfo = device_get_ivars(child);
5250 cfg = &dinfo->cfg;
5251
5252 switch (which) {
5253 case PCI_IVAR_ETHADDR:
5254 /*
5255 * The generic accessor doesn't deal with failure, so
5256 * we set the return value, then return an error.
5257 */
5258 *((uint8_t **) result) = NULL;
5259 return (EINVAL);
5260 case PCI_IVAR_SUBVENDOR:
5261 *result = cfg->subvendor;
5262 break;
5263 case PCI_IVAR_SUBDEVICE:
5264 *result = cfg->subdevice;
5265 break;
5266 case PCI_IVAR_VENDOR:
5267 *result = cfg->vendor;
5268 break;
5269 case PCI_IVAR_DEVICE:
5270 *result = cfg->device;
5271 break;
5272 case PCI_IVAR_DEVID:
5273 *result = (cfg->device << 16) | cfg->vendor;
5274 break;
5275 case PCI_IVAR_CLASS:
5276 *result = cfg->baseclass;
5277 break;
5278 case PCI_IVAR_SUBCLASS:
5279 *result = cfg->subclass;
5280 break;
5281 case PCI_IVAR_PROGIF:
5282 *result = cfg->progif;
5283 break;
5284 case PCI_IVAR_REVID:
5285 *result = cfg->revid;
5286 break;
5287 case PCI_IVAR_INTPIN:
5288 *result = cfg->intpin;
5289 break;
5290 case PCI_IVAR_IRQ:
5291 *result = cfg->intline;
5292 break;
5293 case PCI_IVAR_DOMAIN:
5294 *result = cfg->domain;
5295 break;
5296 case PCI_IVAR_BUS:
5297 *result = cfg->bus;
5298 break;
5299 case PCI_IVAR_SLOT:
5300 *result = cfg->slot;
5301 break;
5302 case PCI_IVAR_FUNCTION:
5303 *result = cfg->func;
5304 break;
5305 case PCI_IVAR_CMDREG:
5306 *result = cfg->cmdreg;
5307 break;
5308 case PCI_IVAR_CACHELNSZ:
5309 *result = cfg->cachelnsz;
5310 break;
5311 case PCI_IVAR_MINGNT:
5312 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5313 *result = -1;
5314 return (EINVAL);
5315 }
5316 *result = cfg->mingnt;
5317 break;
5318 case PCI_IVAR_MAXLAT:
5319 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5320 *result = -1;
5321 return (EINVAL);
5322 }
5323 *result = cfg->maxlat;
5324 break;
5325 case PCI_IVAR_LATTIMER:
5326 *result = cfg->lattimer;
5327 break;
5328 default:
5329 return (ENOENT);
5330 }
5331 return (0);
5332 }
5333
5334 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5335 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5336 {
5337 struct pci_devinfo *dinfo;
5338
5339 dinfo = device_get_ivars(child);
5340
5341 switch (which) {
5342 case PCI_IVAR_INTPIN:
5343 dinfo->cfg.intpin = value;
5344 return (0);
5345 case PCI_IVAR_ETHADDR:
5346 case PCI_IVAR_SUBVENDOR:
5347 case PCI_IVAR_SUBDEVICE:
5348 case PCI_IVAR_VENDOR:
5349 case PCI_IVAR_DEVICE:
5350 case PCI_IVAR_DEVID:
5351 case PCI_IVAR_CLASS:
5352 case PCI_IVAR_SUBCLASS:
5353 case PCI_IVAR_PROGIF:
5354 case PCI_IVAR_REVID:
5355 case PCI_IVAR_IRQ:
5356 case PCI_IVAR_DOMAIN:
5357 case PCI_IVAR_BUS:
5358 case PCI_IVAR_SLOT:
5359 case PCI_IVAR_FUNCTION:
5360 return (EINVAL); /* disallow for now */
5361
5362 default:
5363 return (ENOENT);
5364 }
5365 }
5366
5367 #include "opt_ddb.h"
5368 #ifdef DDB
5369 #include <ddb/ddb.h>
5370 #include <sys/cons.h>
5371
5372 /*
5373 * List resources based on pci map registers, used for within ddb
5374 */
5375
DB_SHOW_COMMAND_FLAGS(pciregs,db_pci_dump,DB_CMD_MEMSAFE)5376 DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE)
5377 {
5378 struct pci_devinfo *dinfo;
5379 struct devlist *devlist_head;
5380 struct pci_conf *p;
5381 const char *name;
5382 int i, error, none_count;
5383
5384 none_count = 0;
5385 /* get the head of the device queue */
5386 devlist_head = &pci_devq;
5387
5388 /*
5389 * Go through the list of devices and print out devices
5390 */
5391 for (error = 0, i = 0,
5392 dinfo = STAILQ_FIRST(devlist_head);
5393 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5394 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5395 /* Populate pd_name and pd_unit */
5396 name = NULL;
5397 if (dinfo->cfg.dev)
5398 name = device_get_name(dinfo->cfg.dev);
5399
5400 p = &dinfo->conf;
5401 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5402 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5403 (name && *name) ? name : "none",
5404 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5405 none_count++,
5406 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5407 p->pc_sel.pc_func, (p->pc_class << 16) |
5408 (p->pc_subclass << 8) | p->pc_progif,
5409 (p->pc_subdevice << 16) | p->pc_subvendor,
5410 (p->pc_device << 16) | p->pc_vendor,
5411 p->pc_revid, p->pc_hdr);
5412 }
5413 }
5414 #endif /* DDB */
5415
5416 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5417 pci_reserve_map(device_t dev, device_t child, int type, int *rid,
5418 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5419 u_int flags)
5420 {
5421 struct pci_devinfo *dinfo = device_get_ivars(child);
5422 struct resource_list *rl = &dinfo->resources;
5423 struct resource *res;
5424 struct pci_map *pm;
5425 uint16_t cmd;
5426 pci_addr_t map, testval;
5427 int mapsize;
5428
5429 res = NULL;
5430
5431 /* If rid is managed by EA, ignore it */
5432 if (pci_ea_is_enabled(child, *rid))
5433 goto out;
5434
5435 pm = pci_find_bar(child, *rid);
5436 if (pm != NULL) {
5437 /* This is a BAR that we failed to allocate earlier. */
5438 mapsize = pm->pm_size;
5439 map = pm->pm_value;
5440 } else {
5441 /*
5442 * Weed out the bogons, and figure out how large the
5443 * BAR/map is. BARs that read back 0 here are bogus
5444 * and unimplemented. Note: atapci in legacy mode are
5445 * special and handled elsewhere in the code. If you
5446 * have a atapci device in legacy mode and it fails
5447 * here, that other code is broken.
5448 */
5449 pci_read_bar(child, *rid, &map, &testval, NULL);
5450
5451 /*
5452 * Determine the size of the BAR and ignore BARs with a size
5453 * of 0. Device ROM BARs use a different mask value.
5454 */
5455 if (PCIR_IS_BIOS(&dinfo->cfg, *rid))
5456 mapsize = pci_romsize(testval);
5457 else
5458 mapsize = pci_mapsize(testval);
5459 if (mapsize == 0)
5460 goto out;
5461 pm = pci_add_bar(child, *rid, map, mapsize);
5462 }
5463
5464 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) {
5465 if (type != SYS_RES_MEMORY) {
5466 if (bootverbose)
5467 device_printf(dev,
5468 "child %s requested type %d for rid %#x,"
5469 " but the BAR says it is an memio\n",
5470 device_get_nameunit(child), type, *rid);
5471 goto out;
5472 }
5473 } else {
5474 if (type != SYS_RES_IOPORT) {
5475 if (bootverbose)
5476 device_printf(dev,
5477 "child %s requested type %d for rid %#x,"
5478 " but the BAR says it is an ioport\n",
5479 device_get_nameunit(child), type, *rid);
5480 goto out;
5481 }
5482 }
5483
5484 /*
5485 * For real BARs, we need to override the size that
5486 * the driver requests, because that's what the BAR
5487 * actually uses and we would otherwise have a
5488 * situation where we might allocate the excess to
5489 * another driver, which won't work.
5490 */
5491 count = ((pci_addr_t)1 << mapsize) * num;
5492 if (RF_ALIGNMENT(flags) < mapsize)
5493 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5494 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5495 flags |= RF_PREFETCHABLE;
5496
5497 /*
5498 * Allocate enough resource, and then write back the
5499 * appropriate BAR for that resource.
5500 */
5501 resource_list_add(rl, type, *rid, start, end, count);
5502 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5503 count, flags & ~RF_ACTIVE);
5504 if (res == NULL) {
5505 resource_list_delete(rl, type, *rid);
5506 device_printf(child,
5507 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5508 count, *rid, type, start, end);
5509 goto out;
5510 }
5511 if (bootverbose)
5512 device_printf(child,
5513 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5514 count, *rid, type, rman_get_start(res));
5515
5516 /* Disable decoding via the CMD register before updating the BAR */
5517 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5518 pci_write_config(child, PCIR_COMMAND,
5519 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5520
5521 map = rman_get_start(res);
5522 pci_write_bar(child, pm, map);
5523
5524 /* Restore the original value of the CMD register */
5525 pci_write_config(child, PCIR_COMMAND, cmd, 2);
5526 out:
5527 return (res);
5528 }
5529
5530 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)5531 pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid,
5532 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
5533 u_int flags)
5534 {
5535 struct pci_devinfo *dinfo;
5536 struct resource_list *rl;
5537 struct resource_list_entry *rle;
5538 struct resource *res;
5539 pcicfgregs *cfg;
5540
5541 /*
5542 * Perform lazy resource allocation
5543 */
5544 dinfo = device_get_ivars(child);
5545 rl = &dinfo->resources;
5546 cfg = &dinfo->cfg;
5547 switch (type) {
5548 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
5549 case PCI_RES_BUS:
5550 return (pci_alloc_secbus(dev, child, rid, start, end, count,
5551 flags));
5552 #endif
5553 case SYS_RES_IRQ:
5554 /*
5555 * Can't alloc legacy interrupt once MSI messages have
5556 * been allocated.
5557 */
5558 if (*rid == 0 && (cfg->msi.msi_alloc > 0 ||
5559 cfg->msix.msix_alloc > 0))
5560 return (NULL);
5561
5562 /*
5563 * If the child device doesn't have an interrupt
5564 * routed and is deserving of an interrupt, try to
5565 * assign it one.
5566 */
5567 if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
5568 (cfg->intpin != 0))
5569 pci_assign_interrupt(dev, child, 0);
5570 break;
5571 case SYS_RES_IOPORT:
5572 case SYS_RES_MEMORY:
5573 #ifdef NEW_PCIB
5574 /*
5575 * PCI-PCI bridge I/O window resources are not BARs.
5576 * For those allocations just pass the request up the
5577 * tree.
5578 */
5579 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
5580 switch (*rid) {
5581 case PCIR_IOBASEL_1:
5582 case PCIR_MEMBASE_1:
5583 case PCIR_PMBASEL_1:
5584 /*
5585 * XXX: Should we bother creating a resource
5586 * list entry?
5587 */
5588 return (bus_generic_alloc_resource(dev, child,
5589 type, rid, start, end, count, flags));
5590 }
5591 }
5592 #endif
5593 /* Reserve resources for this BAR if needed. */
5594 rle = resource_list_find(rl, type, *rid);
5595 if (rle == NULL) {
5596 res = pci_reserve_map(dev, child, type, rid, start, end,
5597 count, num, flags);
5598 if (res == NULL)
5599 return (NULL);
5600 }
5601 }
5602 return (resource_list_alloc(rl, dev, child, type, rid,
5603 start, end, count, flags));
5604 }
5605
5606 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)5607 pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
5608 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
5609 {
5610 #ifdef PCI_IOV
5611 struct pci_devinfo *dinfo;
5612 #endif
5613
5614 if (device_get_parent(child) != dev)
5615 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
5616 type, rid, start, end, count, flags));
5617
5618 #ifdef PCI_IOV
5619 dinfo = device_get_ivars(child);
5620 if (dinfo->cfg.flags & PCICFG_VF) {
5621 switch (type) {
5622 /* VFs can't have I/O BARs. */
5623 case SYS_RES_IOPORT:
5624 return (NULL);
5625 case SYS_RES_MEMORY:
5626 return (pci_vf_alloc_mem_resource(dev, child, rid,
5627 start, end, count, flags));
5628 }
5629
5630 /* Fall through for other types of resource allocations. */
5631 }
5632 #endif
5633
5634 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
5635 count, 1, flags));
5636 }
5637
5638 int
pci_release_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5639 pci_release_resource(device_t dev, device_t child, int type, int rid,
5640 struct resource *r)
5641 {
5642 struct pci_devinfo *dinfo;
5643 struct resource_list *rl;
5644 pcicfgregs *cfg __unused;
5645
5646 if (device_get_parent(child) != dev)
5647 return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child,
5648 type, rid, r));
5649
5650 dinfo = device_get_ivars(child);
5651 cfg = &dinfo->cfg;
5652
5653 #ifdef PCI_IOV
5654 if (cfg->flags & PCICFG_VF) {
5655 switch (type) {
5656 /* VFs can't have I/O BARs. */
5657 case SYS_RES_IOPORT:
5658 return (EDOOFUS);
5659 case SYS_RES_MEMORY:
5660 return (pci_vf_release_mem_resource(dev, child, rid,
5661 r));
5662 }
5663
5664 /* Fall through for other types of resource allocations. */
5665 }
5666 #endif
5667
5668 #ifdef NEW_PCIB
5669 /*
5670 * PCI-PCI bridge I/O window resources are not BARs. For
5671 * those allocations just pass the request up the tree.
5672 */
5673 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
5674 (type == SYS_RES_IOPORT || type == SYS_RES_MEMORY)) {
5675 switch (rid) {
5676 case PCIR_IOBASEL_1:
5677 case PCIR_MEMBASE_1:
5678 case PCIR_PMBASEL_1:
5679 return (bus_generic_release_resource(dev, child, type,
5680 rid, r));
5681 }
5682 }
5683 #endif
5684
5685 rl = &dinfo->resources;
5686 return (resource_list_release(rl, dev, child, type, rid, r));
5687 }
5688
5689 int
pci_activate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5690 pci_activate_resource(device_t dev, device_t child, int type, int rid,
5691 struct resource *r)
5692 {
5693 struct pci_devinfo *dinfo;
5694 int error;
5695
5696 error = bus_generic_activate_resource(dev, child, type, rid, r);
5697 if (error)
5698 return (error);
5699
5700 /* Enable decoding in the command register when activating BARs. */
5701 if (device_get_parent(child) == dev) {
5702 /* Device ROMs need their decoding explicitly enabled. */
5703 dinfo = device_get_ivars(child);
5704 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5705 pci_write_bar(child, pci_find_bar(child, rid),
5706 rman_get_start(r) | PCIM_BIOS_ENABLE);
5707 switch (type) {
5708 case SYS_RES_IOPORT:
5709 case SYS_RES_MEMORY:
5710 error = PCI_ENABLE_IO(dev, child, type);
5711 break;
5712 }
5713 }
5714 return (error);
5715 }
5716
5717 int
pci_deactivate_resource(device_t dev,device_t child,int type,int rid,struct resource * r)5718 pci_deactivate_resource(device_t dev, device_t child, int type,
5719 int rid, struct resource *r)
5720 {
5721 struct pci_devinfo *dinfo;
5722 int error;
5723
5724 error = bus_generic_deactivate_resource(dev, child, type, rid, r);
5725 if (error)
5726 return (error);
5727
5728 /* Disable decoding for device ROMs. */
5729 if (device_get_parent(child) == dev) {
5730 dinfo = device_get_ivars(child);
5731 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5732 pci_write_bar(child, pci_find_bar(child, rid),
5733 rman_get_start(r));
5734 }
5735 return (0);
5736 }
5737
5738 void
pci_child_deleted(device_t dev,device_t child)5739 pci_child_deleted(device_t dev, device_t child)
5740 {
5741 struct resource_list_entry *rle;
5742 struct resource_list *rl;
5743 struct pci_devinfo *dinfo;
5744
5745 dinfo = device_get_ivars(child);
5746 rl = &dinfo->resources;
5747
5748 EVENTHANDLER_INVOKE(pci_delete_device, child);
5749
5750 /* Turn off access to resources we're about to free */
5751 if (bus_child_present(child) != 0) {
5752 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
5753 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
5754
5755 pci_disable_busmaster(child);
5756 }
5757
5758 /* Free all allocated resources */
5759 STAILQ_FOREACH(rle, rl, link) {
5760 if (rle->res) {
5761 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5762 resource_list_busy(rl, rle->type, rle->rid)) {
5763 pci_printf(&dinfo->cfg,
5764 "Resource still owned, oops. "
5765 "(type=%d, rid=%d, addr=%lx)\n",
5766 rle->type, rle->rid,
5767 rman_get_start(rle->res));
5768 bus_release_resource(child, rle->type, rle->rid,
5769 rle->res);
5770 }
5771 resource_list_unreserve(rl, dev, child, rle->type,
5772 rle->rid);
5773 }
5774 }
5775 resource_list_free(rl);
5776
5777 pci_freecfg(dinfo);
5778 }
5779
5780 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)5781 pci_delete_resource(device_t dev, device_t child, int type, int rid)
5782 {
5783 struct pci_devinfo *dinfo;
5784 struct resource_list *rl;
5785 struct resource_list_entry *rle;
5786
5787 if (device_get_parent(child) != dev)
5788 return;
5789
5790 dinfo = device_get_ivars(child);
5791 rl = &dinfo->resources;
5792 rle = resource_list_find(rl, type, rid);
5793 if (rle == NULL)
5794 return;
5795
5796 if (rle->res) {
5797 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5798 resource_list_busy(rl, type, rid)) {
5799 device_printf(dev, "delete_resource: "
5800 "Resource still owned by child, oops. "
5801 "(type=%d, rid=%d, addr=%jx)\n",
5802 type, rid, rman_get_start(rle->res));
5803 return;
5804 }
5805 resource_list_unreserve(rl, dev, child, type, rid);
5806 }
5807 resource_list_delete(rl, type, rid);
5808 }
5809
5810 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)5811 pci_get_resource_list (device_t dev, device_t child)
5812 {
5813 struct pci_devinfo *dinfo = device_get_ivars(child);
5814
5815 return (&dinfo->resources);
5816 }
5817
5818 #ifdef IOMMU
5819 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5820 pci_get_dma_tag(device_t bus, device_t dev)
5821 {
5822 bus_dma_tag_t tag;
5823 struct pci_softc *sc;
5824
5825 if (device_get_parent(dev) == bus) {
5826 /* try iommu and return if it works */
5827 tag = iommu_get_dma_tag(bus, dev);
5828 } else
5829 tag = NULL;
5830 if (tag == NULL) {
5831 sc = device_get_softc(bus);
5832 tag = sc->sc_dma_tag;
5833 }
5834 return (tag);
5835 }
5836 #else
5837 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5838 pci_get_dma_tag(device_t bus, device_t dev)
5839 {
5840 struct pci_softc *sc = device_get_softc(bus);
5841
5842 return (sc->sc_dma_tag);
5843 }
5844 #endif
5845
5846 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)5847 pci_read_config_method(device_t dev, device_t child, int reg, int width)
5848 {
5849 struct pci_devinfo *dinfo = device_get_ivars(child);
5850 pcicfgregs *cfg = &dinfo->cfg;
5851
5852 #ifdef PCI_IOV
5853 /*
5854 * SR-IOV VFs don't implement the VID or DID registers, so we have to
5855 * emulate them here.
5856 */
5857 if (cfg->flags & PCICFG_VF) {
5858 if (reg == PCIR_VENDOR) {
5859 switch (width) {
5860 case 4:
5861 return (cfg->device << 16 | cfg->vendor);
5862 case 2:
5863 return (cfg->vendor);
5864 case 1:
5865 return (cfg->vendor & 0xff);
5866 default:
5867 return (0xffffffff);
5868 }
5869 } else if (reg == PCIR_DEVICE) {
5870 switch (width) {
5871 /* Note that an unaligned 4-byte read is an error. */
5872 case 2:
5873 return (cfg->device);
5874 case 1:
5875 return (cfg->device & 0xff);
5876 default:
5877 return (0xffffffff);
5878 }
5879 }
5880 }
5881 #endif
5882
5883 return (PCIB_READ_CONFIG(device_get_parent(dev),
5884 cfg->bus, cfg->slot, cfg->func, reg, width));
5885 }
5886
5887 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)5888 pci_write_config_method(device_t dev, device_t child, int reg,
5889 uint32_t val, int width)
5890 {
5891 struct pci_devinfo *dinfo = device_get_ivars(child);
5892 pcicfgregs *cfg = &dinfo->cfg;
5893
5894 PCIB_WRITE_CONFIG(device_get_parent(dev),
5895 cfg->bus, cfg->slot, cfg->func, reg, val, width);
5896 }
5897
5898 int
pci_child_location_method(device_t dev,device_t child,struct sbuf * sb)5899 pci_child_location_method(device_t dev, device_t child, struct sbuf *sb)
5900 {
5901
5902 sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
5903 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
5904 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
5905 return (0);
5906 }
5907
5908 int
pci_child_pnpinfo_method(device_t dev,device_t child,struct sbuf * sb)5909 pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb)
5910 {
5911 struct pci_devinfo *dinfo;
5912 pcicfgregs *cfg;
5913
5914 dinfo = device_get_ivars(child);
5915 cfg = &dinfo->cfg;
5916 sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
5917 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
5918 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
5919 cfg->progif);
5920 return (0);
5921 }
5922
5923 int
pci_get_device_path_method(device_t bus,device_t child,const char * locator,struct sbuf * sb)5924 pci_get_device_path_method(device_t bus, device_t child, const char *locator,
5925 struct sbuf *sb)
5926 {
5927 device_t parent = device_get_parent(bus);
5928 int rv;
5929
5930 if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) {
5931 rv = bus_generic_get_device_path(parent, bus, locator, sb);
5932 if (rv == 0) {
5933 sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child),
5934 pci_get_function(child));
5935 }
5936 return (0);
5937 }
5938 return (bus_generic_get_device_path(bus, child, locator, sb));
5939 }
5940
5941 int
pci_assign_interrupt_method(device_t dev,device_t child)5942 pci_assign_interrupt_method(device_t dev, device_t child)
5943 {
5944 struct pci_devinfo *dinfo = device_get_ivars(child);
5945 pcicfgregs *cfg = &dinfo->cfg;
5946
5947 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
5948 cfg->intpin));
5949 }
5950
5951 static void
pci_lookup(void * arg,const char * name,device_t * dev)5952 pci_lookup(void *arg, const char *name, device_t *dev)
5953 {
5954 long val;
5955 char *end;
5956 int domain, bus, slot, func;
5957
5958 if (*dev != NULL)
5959 return;
5960
5961 /*
5962 * Accept pciconf-style selectors of either pciD:B:S:F or
5963 * pciB:S:F. In the latter case, the domain is assumed to
5964 * be zero.
5965 */
5966 if (strncmp(name, "pci", 3) != 0)
5967 return;
5968 val = strtol(name + 3, &end, 10);
5969 if (val < 0 || val > INT_MAX || *end != ':')
5970 return;
5971 domain = val;
5972 val = strtol(end + 1, &end, 10);
5973 if (val < 0 || val > INT_MAX || *end != ':')
5974 return;
5975 bus = val;
5976 val = strtol(end + 1, &end, 10);
5977 if (val < 0 || val > INT_MAX)
5978 return;
5979 slot = val;
5980 if (*end == ':') {
5981 val = strtol(end + 1, &end, 10);
5982 if (val < 0 || val > INT_MAX || *end != '\0')
5983 return;
5984 func = val;
5985 } else if (*end == '\0') {
5986 func = slot;
5987 slot = bus;
5988 bus = domain;
5989 domain = 0;
5990 } else
5991 return;
5992
5993 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
5994 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
5995 return;
5996
5997 *dev = pci_find_dbsf(domain, bus, slot, func);
5998 }
5999
6000 static int
pci_modevent(module_t mod,int what,void * arg)6001 pci_modevent(module_t mod, int what, void *arg)
6002 {
6003 static struct cdev *pci_cdev;
6004 static eventhandler_tag tag;
6005
6006 switch (what) {
6007 case MOD_LOAD:
6008 STAILQ_INIT(&pci_devq);
6009 pci_generation = 0;
6010 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6011 "pci");
6012 pci_load_vendor_data();
6013 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6014 1000);
6015 break;
6016
6017 case MOD_UNLOAD:
6018 if (tag != NULL)
6019 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6020 destroy_dev(pci_cdev);
6021 break;
6022 }
6023
6024 return (0);
6025 }
6026
6027 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6028 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6029 {
6030 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6031 struct pcicfg_pcie *cfg;
6032 int version, pos;
6033
6034 cfg = &dinfo->cfg.pcie;
6035 pos = cfg->pcie_location;
6036
6037 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6038
6039 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6040
6041 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6042 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6043 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6044 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6045
6046 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6047 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6048 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6049 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6050
6051 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6052 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6053 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6054
6055 if (version > 1) {
6056 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6057 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6058 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6059 }
6060 #undef WREG
6061 }
6062
6063 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6064 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6065 {
6066 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6067 dinfo->cfg.pcix.pcix_command, 2);
6068 }
6069
6070 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6071 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6072 {
6073
6074 /*
6075 * Restore the device to full power mode. We must do this
6076 * before we restore the registers because moving from D3 to
6077 * D0 will cause the chip's BARs and some other registers to
6078 * be reset to some unknown power on reset values. Cut down
6079 * the noise on boot by doing nothing if we are already in
6080 * state D0.
6081 */
6082 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6083 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6084 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6085 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6086 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6087 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6088 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6089 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6090 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6091 case PCIM_HDRTYPE_NORMAL:
6092 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6093 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6094 break;
6095 case PCIM_HDRTYPE_BRIDGE:
6096 pci_write_config(dev, PCIR_SECLAT_1,
6097 dinfo->cfg.bridge.br_seclat, 1);
6098 pci_write_config(dev, PCIR_SUBBUS_1,
6099 dinfo->cfg.bridge.br_subbus, 1);
6100 pci_write_config(dev, PCIR_SECBUS_1,
6101 dinfo->cfg.bridge.br_secbus, 1);
6102 pci_write_config(dev, PCIR_PRIBUS_1,
6103 dinfo->cfg.bridge.br_pribus, 1);
6104 pci_write_config(dev, PCIR_BRIDGECTL_1,
6105 dinfo->cfg.bridge.br_control, 2);
6106 break;
6107 case PCIM_HDRTYPE_CARDBUS:
6108 pci_write_config(dev, PCIR_SECLAT_2,
6109 dinfo->cfg.bridge.br_seclat, 1);
6110 pci_write_config(dev, PCIR_SUBBUS_2,
6111 dinfo->cfg.bridge.br_subbus, 1);
6112 pci_write_config(dev, PCIR_SECBUS_2,
6113 dinfo->cfg.bridge.br_secbus, 1);
6114 pci_write_config(dev, PCIR_PRIBUS_2,
6115 dinfo->cfg.bridge.br_pribus, 1);
6116 pci_write_config(dev, PCIR_BRIDGECTL_2,
6117 dinfo->cfg.bridge.br_control, 2);
6118 break;
6119 }
6120 pci_restore_bars(dev);
6121
6122 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6123 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6124
6125 /*
6126 * Restore extended capabilities for PCI-Express and PCI-X
6127 */
6128 if (dinfo->cfg.pcie.pcie_location != 0)
6129 pci_cfg_restore_pcie(dev, dinfo);
6130 if (dinfo->cfg.pcix.pcix_location != 0)
6131 pci_cfg_restore_pcix(dev, dinfo);
6132
6133 /* Restore MSI and MSI-X configurations if they are present. */
6134 if (dinfo->cfg.msi.msi_location != 0)
6135 pci_resume_msi(dev);
6136 if (dinfo->cfg.msix.msix_location != 0)
6137 pci_resume_msix(dev);
6138
6139 #ifdef PCI_IOV
6140 if (dinfo->cfg.iov != NULL)
6141 pci_iov_cfg_restore(dev, dinfo);
6142 #endif
6143 }
6144
6145 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6146 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6147 {
6148 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6149 struct pcicfg_pcie *cfg;
6150 int version, pos;
6151
6152 cfg = &dinfo->cfg.pcie;
6153 pos = cfg->pcie_location;
6154
6155 cfg->pcie_flags = RREG(PCIER_FLAGS);
6156
6157 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6158
6159 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6160
6161 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6162 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6163 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6164 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6165
6166 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6167 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6168 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6169 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6170
6171 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6172 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6173 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6174
6175 if (version > 1) {
6176 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6177 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6178 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6179 }
6180 #undef RREG
6181 }
6182
6183 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6184 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6185 {
6186 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6187 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6188 }
6189
6190 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6191 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6192 {
6193 uint32_t cls;
6194 int ps;
6195
6196 /*
6197 * Some drivers apparently write to these registers w/o updating our
6198 * cached copy. No harm happens if we update the copy, so do so here
6199 * so we can restore them. The COMMAND register is modified by the
6200 * bus w/o updating the cache. This should represent the normally
6201 * writable portion of the 'defined' part of type 0/1/2 headers.
6202 */
6203 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6204 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6205 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6206 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6207 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6208 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6209 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6210 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6211 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6212 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6213 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6214 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6215 case PCIM_HDRTYPE_NORMAL:
6216 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6217 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6218 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6219 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6220 break;
6221 case PCIM_HDRTYPE_BRIDGE:
6222 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6223 PCIR_SECLAT_1, 1);
6224 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6225 PCIR_SUBBUS_1, 1);
6226 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6227 PCIR_SECBUS_1, 1);
6228 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6229 PCIR_PRIBUS_1, 1);
6230 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6231 PCIR_BRIDGECTL_1, 2);
6232 break;
6233 case PCIM_HDRTYPE_CARDBUS:
6234 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6235 PCIR_SECLAT_2, 1);
6236 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6237 PCIR_SUBBUS_2, 1);
6238 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6239 PCIR_SECBUS_2, 1);
6240 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6241 PCIR_PRIBUS_2, 1);
6242 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6243 PCIR_BRIDGECTL_2, 2);
6244 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6245 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6246 break;
6247 }
6248
6249 if (dinfo->cfg.pcie.pcie_location != 0)
6250 pci_cfg_save_pcie(dev, dinfo);
6251
6252 if (dinfo->cfg.pcix.pcix_location != 0)
6253 pci_cfg_save_pcix(dev, dinfo);
6254
6255 #ifdef PCI_IOV
6256 if (dinfo->cfg.iov != NULL)
6257 pci_iov_cfg_save(dev, dinfo);
6258 #endif
6259
6260 /*
6261 * don't set the state for display devices, base peripherals and
6262 * memory devices since bad things happen when they are powered down.
6263 * We should (a) have drivers that can easily detach and (b) use
6264 * generic drivers for these devices so that some device actually
6265 * attaches. We need to make sure that when we implement (a) we don't
6266 * power the device down on a reattach.
6267 */
6268 cls = pci_get_class(dev);
6269 if (!setstate)
6270 return;
6271 switch (pci_do_power_nodriver)
6272 {
6273 case 0: /* NO powerdown at all */
6274 return;
6275 case 1: /* Conservative about what to power down */
6276 if (cls == PCIC_STORAGE)
6277 return;
6278 /*FALLTHROUGH*/
6279 case 2: /* Aggressive about what to power down */
6280 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6281 cls == PCIC_BASEPERIPH)
6282 return;
6283 /*FALLTHROUGH*/
6284 case 3: /* Power down everything */
6285 break;
6286 }
6287 /*
6288 * PCI spec says we can only go into D3 state from D0 state.
6289 * Transition from D[12] into D0 before going to D3 state.
6290 */
6291 ps = pci_get_powerstate(dev);
6292 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6293 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6294 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6295 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6296 }
6297
6298 /* Wrapper APIs suitable for device driver use. */
6299 void
pci_save_state(device_t dev)6300 pci_save_state(device_t dev)
6301 {
6302 struct pci_devinfo *dinfo;
6303
6304 dinfo = device_get_ivars(dev);
6305 pci_cfg_save(dev, dinfo, 0);
6306 }
6307
6308 void
pci_restore_state(device_t dev)6309 pci_restore_state(device_t dev)
6310 {
6311 struct pci_devinfo *dinfo;
6312
6313 dinfo = device_get_ivars(dev);
6314 pci_cfg_restore(dev, dinfo);
6315 }
6316
6317 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6318 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6319 uintptr_t *id)
6320 {
6321
6322 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6323 }
6324
6325 /* Find the upstream port of a given PCI device in a root complex. */
6326 device_t
pci_find_pcie_root_port(device_t dev)6327 pci_find_pcie_root_port(device_t dev)
6328 {
6329 struct pci_devinfo *dinfo;
6330 devclass_t pci_class;
6331 device_t pcib, bus;
6332
6333 pci_class = devclass_find("pci");
6334 KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class,
6335 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6336
6337 /*
6338 * Walk the bridge hierarchy until we find a PCI-e root
6339 * port or a non-PCI device.
6340 */
6341 for (;;) {
6342 bus = device_get_parent(dev);
6343 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6344 device_get_nameunit(dev)));
6345
6346 pcib = device_get_parent(bus);
6347 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6348 device_get_nameunit(bus)));
6349
6350 /*
6351 * pcib's parent must be a PCI bus for this to be a
6352 * PCI-PCI bridge.
6353 */
6354 if (device_get_devclass(device_get_parent(pcib)) != pci_class)
6355 return (NULL);
6356
6357 dinfo = device_get_ivars(pcib);
6358 if (dinfo->cfg.pcie.pcie_location != 0 &&
6359 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6360 return (pcib);
6361
6362 dev = pcib;
6363 }
6364 }
6365
6366 /*
6367 * Wait for pending transactions to complete on a PCI-express function.
6368 *
6369 * The maximum delay is specified in milliseconds in max_delay. Note
6370 * that this function may sleep.
6371 *
6372 * Returns true if the function is idle and false if the timeout is
6373 * exceeded. If dev is not a PCI-express function, this returns true.
6374 */
6375 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6376 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6377 {
6378 struct pci_devinfo *dinfo = device_get_ivars(dev);
6379 uint16_t sta;
6380 int cap;
6381
6382 cap = dinfo->cfg.pcie.pcie_location;
6383 if (cap == 0)
6384 return (true);
6385
6386 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6387 while (sta & PCIEM_STA_TRANSACTION_PND) {
6388 if (max_delay == 0)
6389 return (false);
6390
6391 /* Poll once every 100 milliseconds up to the timeout. */
6392 if (max_delay > 100) {
6393 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6394 max_delay -= 100;
6395 } else {
6396 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6397 C_HARDCLOCK);
6398 max_delay = 0;
6399 }
6400 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6401 }
6402
6403 return (true);
6404 }
6405
6406 /*
6407 * Determine the maximum Completion Timeout in microseconds.
6408 *
6409 * For non-PCI-express functions this returns 0.
6410 */
6411 int
pcie_get_max_completion_timeout(device_t dev)6412 pcie_get_max_completion_timeout(device_t dev)
6413 {
6414 struct pci_devinfo *dinfo = device_get_ivars(dev);
6415 int cap;
6416
6417 cap = dinfo->cfg.pcie.pcie_location;
6418 if (cap == 0)
6419 return (0);
6420
6421 /*
6422 * Functions using the 1.x spec use the default timeout range of
6423 * 50 microseconds to 50 milliseconds. Functions that do not
6424 * support programmable timeouts also use this range.
6425 */
6426 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
6427 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
6428 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
6429 return (50 * 1000);
6430
6431 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
6432 PCIEM_CTL2_COMP_TIMO_VAL) {
6433 case PCIEM_CTL2_COMP_TIMO_100US:
6434 return (100);
6435 case PCIEM_CTL2_COMP_TIMO_10MS:
6436 return (10 * 1000);
6437 case PCIEM_CTL2_COMP_TIMO_55MS:
6438 return (55 * 1000);
6439 case PCIEM_CTL2_COMP_TIMO_210MS:
6440 return (210 * 1000);
6441 case PCIEM_CTL2_COMP_TIMO_900MS:
6442 return (900 * 1000);
6443 case PCIEM_CTL2_COMP_TIMO_3500MS:
6444 return (3500 * 1000);
6445 case PCIEM_CTL2_COMP_TIMO_13S:
6446 return (13 * 1000 * 1000);
6447 case PCIEM_CTL2_COMP_TIMO_64S:
6448 return (64 * 1000 * 1000);
6449 default:
6450 return (50 * 1000);
6451 }
6452 }
6453
6454 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)6455 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
6456 {
6457 struct pci_devinfo *dinfo = device_get_ivars(dev);
6458 const char *s;
6459 int aer;
6460 uint32_t r, r1;
6461 uint16_t rs;
6462
6463 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
6464 s = "Correctable";
6465 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
6466 s = "Uncorrectable (Non-Fatal)";
6467 else
6468 s = "Uncorrectable (Fatal)";
6469 device_printf(dev, "%s PCIe error reported by APEI\n", s);
6470 if (aerp) {
6471 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
6472 r = le32dec(aerp + PCIR_AER_COR_STATUS);
6473 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
6474 } else {
6475 r = le32dec(aerp + PCIR_AER_UC_STATUS);
6476 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
6477 }
6478 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
6479 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
6480 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
6481 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
6482 printf(" severity 0x%08x first %d\n",
6483 r, rs & 0x1f);
6484 } else
6485 printf("\n");
6486 }
6487
6488 /* As kind of recovery just report and clear the error statuses. */
6489 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6490 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6491 if (r != 0) {
6492 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6493 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
6494 }
6495
6496 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6497 if (r != 0) {
6498 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6499 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
6500 }
6501 }
6502 if (dinfo->cfg.pcie.pcie_location != 0) {
6503 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6504 PCIER_DEVICE_STA, 2);
6505 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
6506 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6507 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6508 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6509 PCIER_DEVICE_STA, rs, 2);
6510 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
6511 }
6512 }
6513 }
6514
6515 /*
6516 * Perform a Function Level Reset (FLR) on a device.
6517 *
6518 * This function first waits for any pending transactions to complete
6519 * within the timeout specified by max_delay. If transactions are
6520 * still pending, the function will return false without attempting a
6521 * reset.
6522 *
6523 * If dev is not a PCI-express function or does not support FLR, this
6524 * function returns false.
6525 *
6526 * Note that no registers are saved or restored. The caller is
6527 * responsible for saving and restoring any registers including
6528 * PCI-standard registers via pci_save_state() and
6529 * pci_restore_state().
6530 */
6531 bool
pcie_flr(device_t dev,u_int max_delay,bool force)6532 pcie_flr(device_t dev, u_int max_delay, bool force)
6533 {
6534 struct pci_devinfo *dinfo = device_get_ivars(dev);
6535 uint16_t cmd, ctl;
6536 int compl_delay;
6537 int cap;
6538
6539 cap = dinfo->cfg.pcie.pcie_location;
6540 if (cap == 0)
6541 return (false);
6542
6543 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR))
6544 return (false);
6545
6546 /*
6547 * Disable busmastering to prevent generation of new
6548 * transactions while waiting for the device to go idle. If
6549 * the idle timeout fails, the command register is restored
6550 * which will re-enable busmastering.
6551 */
6552 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
6553 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
6554 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
6555 if (!force) {
6556 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
6557 return (false);
6558 }
6559 pci_printf(&dinfo->cfg,
6560 "Resetting with transactions pending after %d ms\n",
6561 max_delay);
6562
6563 /*
6564 * Extend the post-FLR delay to cover the maximum
6565 * Completion Timeout delay of anything in flight
6566 * during the FLR delay. Enforce a minimum delay of
6567 * at least 10ms.
6568 */
6569 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
6570 if (compl_delay < 10)
6571 compl_delay = 10;
6572 } else
6573 compl_delay = 0;
6574
6575 /* Initiate the reset. */
6576 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
6577 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
6578 PCIEM_CTL_INITIATE_FLR, 2);
6579
6580 /* Wait for 100ms. */
6581 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
6582
6583 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
6584 PCIEM_STA_TRANSACTION_PND)
6585 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
6586 return (true);
6587 }
6588
6589 /*
6590 * Attempt a power-management reset by cycling the device in/out of D3
6591 * state. PCI spec says we can only go into D3 state from D0 state.
6592 * Transition from D[12] into D0 before going to D3 state.
6593 */
6594 int
pci_power_reset(device_t dev)6595 pci_power_reset(device_t dev)
6596 {
6597 int ps;
6598
6599 ps = pci_get_powerstate(dev);
6600 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6601 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6602 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6603 pci_set_powerstate(dev, ps);
6604 return (0);
6605 }
6606
6607 /*
6608 * Try link drop and retrain of the downstream port of upstream
6609 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
6610 * cause Conventional Hot reset of the device in the slot.
6611 * Alternative, for PCIe, could be the secondary bus reset initiatied
6612 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
6613 */
6614 int
pcie_link_reset(device_t port,int pcie_location)6615 pcie_link_reset(device_t port, int pcie_location)
6616 {
6617 uint16_t v;
6618
6619 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
6620 v |= PCIEM_LINK_CTL_LINK_DIS;
6621 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6622 pause_sbt("pcier1", mstosbt(20), 0, 0);
6623 v &= ~PCIEM_LINK_CTL_LINK_DIS;
6624 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
6625 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6626 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
6627 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
6628 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
6629 }
6630
6631 static int
pci_reset_post(device_t dev,device_t child)6632 pci_reset_post(device_t dev, device_t child)
6633 {
6634
6635 if (dev == device_get_parent(child))
6636 pci_restore_state(child);
6637 return (0);
6638 }
6639
6640 static int
pci_reset_prepare(device_t dev,device_t child)6641 pci_reset_prepare(device_t dev, device_t child)
6642 {
6643
6644 if (dev == device_get_parent(child))
6645 pci_save_state(child);
6646 return (0);
6647 }
6648
6649 static int
pci_reset_child(device_t dev,device_t child,int flags)6650 pci_reset_child(device_t dev, device_t child, int flags)
6651 {
6652 int error;
6653
6654 if (dev == NULL || device_get_parent(child) != dev)
6655 return (0);
6656 if ((flags & DEVF_RESET_DETACH) != 0) {
6657 error = device_get_state(child) == DS_ATTACHED ?
6658 device_detach(child) : 0;
6659 } else {
6660 error = BUS_SUSPEND_CHILD(dev, child);
6661 }
6662 if (error == 0) {
6663 if (!pcie_flr(child, 1000, false)) {
6664 error = BUS_RESET_PREPARE(dev, child);
6665 if (error == 0)
6666 pci_power_reset(child);
6667 BUS_RESET_POST(dev, child);
6668 }
6669 if ((flags & DEVF_RESET_DETACH) != 0)
6670 device_probe_and_attach(child);
6671 else
6672 BUS_RESUME_CHILD(dev, child);
6673 }
6674 return (error);
6675 }
6676
6677 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)6678 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
6679 {
6680 bool match;
6681 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
6682
6683 vendor = pci_get_vendor(child);
6684 device = pci_get_device(child);
6685 subvendor = pci_get_subvendor(child);
6686 subdevice = pci_get_subdevice(child);
6687 class = pci_get_class(child);
6688 subclass = pci_get_subclass(child);
6689 revid = pci_get_revid(child);
6690 while (nelt-- > 0) {
6691 match = true;
6692 if (id->match_flag_vendor)
6693 match &= vendor == id->vendor;
6694 if (id->match_flag_device)
6695 match &= device == id->device;
6696 if (id->match_flag_subvendor)
6697 match &= subvendor == id->subvendor;
6698 if (id->match_flag_subdevice)
6699 match &= subdevice == id->subdevice;
6700 if (id->match_flag_class)
6701 match &= class == id->class_id;
6702 if (id->match_flag_subclass)
6703 match &= subclass == id->subclass;
6704 if (id->match_flag_revid)
6705 match &= revid == id->revid;
6706 if (match)
6707 return (id);
6708 id++;
6709 }
6710 return (NULL);
6711 }
6712
6713 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)6714 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
6715 {
6716 const char *dev_name;
6717 device_t dev;
6718
6719 dev = dinfo->cfg.dev;
6720 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
6721 dinfo->cfg.slot, dinfo->cfg.func);
6722 dev_name = device_get_name(dev);
6723 if (dev_name != NULL)
6724 printf(" (%s%d)", dev_name, device_get_unit(dev));
6725 }
6726
6727 void
pci_print_faulted_dev(void)6728 pci_print_faulted_dev(void)
6729 {
6730 struct pci_devinfo *dinfo;
6731 device_t dev;
6732 int aer, i;
6733 uint32_t r1, r2;
6734 uint16_t status;
6735
6736 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6737 dev = dinfo->cfg.dev;
6738 status = pci_read_config(dev, PCIR_STATUS, 2);
6739 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6740 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6741 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6742 if (status != 0) {
6743 pci_print_faulted_dev_name(dinfo);
6744 printf(" error 0x%04x\n", status);
6745 }
6746 if (dinfo->cfg.pcie.pcie_location != 0) {
6747 status = pci_read_config(dev,
6748 dinfo->cfg.pcie.pcie_location +
6749 PCIER_DEVICE_STA, 2);
6750 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
6751 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6752 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6753 pci_print_faulted_dev_name(dinfo);
6754 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
6755 pci_read_config(dev,
6756 dinfo->cfg.pcie.pcie_location +
6757 PCIER_DEVICE_CTL, 2),
6758 status);
6759 }
6760 }
6761 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6762 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6763 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6764 if (r1 != 0 || r2 != 0) {
6765 pci_print_faulted_dev_name(dinfo);
6766 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
6767 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
6768 r1, pci_read_config(dev, aer +
6769 PCIR_AER_UC_MASK, 4),
6770 pci_read_config(dev, aer +
6771 PCIR_AER_UC_SEVERITY, 4),
6772 r2, pci_read_config(dev, aer +
6773 PCIR_AER_COR_MASK, 4),
6774 pci_read_config(dev, aer +
6775 PCIR_AER_CAP_CONTROL, 4));
6776 for (i = 0; i < 4; i++) {
6777 r1 = pci_read_config(dev, aer +
6778 PCIR_AER_HEADER_LOG + i * 4, 4);
6779 printf(" HL%d: 0x%08x\n", i, r1);
6780 }
6781 }
6782 }
6783 }
6784 }
6785
6786 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(pcierr,pci_print_faulted_dev_db,DB_CMD_MEMSAFE)6787 DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE)
6788 {
6789
6790 pci_print_faulted_dev();
6791 }
6792
6793 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)6794 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
6795 {
6796 device_t dev;
6797 int aer;
6798 uint32_t r;
6799
6800 dev = dinfo->cfg.dev;
6801 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6802 PCIER_DEVICE_STA, 2);
6803 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6804 PCIER_DEVICE_STA, r, 2);
6805
6806 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
6807 return;
6808 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6809 if (r != 0)
6810 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6811 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6812 if (r != 0)
6813 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6814 }
6815
DB_COMMAND_FLAGS(pci_clearerr,db_pci_clearerr,DB_CMD_MEMSAFE)6816 DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE)
6817 {
6818 struct pci_devinfo *dinfo;
6819 device_t dev;
6820 uint16_t status, status1;
6821
6822 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6823 dev = dinfo->cfg.dev;
6824 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
6825 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6826 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6827 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6828 if (status1 != 0) {
6829 status &= ~status1;
6830 pci_write_config(dev, PCIR_STATUS, status, 2);
6831 }
6832 if (dinfo->cfg.pcie.pcie_location != 0)
6833 db_clear_pcie_errors(dinfo);
6834 }
6835 }
6836 #endif
6837