1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Michal Meloun <[email protected]>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 */
28
29 /* Rockchip PCIe controller driver */
30
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/gpio.h>
36 #include <sys/proc.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/rman.h>
43
44 #include <machine/bus.h>
45 #include <machine/intr.h>
46 #include <machine/resource.h>
47
48 #include <dev/extres/clk/clk.h>
49 #include <dev/extres/hwreset/hwreset.h>
50 #include <dev/extres/phy/phy.h>
51 #include <dev/extres/regulator/regulator.h>
52 #include <dev/gpio/gpiobusvar.h>
53 #include <dev/ofw/ofw_bus.h>
54 #include <dev/ofw/ofw_bus_subr.h>
55 #include <dev/ofw/ofw_pci.h>
56 #include <dev/ofw/ofwpci.h>
57 #include <dev/pci/pcivar.h>
58 #include <dev/pci/pcireg.h>
59 #include <dev/pci/pcib_private.h>
60
61 #include <dev/ofw/ofw_bus.h>
62
63 #include "pcib_if.h"
64
65 #define ATU_CFG_BUS(x) (((x) & 0x0ff) << 20)
66 #define ATU_CFG_SLOT(x) (((x) & 0x01f) << 15)
67 #define ATU_CFG_FUNC(x) (((x) & 0x007) << 12)
68 #define ATU_CFG_REG(x) (((x) & 0xfff) << 0)
69
70 #define ATU_TYPE_MEM 0x2
71 #define ATU_TYPE_IO 0x6
72 #define ATU_TYPE_CFG0 0xA
73 #define ATU_TYPE_CFG1 0xB
74 #define ATY_TYPE_NOR_MSG 0xC
75
76 #define ATU_OB_REGIONS 33
77 #define ATU_OB_REGION_SHIFT 20
78 #define ATU_OB_REGION_SIZE (1 << ATU_OB_REGION_SHIFT)
79 #define ATU_OB_REGION_0_SIZE (( ATU_OB_REGIONS - 1) * ATU_OB_REGION_SIZE)
80
81 #define ATU_IB_REGIONS 3
82
83 #define PCIE_CLIENT_BASIC_STRAP_CONF 0x000000
84 #define STRAP_CONF_GEN_2 (1 << 7)
85 #define STRAP_CONF_MODE_RC (1 << 6)
86 #define STRAP_CONF_LANES(n) ((((n) / 2) & 0x3) << 4)
87 #define STRAP_CONF_ARI_EN (1 << 3)
88 #define STRAP_CONF_SR_IOV_EN (1 << 2)
89 #define STRAP_CONF_LINK_TRAIN_EN (1 << 1)
90 #define STRAP_CONF_CONF_EN (1 << 0)
91 #define PCIE_CLIENT_HOT_RESET_CTRL 0x000018
92 #define HOT_RESET_CTRL_LINK_DOWN_RESET (1 << 1)
93 #define HOT_RESET_CTRL_HOT_RESET_IN (1 << 0)
94 #define PCIE_CLIENT_BASIC_STATUS0 0x000044
95 #define PCIE_CLIENT_BASIC_STATUS1 0x000048
96 #define STATUS1_LINK_ST_GET(x) (((x) >> 20) & 0x3)
97 #define STATUS1_LINK_ST_UP 3
98 #define PCIE_CLIENT_INT_MASK 0x00004C
99 #define PCIE_CLIENT_INT_STATUS 0x000050
100 #define PCIE_CLIENT_INT_LEGACY_DONE (1 << 15)
101 #define PCIE_CLIENT_INT_MSG (1 << 14)
102 #define PCIE_CLIENT_INT_HOT_RST (1 << 13)
103 #define PCIE_CLIENT_INT_DPA (1 << 12)
104 #define PCIE_CLIENT_INT_FATAL_ERR (1 << 11)
105 #define PCIE_CLIENT_INT_NFATAL_ERR (1 << 10)
106 #define PCIE_CLIENT_INT_CORR_ERR (1 << 9)
107 #define PCIE_CLIENT_INT_INTD (1 << 8)
108 #define PCIE_CLIENT_INT_INTC (1 << 7)
109 #define PCIE_CLIENT_INT_INTB (1 << 6)
110 #define PCIE_CLIENT_INT_INTA (1 << 5)
111 #define PCIE_CLIENT_INT_LOCAL (1 << 4)
112 #define PCIE_CLIENT_INT_UDMA (1 << 3)
113 #define PCIE_CLIENT_INT_PHY (1 << 2)
114 #define PCIE_CLIENT_INT_HOT_PLUG (1 << 1)
115 #define PCIE_CLIENT_INT_PWR_STCG (1 << 0)
116 #define PCIE_CLIENT_INT_LEGACY (PCIE_CLIENT_INT_INTA | \
117 PCIE_CLIENT_INT_INTB | \
118 PCIE_CLIENT_INT_INTC | \
119 PCIE_CLIENT_INT_INTD)
120
121 #define PCIE_CORE_CTRL0 0x900000
122 #define CORE_CTRL_LANES_GET(x) (((x) >> 20) & 0x3)
123 #define PCIE_CORE_CTRL1 0x900004
124 #define PCIE_CORE_CONFIG_VENDOR 0x900044
125 #define PCIE_CORE_INT_STATUS 0x90020c
126 #define PCIE_CORE_INT_PRFPE (1 << 0)
127 #define PCIE_CORE_INT_CRFPE (1 << 1)
128 #define PCIE_CORE_INT_RRPE (1 << 2)
129 #define PCIE_CORE_INT_PRFO (1 << 3)
130 #define PCIE_CORE_INT_CRFO (1 << 4)
131 #define PCIE_CORE_INT_RT (1 << 5)
132 #define PCIE_CORE_INT_RTR (1 << 6)
133 #define PCIE_CORE_INT_PE (1 << 7)
134 #define PCIE_CORE_INT_MTR (1 << 8)
135 #define PCIE_CORE_INT_UCR (1 << 9)
136 #define PCIE_CORE_INT_FCE (1 << 10)
137 #define PCIE_CORE_INT_CT (1 << 11)
138 #define PCIE_CORE_INT_UTC (1 << 18)
139 #define PCIE_CORE_INT_MMVC (1 << 19)
140 #define PCIE_CORE_INT_MASK 0x900210
141 #define PCIE_CORE_PHY_FUNC_CONF 0x9002C0
142 #define PCIE_CORE_RC_BAR_CONF 0x900300
143
144 #define PCIE_RC_CONFIG_STD_BASE 0x800000
145 #define PCIE_RC_CONFIG_PRIV_BASE 0xA00000
146 #define PCIE_RC_CONFIG_DCSR 0xA000C8
147 #define PCIE_RC_CONFIG_DCSR_MPS_MASK (0x7 << 5)
148 #define PCIE_RC_CONFIG_DCSR_MPS_128 (0 << 5)
149 #define PCIE_RC_CONFIG_DCSR_MPS_256 (1 << 5)
150 #define PCIE_RC_CONFIG_LINK_CAP 0xA00CC
151 #define PCIE_RC_CONFIG_LINK_CAP_L0S (1 << 10)
152
153 #define PCIE_RC_CONFIG_LCS 0xA000D0
154 #define PCIE_RC_CONFIG_THP_CAP 0xA00274
155 #define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK 0xFFF00000
156
157 #define PCIE_CORE_OB_ADDR0(n) (0xC00000 + 0x20 * (n) + 0x00)
158 #define PCIE_CORE_OB_ADDR1(n) (0xC00000 + 0x20 * (n) + 0x04)
159 #define PCIE_CORE_OB_DESC0(n) (0xC00000 + 0x20 * (n) + 0x08)
160 #define PCIE_CORE_OB_DESC1(n) (0xC00000 + 0x20 * (n) + 0x0C)
161 #define PCIE_CORE_OB_DESC2(n) (0xC00000 + 0x20 * (n) + 0x10)
162 #define PCIE_CORE_OB_DESC3(n) (0xC00000 + 0x20 * (n) + 0x14)
163
164 #define PCIE_CORE_IB_ADDR0(n) (0xC00800 + 0x8 * (n) + 0x00)
165 #define PCIE_CORE_IB_ADDR1(n) (0xC00800 + 0x8 * (n) + 0x04)
166
167 #define PRIV_CFG_RD4(sc, reg) \
168 (uint32_t)rk_pcie_local_cfg_read(sc, true, reg, 4)
169 #define PRIV_CFG_RD2(sc, reg) \
170 (uint16_t)rk_pcie_local_cfg_read(sc, true, reg, 2)
171 #define PRIV_CFG_RD1(sc, reg) \
172 (uint8_t)rk_pcie_local_cfg_read(sc, true, reg, 1)
173 #define PRIV_CFG_WR4(sc, reg, val) \
174 rk_pcie_local_cfg_write(sc, true, reg, val, 4)
175 #define PRIV_CFG_WR2(sc, reg, val) \
176 rk_pcie_local_cfg_write(sc, true, reg, val, 2)
177 #define PRIV_CFG_WR1(sc, reg, val) \
178 rk_pcie_local_cfg_write(sc, true, reg, val, 1)
179
180 #define APB_WR4(_sc, _r, _v) bus_write_4((_sc)->apb_mem_res, (_r), (_v))
181 #define APB_RD4(_sc, _r) bus_read_4((_sc)->apb_mem_res, (_r))
182
183 #define MAX_LANES 4
184
185 #define RK_PCIE_ENABLE_MSI
186 #define RK_PCIE_ENABLE_MSIX
187
188 struct rk_pcie_softc {
189 struct ofw_pci_softc ofw_pci; /* Must be first */
190
191 struct resource *axi_mem_res;
192 struct resource *apb_mem_res;
193 struct resource *client_irq_res;
194 struct resource *legacy_irq_res;
195 struct resource *sys_irq_res;
196 void *client_irq_cookie;
197 void *legacy_irq_cookie;
198 void *sys_irq_cookie;
199
200 device_t dev;
201 phandle_t node;
202 struct mtx mtx;
203
204 struct ofw_pci_range mem_range;
205 struct ofw_pci_range pref_mem_range;
206 struct ofw_pci_range io_range;
207
208 bool coherent;
209 bus_dma_tag_t dmat;
210
211 int num_lanes;
212 bool link_is_gen2;
213 bool no_l0s;
214
215 u_int bus_start;
216 u_int bus_end;
217 u_int root_bus;
218 u_int sub_bus;
219
220 regulator_t supply_12v;
221 regulator_t supply_3v3;
222 regulator_t supply_1v8;
223 regulator_t supply_0v9;
224 hwreset_t hwreset_core;
225 hwreset_t hwreset_mgmt;
226 hwreset_t hwreset_mgmt_sticky;
227 hwreset_t hwreset_pipe;
228 hwreset_t hwreset_pm;
229 hwreset_t hwreset_aclk;
230 hwreset_t hwreset_pclk;
231 clk_t clk_aclk;
232 clk_t clk_aclk_perf;
233 clk_t clk_hclk;
234 clk_t clk_pm;
235 phy_t phys[MAX_LANES];
236 gpio_pin_t gpio_ep;
237 };
238
239 /* Compatible devices. */
240 static struct ofw_compat_data compat_data[] = {
241 {"rockchip,rk3399-pcie", 1},
242 {NULL, 0},
243 };
244
245 static uint32_t
rk_pcie_local_cfg_read(struct rk_pcie_softc * sc,bool priv,u_int reg,int bytes)246 rk_pcie_local_cfg_read(struct rk_pcie_softc *sc, bool priv, u_int reg,
247 int bytes)
248 {
249 uint32_t val;
250 bus_addr_t base;
251
252 if (priv)
253 base = PCIE_RC_CONFIG_PRIV_BASE;
254 else
255 base = PCIE_RC_CONFIG_STD_BASE;
256
257 switch (bytes) {
258 case 4:
259 val = bus_read_4(sc->apb_mem_res, base + reg);
260 break;
261 case 2:
262 val = bus_read_2(sc->apb_mem_res, base + reg);
263 break;
264 case 1:
265 val = bus_read_1(sc->apb_mem_res, base + reg);
266 break;
267 default:
268 val = 0xFFFFFFFF;
269 }
270 return (val);
271 }
272
273 static void
rk_pcie_local_cfg_write(struct rk_pcie_softc * sc,bool priv,u_int reg,uint32_t val,int bytes)274 rk_pcie_local_cfg_write(struct rk_pcie_softc *sc, bool priv, u_int reg,
275 uint32_t val, int bytes)
276 {
277 uint32_t val2;
278 bus_addr_t base;
279
280 if (priv)
281 base = PCIE_RC_CONFIG_PRIV_BASE;
282 else
283 base = PCIE_RC_CONFIG_STD_BASE;
284
285 switch (bytes) {
286 case 4:
287 bus_write_4(sc->apb_mem_res, base + reg, val);
288 break;
289 case 2:
290 val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
291 val2 &= ~(0xffff << ((reg & 3) << 3));
292 val2 |= ((val & 0xffff) << ((reg & 3) << 3));
293 bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
294 break;
295 case 1:
296 val2 = bus_read_4(sc->apb_mem_res, base + (reg & ~3));
297 val2 &= ~(0xff << ((reg & 3) << 3));
298 val2 |= ((val & 0xff) << ((reg & 3) << 3));
299 bus_write_4(sc->apb_mem_res, base + (reg & ~3), val2);
300 break;
301 }
302 }
303
304 static bool
rk_pcie_check_dev(struct rk_pcie_softc * sc,u_int bus,u_int slot,u_int func,u_int reg)305 rk_pcie_check_dev(struct rk_pcie_softc *sc, u_int bus, u_int slot, u_int func,
306 u_int reg)
307 {
308 uint32_t val;
309
310 if (bus < sc->bus_start || bus > sc->bus_end || slot > PCI_SLOTMAX ||
311 func > PCI_FUNCMAX || reg > PCIE_REGMAX)
312 return (false);
313
314 if (bus == sc->root_bus) {
315 /* we have only 1 device with 1 function root port */
316 if (slot > 0 || func > 0)
317 return (false);
318 return (true);
319 }
320
321 /* link is needed for accessing non-root busses */
322 val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
323 if (STATUS1_LINK_ST_GET(val) != STATUS1_LINK_ST_UP)
324 return (false);
325
326 /* only one device can be on first subordinate bus */
327 if (bus == sc->sub_bus && slot != 0 )
328 return (false);
329 return (true);
330 }
331
332 static void
rk_pcie_map_out_atu(struct rk_pcie_softc * sc,int idx,int type,int num_bits,uint64_t pa)333 rk_pcie_map_out_atu(struct rk_pcie_softc *sc, int idx, int type,
334 int num_bits, uint64_t pa)
335 {
336 uint32_t addr0;
337 uint64_t max_size __diagused;
338
339 /* Check HW constrains */
340 max_size = idx == 0 ? ATU_OB_REGION_0_SIZE: ATU_OB_REGION_SIZE;
341 KASSERT(idx < ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
342 KASSERT(num_bits >= 7 && num_bits <= 63,
343 ("Bit width of region is invalid: %d\n", num_bits));
344 KASSERT(max_size <= (1ULL << (num_bits + 1)),
345 ("Bit width is invalid for given region[%d]: %d\n", idx, num_bits));
346
347 addr0 = (uint32_t)pa & 0xFFFFFF00;
348 addr0 |= num_bits;
349 APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), addr0);
350 APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), (uint32_t)(pa >> 32));
351 APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 | type);
352 APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
353
354 /* Readback for sync */
355 APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
356 }
357
358 static void
rk_pcie_map_cfg_atu(struct rk_pcie_softc * sc,int idx,int type)359 rk_pcie_map_cfg_atu(struct rk_pcie_softc *sc, int idx, int type)
360 {
361
362 /* Check HW constrains */
363 KASSERT(idx < ATU_OB_REGIONS, ("Invalid region index: %d\n", idx));
364
365 /*
366 * Config window is only 25 bits width, so we cannot encode full bus
367 * range into it. Remaining bits of bus number should be taken from
368 * DESC1 field.
369 */
370 APB_WR4(sc, PCIE_CORE_OB_ADDR0(idx), 25 - 1);
371 APB_WR4(sc, PCIE_CORE_OB_ADDR1(idx), 0);
372 APB_WR4(sc, PCIE_CORE_OB_DESC0(idx), 1 << 23 | type);
373 APB_WR4(sc, PCIE_CORE_OB_DESC1(idx), sc->root_bus);
374
375 /* Readback for sync */
376 APB_RD4(sc, PCIE_CORE_OB_DESC1(idx));
377
378 }
379
380 static void
rk_pcie_map_in_atu(struct rk_pcie_softc * sc,int idx,int num_bits,uint64_t pa)381 rk_pcie_map_in_atu(struct rk_pcie_softc *sc, int idx, int num_bits, uint64_t pa)
382 {
383 uint32_t addr0;
384
385 /* Check HW constrains */
386 KASSERT(idx < ATU_IB_REGIONS, ("Invalid region index: %d\n", idx));
387 KASSERT(num_bits >= 7 && num_bits <= 63,
388 ("Bit width of region is invalid: %d\n", num_bits));
389
390 addr0 = (uint32_t)pa & 0xFFFFFF00;
391 addr0 |= num_bits;
392 APB_WR4(sc, PCIE_CORE_IB_ADDR0(idx), addr0);
393 APB_WR4(sc, PCIE_CORE_IB_ADDR1(idx), (uint32_t)(pa >> 32));
394
395 /* Readback for sync */
396 APB_RD4(sc, PCIE_CORE_IB_ADDR1(idx));
397 }
398
399 static int
rk_pcie_decode_ranges(struct rk_pcie_softc * sc,struct ofw_pci_range * ranges,int nranges)400 rk_pcie_decode_ranges(struct rk_pcie_softc *sc, struct ofw_pci_range *ranges,
401 int nranges)
402 {
403 int i;
404
405 for (i = 0; i < nranges; i++) {
406 switch(ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) {
407 case OFW_PCI_PHYS_HI_SPACE_IO:
408 if (sc->io_range.size != 0) {
409 device_printf(sc->dev,
410 "Duplicated IO range found in DT\n");
411 return (ENXIO);
412 }
413 sc->io_range = ranges[i];
414 break;
415 case OFW_PCI_PHYS_HI_SPACE_MEM32:
416 case OFW_PCI_PHYS_HI_SPACE_MEM64:
417 if (ranges[i].pci_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) {
418 if (sc->pref_mem_range.size != 0) {
419 device_printf(sc->dev,
420 "Duplicated memory range found "
421 "in DT\n");
422 return (ENXIO);
423 }
424 sc->pref_mem_range = ranges[i];
425 } else {
426 if (sc->mem_range.size != 0) {
427 device_printf(sc->dev,
428 "Duplicated memory range found "
429 "in DT\n");
430 return (ENXIO);
431 }
432 sc->mem_range = ranges[i];
433 }
434 }
435 }
436 if (sc->mem_range.size == 0) {
437 device_printf(sc->dev,
438 " At least memory range should be defined in DT.\n");
439 return (ENXIO);
440 }
441 return (0);
442 }
443
444 /*-----------------------------------------------------------------------------
445 *
446 * P C I B I N T E R F A C E
447 */
448 static uint32_t
rk_pcie_read_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,int bytes)449 rk_pcie_read_config(device_t dev, u_int bus, u_int slot,
450 u_int func, u_int reg, int bytes)
451 {
452 struct rk_pcie_softc *sc;
453 uint32_t d32, data;
454 uint16_t d16;
455 uint8_t d8;
456 uint64_t addr;
457 int type, ret;
458
459 sc = device_get_softc(dev);
460
461 if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
462 return (0xFFFFFFFFU);
463 if (bus == sc->root_bus)
464 return (rk_pcie_local_cfg_read(sc, false, reg, bytes));
465
466 addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
467 ATU_CFG_REG(reg);
468 type = bus == sc->sub_bus ? ATU_TYPE_CFG0: ATU_TYPE_CFG1;
469 rk_pcie_map_cfg_atu(sc, 0, type);
470
471 ret = -1;
472 switch (bytes) {
473 case 1:
474 ret = bus_peek_1(sc->axi_mem_res, addr, &d8);
475 data = d8;
476 break;
477 case 2:
478 ret = bus_peek_2(sc->axi_mem_res, addr, &d16);
479 data = d16;
480 break;
481 case 4:
482 ret = bus_peek_4(sc->axi_mem_res, addr, &d32);
483 data = d32;
484 break;
485 }
486 if (ret != 0)
487 data = 0xFFFFFFFF;
488 return (data);
489 }
490
491 static void
rk_pcie_write_config(device_t dev,u_int bus,u_int slot,u_int func,u_int reg,uint32_t val,int bytes)492 rk_pcie_write_config(device_t dev, u_int bus, u_int slot,
493 u_int func, u_int reg, uint32_t val, int bytes)
494 {
495 struct rk_pcie_softc *sc;
496 uint64_t addr;
497 int type;
498
499 sc = device_get_softc(dev);
500
501 if (!rk_pcie_check_dev(sc, bus, slot, func, reg))
502 return;
503
504 if (bus == sc->root_bus)
505 return (rk_pcie_local_cfg_write(sc, false, reg, val, bytes));
506
507 addr = ATU_CFG_BUS(bus) | ATU_CFG_SLOT(slot) | ATU_CFG_FUNC(func) |
508 ATU_CFG_REG(reg);
509 type = bus == sc->sub_bus ? ATU_TYPE_CFG0: ATU_TYPE_CFG1;
510 rk_pcie_map_cfg_atu(sc, 0, type);
511
512 switch (bytes) {
513 case 1:
514 bus_poke_1(sc->axi_mem_res, addr, (uint8_t)val);
515 break;
516 case 2:
517 bus_poke_2(sc->axi_mem_res, addr, (uint16_t)val);
518 break;
519 case 4:
520 bus_poke_4(sc->axi_mem_res, addr, val);
521 break;
522 default:
523 break;
524 }
525 }
526
527 #ifdef RK_PCIE_ENABLE_MSI
528 static int
rk_pcie_alloc_msi(device_t pci,device_t child,int count,int maxcount,int * irqs)529 rk_pcie_alloc_msi(device_t pci, device_t child, int count,
530 int maxcount, int *irqs)
531 {
532 phandle_t msi_parent;
533 int rv;
534
535 rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
536 &msi_parent, NULL);
537 if (rv != 0)
538 return (rv);
539
540 rv = intr_alloc_msi(pci, child, msi_parent, count, maxcount,irqs);
541 return (rv);
542 }
543
544 static int
rk_pcie_release_msi(device_t pci,device_t child,int count,int * irqs)545 rk_pcie_release_msi(device_t pci, device_t child, int count, int *irqs)
546 {
547 phandle_t msi_parent;
548 int rv;
549
550 rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
551 &msi_parent, NULL);
552 if (rv != 0)
553 return (rv);
554 rv = intr_release_msi(pci, child, msi_parent, count, irqs);
555 return (rv);
556 }
557 #endif
558
559 static int
rk_pcie_map_msi(device_t pci,device_t child,int irq,uint64_t * addr,uint32_t * data)560 rk_pcie_map_msi(device_t pci, device_t child, int irq, uint64_t *addr,
561 uint32_t *data)
562 {
563 phandle_t msi_parent;
564 int rv;
565
566 rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
567 &msi_parent, NULL);
568 if (rv != 0)
569 return (rv);
570 rv = intr_map_msi(pci, child, msi_parent, irq, addr, data);
571 return (rv);
572 }
573
574 #ifdef RK_PCIE_ENABLE_MSIX
575 static int
rk_pcie_alloc_msix(device_t pci,device_t child,int * irq)576 rk_pcie_alloc_msix(device_t pci, device_t child, int *irq)
577 {
578 phandle_t msi_parent;
579 int rv;
580
581 rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
582 &msi_parent, NULL);
583 if (rv != 0)
584 return (rv);
585 rv = intr_alloc_msix(pci, child, msi_parent, irq);
586 return (rv);
587 }
588
589 static int
rk_pcie_release_msix(device_t pci,device_t child,int irq)590 rk_pcie_release_msix(device_t pci, device_t child, int irq)
591 {
592 phandle_t msi_parent;
593 int rv;
594
595 rv = ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child),
596 &msi_parent, NULL);
597 if (rv != 0)
598 return (rv);
599 rv = intr_release_msix(pci, child, msi_parent, irq);
600 return (rv);
601 }
602 #endif
603
604 static int
rk_pcie_get_id(device_t pci,device_t child,enum pci_id_type type,uintptr_t * id)605 rk_pcie_get_id(device_t pci, device_t child, enum pci_id_type type,
606 uintptr_t *id)
607 {
608 phandle_t node;
609 int rv;
610 uint32_t rid;
611 uint16_t pci_rid;
612
613 if (type != PCI_ID_MSI)
614 return (pcib_get_id(pci, child, type, id));
615
616 node = ofw_bus_get_node(pci);
617 pci_rid = pci_get_rid(child);
618
619 rv = ofw_bus_msimap(node, pci_rid, NULL, &rid);
620 if (rv != 0)
621 return (rv);
622
623 *id = rid;
624 return (0);
625 }
626
627 static int
rk_pcie_route_interrupt(device_t bus,device_t dev,int pin)628 rk_pcie_route_interrupt(device_t bus, device_t dev, int pin)
629 {
630 struct rk_pcie_softc *sc;
631 u_int irq;
632
633 sc = device_get_softc(bus);
634 irq = intr_map_clone_irq(rman_get_start(sc->legacy_irq_res));
635 device_printf(bus, "route pin %d for device %d.%d to %u\n",
636 pin, pci_get_slot(dev), pci_get_function(dev), irq);
637
638 return (irq);
639 }
640
641 /*-----------------------------------------------------------------------------
642 *
643 * B U S / D E V I C E I N T E R F A C E
644 */
645 static int
rk_pcie_parse_fdt_resources(struct rk_pcie_softc * sc)646 rk_pcie_parse_fdt_resources(struct rk_pcie_softc *sc)
647 {
648 int i, rv;
649 char buf[16];
650
651 /* Regulators. All are optional. */
652 rv = regulator_get_by_ofw_property(sc->dev, 0,
653 "vpcie12v-supply", &sc->supply_12v);
654 if (rv != 0 && rv != ENOENT) {
655 device_printf(sc->dev,"Cannot get 'vpcie12' regulator\n");
656 return (ENXIO);
657 }
658 rv = regulator_get_by_ofw_property(sc->dev, 0,
659 "vpcie3v3-supply", &sc->supply_3v3);
660 if (rv != 0 && rv != ENOENT) {
661 device_printf(sc->dev,"Cannot get 'vpcie3v3' regulator\n");
662 return (ENXIO);
663 }
664 rv = regulator_get_by_ofw_property(sc->dev, 0,
665 "vpcie1v8-supply", &sc->supply_1v8);
666 if (rv != 0 && rv != ENOENT) {
667 device_printf(sc->dev,"Cannot get 'vpcie1v8' regulator\n");
668 return (ENXIO);
669 }
670 rv = regulator_get_by_ofw_property(sc->dev, 0,
671 "vpcie0v9-supply", &sc->supply_0v9);
672 if (rv != 0 && rv != ENOENT) {
673 device_printf(sc->dev,"Cannot get 'vpcie0v9' regulator\n");
674 return (ENXIO);
675 }
676
677 /* Resets. */
678 rv = hwreset_get_by_ofw_name(sc->dev, 0, "core", &sc->hwreset_core);
679 if (rv != 0) {
680 device_printf(sc->dev, "Cannot get 'core' reset\n");
681 return (ENXIO);
682 }
683 rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt", &sc->hwreset_mgmt);
684 if (rv != 0) {
685 device_printf(sc->dev, "Cannot get 'mgmt' reset\n");
686 return (ENXIO);
687 }
688 rv = hwreset_get_by_ofw_name(sc->dev, 0, "mgmt-sticky",
689 &sc->hwreset_mgmt_sticky);
690 if (rv != 0) {
691 device_printf(sc->dev, "Cannot get 'mgmt-sticky' reset\n");
692 return (ENXIO);
693 }
694 rv = hwreset_get_by_ofw_name(sc->dev, 0, "pipe", &sc->hwreset_pipe);
695 if (rv != 0) {
696 device_printf(sc->dev, "Cannot get 'pipe' reset\n");
697 return (ENXIO);
698 }
699 rv = hwreset_get_by_ofw_name(sc->dev, 0, "pm", &sc->hwreset_pm);
700 if (rv != 0) {
701 device_printf(sc->dev, "Cannot get 'pm' reset\n");
702 return (ENXIO);
703 }
704 rv = hwreset_get_by_ofw_name(sc->dev, 0, "aclk", &sc->hwreset_aclk);
705 if (rv != 0) {
706 device_printf(sc->dev, "Cannot get 'aclk' reset\n");
707 return (ENXIO);
708 }
709 rv = hwreset_get_by_ofw_name(sc->dev, 0, "pclk", &sc->hwreset_pclk);
710 if (rv != 0) {
711 device_printf(sc->dev, "Cannot get 'pclk' reset\n");
712 return (ENXIO);
713 }
714
715 /* Clocks. */
716 rv = clk_get_by_ofw_name(sc->dev, 0, "aclk", &sc->clk_aclk);
717 if (rv != 0) {
718 device_printf(sc->dev, "Cannot get 'aclk' clock\n");
719 return (ENXIO);
720 }
721 rv = clk_get_by_ofw_name(sc->dev, 0, "aclk-perf", &sc->clk_aclk_perf);
722 if (rv != 0) {
723 device_printf(sc->dev, "Cannot get 'aclk-perf' clock\n");
724 return (ENXIO);
725 }
726 rv = clk_get_by_ofw_name(sc->dev, 0, "hclk", &sc->clk_hclk);
727 if (rv != 0) {
728 device_printf(sc->dev, "Cannot get 'hclk' clock\n");
729 return (ENXIO);
730 }
731 rv = clk_get_by_ofw_name(sc->dev, 0, "pm", &sc->clk_pm);
732 if (rv != 0) {
733 device_printf(sc->dev, "Cannot get 'pm' clock\n");
734 return (ENXIO);
735 }
736
737 /* Phys. */
738 for (i = 0; i < MAX_LANES; i++ ) {
739 sprintf (buf, "pcie-phy-%d", i);
740 rv = phy_get_by_ofw_name(sc->dev, 0, buf, sc->phys + i);
741 if (rv != 0) {
742 device_printf(sc->dev, "Cannot get '%s' phy\n", buf);
743 return (ENXIO);
744 }
745 }
746
747 /* GPIO for PERST#. Optional */
748 rv = gpio_pin_get_by_ofw_property(sc->dev, sc->node, "ep-gpios",
749 &sc->gpio_ep);
750 if (rv != 0 && rv != ENOENT) {
751 device_printf(sc->dev, "Cannot get 'ep-gpios' gpio\n");
752 return (ENXIO);
753 }
754
755 return (0);
756 }
757
758 static int
rk_pcie_enable_resources(struct rk_pcie_softc * sc)759 rk_pcie_enable_resources(struct rk_pcie_softc *sc)
760 {
761 int i, rv;
762 uint32_t val;
763
764 /* Assert all resets */
765 rv = hwreset_assert(sc->hwreset_pclk);
766 if (rv != 0) {
767 device_printf(sc->dev, "Cannot assert 'pclk' reset\n");
768 return (rv);
769 }
770 rv = hwreset_assert(sc->hwreset_aclk);
771 if (rv != 0) {
772 device_printf(sc->dev, "Cannot assert 'aclk' reset\n");
773 return (rv);
774 }
775 rv = hwreset_assert(sc->hwreset_pm);
776 if (rv != 0) {
777 device_printf(sc->dev, "Cannot assert 'pm' reset\n");
778 return (rv);
779 }
780 rv = hwreset_assert(sc->hwreset_pipe);
781 if (rv != 0) {
782 device_printf(sc->dev, "Cannot assert 'pipe' reset\n");
783 return (rv);
784 }
785 rv = hwreset_assert(sc->hwreset_mgmt_sticky);
786 if (rv != 0) {
787 device_printf(sc->dev, "Cannot assert 'mgmt_sticky' reset\n");
788 return (rv);
789 }
790 rv = hwreset_assert(sc->hwreset_mgmt);
791 if (rv != 0) {
792 device_printf(sc->dev, "Cannot assert 'hmgmt' reset\n");
793 return (rv);
794 }
795 rv = hwreset_assert(sc->hwreset_core);
796 if (rv != 0) {
797 device_printf(sc->dev, "Cannot assert 'hcore' reset\n");
798 return (rv);
799 }
800 DELAY(10000);
801
802 /* Enable clockls */
803 rv = clk_enable(sc->clk_aclk);
804 if (rv != 0) {
805 device_printf(sc->dev, "Cannot enable 'aclk' clock\n");
806 return (rv);
807 }
808 rv = clk_enable(sc->clk_aclk_perf);
809 if (rv != 0) {
810 device_printf(sc->dev, "Cannot enable 'aclk_perf' clock\n");
811 return (rv);
812 }
813 rv = clk_enable(sc->clk_hclk);
814 if (rv != 0) {
815 device_printf(sc->dev, "Cannot enable 'hclk' clock\n");
816 return (rv);
817 }
818 rv = clk_enable(sc->clk_pm);
819 if (rv != 0) {
820 device_printf(sc->dev, "Cannot enable 'pm' clock\n");
821 return (rv);
822 }
823
824 /* Power up regulators */
825 if (sc->supply_12v != NULL) {
826 rv = regulator_enable(sc->supply_12v);
827 if (rv != 0) {
828 device_printf(sc->dev,
829 "Cannot enable 'vpcie12' regulator\n");
830 return (rv);
831 }
832 }
833 if (sc->supply_3v3 != NULL) {
834 rv = regulator_enable(sc->supply_3v3);
835 if (rv != 0) {
836 device_printf(sc->dev,
837 "Cannot enable 'vpcie3v3' regulator\n");
838 return (rv);
839 }
840 }
841 if (sc->supply_1v8 != NULL) {
842 rv = regulator_enable(sc->supply_1v8);
843 if (rv != 0) {
844 device_printf(sc->dev,
845 "Cannot enable 'vpcie1v8' regulator\n");
846 return (rv);
847 }
848 }
849 if (sc->supply_0v9 != NULL) {
850 rv = regulator_enable(sc->supply_0v9);
851 if (rv != 0) {
852 device_printf(sc->dev,
853 "Cannot enable 'vpcie1v8' regulator\n");
854 return (rv);
855 }
856 }
857 DELAY(1000);
858
859 /* Deassert basic resets*/
860 rv = hwreset_deassert(sc->hwreset_pm);
861 if (rv != 0) {
862 device_printf(sc->dev, "Cannot deassert 'pm' reset\n");
863 return (rv);
864 }
865 rv = hwreset_deassert(sc->hwreset_aclk);
866 if (rv != 0) {
867 device_printf(sc->dev, "Cannot deassert 'aclk' reset\n");
868 return (rv);
869 }
870 rv = hwreset_deassert(sc->hwreset_pclk);
871 if (rv != 0) {
872 device_printf(sc->dev, "Cannot deassert 'pclk' reset\n");
873 return (rv);
874 }
875
876 /* Set basic PCIe core mode (RC, lanes, gen1 or 2) */
877 val = STRAP_CONF_GEN_2 << 16 |
878 (sc->link_is_gen2 ? STRAP_CONF_GEN_2: 0);
879 val |= STRAP_CONF_MODE_RC << 16 | STRAP_CONF_MODE_RC;
880 val |= STRAP_CONF_LANES(~0) << 16 | STRAP_CONF_LANES(sc->num_lanes);
881 val |= STRAP_CONF_ARI_EN << 16 | STRAP_CONF_ARI_EN;
882 val |= STRAP_CONF_CONF_EN << 16 | STRAP_CONF_CONF_EN;
883 APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF, val);
884
885 for (i = 0; i < MAX_LANES; i++) {
886 rv = phy_enable(sc->phys[i]);
887 if (rv != 0) {
888 device_printf(sc->dev, "Cannot enable phy %d\n", i);
889 return (rv);
890 }
891 }
892
893 /* Deassert rest of resets - order is important ! */
894 rv = hwreset_deassert(sc->hwreset_mgmt_sticky);
895 if (rv != 0) {
896 device_printf(sc->dev, "Cannot deassert 'mgmt_sticky' reset\n");
897 return (rv);
898 }
899 rv = hwreset_deassert(sc->hwreset_core);
900 if (rv != 0) {
901 device_printf(sc->dev, "Cannot deassert 'core' reset\n");
902 return (rv);
903 }
904 rv = hwreset_deassert(sc->hwreset_mgmt);
905 if (rv != 0) {
906 device_printf(sc->dev, "Cannot deassert 'mgmt' reset\n");
907 return (rv);
908 }
909 rv = hwreset_deassert(sc->hwreset_pipe);
910 if (rv != 0) {
911 device_printf(sc->dev, "Cannot deassert 'pipe' reset\n");
912 return (rv);
913 }
914 return (0);
915 }
916
917 static int
rk_pcie_setup_hw(struct rk_pcie_softc * sc)918 rk_pcie_setup_hw(struct rk_pcie_softc *sc)
919 {
920 uint32_t val;
921 int i, rv;
922
923 /* Assert PERST# if defined */
924 if (sc->gpio_ep != NULL) {
925 rv = gpio_pin_set_active(sc->gpio_ep, 0);
926 if (rv != 0) {
927 device_printf(sc->dev,
928 "Cannot clear 'gpio-ep' gpio\n");
929 return (rv);
930 }
931 }
932
933 rv = rk_pcie_enable_resources(sc);
934 if (rv != 0)
935 return(rv);
936
937 /* Fix wrong default value for transmited FTS for L0s exit */
938 val = APB_RD4(sc, PCIE_CORE_CTRL1);
939 val |= 0xFFFF << 8;
940 APB_WR4(sc, PCIE_CORE_CTRL1, val);
941
942 /* Setup PCIE Link Status & Control register */
943 val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
944 val |= PCIEM_LINK_CTL_COMMON_CLOCK;
945 APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
946 val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
947 val |= PCIEM_LINK_CTL_RCB;
948 APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
949
950 /* Enable training for GEN1 */
951 APB_WR4(sc, PCIE_CLIENT_BASIC_STRAP_CONF,
952 STRAP_CONF_LINK_TRAIN_EN << 16 | STRAP_CONF_LINK_TRAIN_EN);
953
954 /* Deassert PERST# if defined */
955 if (sc->gpio_ep != NULL) {
956 rv = gpio_pin_set_active(sc->gpio_ep, 1);
957 if (rv != 0) {
958 device_printf(sc->dev, "Cannot set 'gpio-ep' gpio\n");
959 return (rv);
960 }
961 }
962
963 /* Wait for link */
964 for (i = 500; i > 0; i--) {
965 val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
966 if (STATUS1_LINK_ST_GET(val) == STATUS1_LINK_ST_UP)
967 break;
968 DELAY(1000);
969 }
970 if (i <= 0) {
971 device_printf(sc->dev,
972 "Gen1 link training timeouted: 0x%08X.\n", val);
973 return (0);
974 }
975
976 if (sc->link_is_gen2) {
977 val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
978 val |= PCIEM_LINK_CTL_RETRAIN_LINK;
979 APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
980
981 /* Wait for link */
982 for (i = 500; i > 0; i--) {
983 val = APB_RD4(sc, PCIE_CLIENT_BASIC_STATUS1);
984 if (STATUS1_LINK_ST_GET(val) ==
985 STATUS1_LINK_ST_UP)
986 break;
987 DELAY(1000);
988 }
989 if (i <= 0)
990 device_printf(sc->dev, "Gen2 link training "
991 "timeouted: 0x%08X.\n", val);
992 }
993
994 val = APB_RD4(sc, PCIE_CORE_CTRL0);
995 val = CORE_CTRL_LANES_GET(val);
996 if (bootverbose)
997 device_printf(sc->dev, "Link width: %d\n", 1 << val);
998
999 return (0);
1000 }
1001
1002 static int
rk_pcie_setup_sw(struct rk_pcie_softc * sc)1003 rk_pcie_setup_sw(struct rk_pcie_softc *sc)
1004 {
1005 uint32_t val;
1006 int i, region;
1007
1008 pcib_bridge_init(sc->dev);
1009
1010 /* Setup config registers */
1011 APB_WR4(sc, PCIE_CORE_CONFIG_VENDOR, 0x1D87); /* Rockchip vendor ID*/
1012 PRIV_CFG_WR1(sc, PCIR_CLASS, PCIC_BRIDGE);
1013 PRIV_CFG_WR1(sc, PCIR_SUBCLASS, PCIS_BRIDGE_PCI);
1014 PRIV_CFG_WR1(sc, PCIR_PRIBUS_1, sc->root_bus);
1015 PRIV_CFG_WR1(sc, PCIR_SECBUS_1, sc->sub_bus);
1016 PRIV_CFG_WR1(sc, PCIR_SUBBUS_1, sc->bus_end);
1017 PRIV_CFG_WR2(sc, PCIR_COMMAND, PCIM_CMD_MEMEN |
1018 PCIM_CMD_BUSMASTEREN | PCIM_CMD_SERRESPEN);
1019
1020 /* Don't advertise L1 power substate */
1021 val = APB_RD4(sc, PCIE_RC_CONFIG_THP_CAP);
1022 val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
1023 APB_WR4(sc, PCIE_RC_CONFIG_THP_CAP, val);
1024
1025 /* Don't advertise L0s */
1026 if (sc->no_l0s) {
1027 val = APB_RD4(sc, PCIE_RC_CONFIG_LINK_CAP);
1028 val &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
1029 APB_WR4(sc, PCIE_RC_CONFIG_LINK_CAP_L0S, val);
1030 }
1031
1032 /*Adjust maximum payload size*/
1033 val = APB_RD4(sc, PCIE_RC_CONFIG_DCSR);
1034 val &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
1035 val |= PCIE_RC_CONFIG_DCSR_MPS_128;
1036 APB_WR4(sc, PCIE_RC_CONFIG_DCSR, val);
1037
1038 /*
1039 * Prepare IB ATU
1040 * map whole address range in 1:1 mappings
1041 */
1042 rk_pcie_map_in_atu(sc, 2, 64 - 1, 0);
1043
1044 /* Prepare OB ATU */
1045 /* - region 0 (32 MB) is used for config access */
1046 region = 0;
1047 rk_pcie_map_out_atu(sc, region++, ATU_TYPE_CFG0, 25 - 1, 0);
1048
1049 /* - then map memory (by using 1MB regions */
1050 for (i = 0; i < sc->mem_range.size / ATU_OB_REGION_SIZE; i++) {
1051 rk_pcie_map_out_atu(sc, region++, ATU_TYPE_MEM,
1052 ATU_OB_REGION_SHIFT - 1,
1053 sc->mem_range.pci + ATU_OB_REGION_SIZE * i);
1054 }
1055
1056 /* - IO space is next, one region typically*/
1057 for (i = 0; i < sc->io_range.size / ATU_OB_REGION_SIZE; i++) {
1058 rk_pcie_map_out_atu(sc, region++, ATU_TYPE_IO,
1059 ATU_OB_REGION_SHIFT - 1,
1060 sc->io_range.pci + ATU_OB_REGION_SIZE * i);
1061 }
1062 APB_WR4(sc, PCIE_CORE_RC_BAR_CONF, 0);
1063 return (0);
1064 }
1065
1066 static int
rk_pcie_sys_irq(void * arg)1067 rk_pcie_sys_irq(void *arg)
1068 {
1069 struct rk_pcie_softc *sc;
1070 uint32_t irq;
1071
1072 sc = (struct rk_pcie_softc *)arg;
1073 irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1074 if (irq & PCIE_CLIENT_INT_LOCAL) {
1075 irq = APB_RD4(sc, PCIE_CORE_INT_STATUS);
1076 APB_WR4(sc, PCIE_CORE_INT_STATUS, irq);
1077 APB_WR4(sc, PCIE_CLIENT_INT_STATUS, PCIE_CLIENT_INT_LOCAL);
1078
1079 device_printf(sc->dev, "'sys' interrupt received: 0x%04X\n",
1080 irq);
1081 }
1082
1083 return (FILTER_HANDLED);
1084 }
1085
1086 static int
rk_pcie_client_irq(void * arg)1087 rk_pcie_client_irq(void *arg)
1088 {
1089 struct rk_pcie_softc *sc;
1090 uint32_t irq;
1091
1092 sc = (struct rk_pcie_softc *)arg;
1093 irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1094 /* Clear causes handled by other interrups */
1095 irq &= ~PCIE_CLIENT_INT_LOCAL;
1096 irq &= ~PCIE_CLIENT_INT_LEGACY;
1097 APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
1098
1099 device_printf(sc->dev, "'client' interrupt received: 0x%04X\n", irq);
1100
1101 return (FILTER_HANDLED);
1102 }
1103
1104 static int
rk_pcie_legacy_irq(void * arg)1105 rk_pcie_legacy_irq(void *arg)
1106 {
1107 struct rk_pcie_softc *sc;
1108 uint32_t irq;
1109
1110 sc = (struct rk_pcie_softc *)arg;
1111 irq = APB_RD4(sc, PCIE_CLIENT_INT_STATUS);
1112 irq &= PCIE_CLIENT_INT_LEGACY;
1113 APB_WR4(sc, PCIE_CLIENT_INT_STATUS, irq);
1114
1115 /* all legacy interrupt are shared, do nothing */
1116 return (FILTER_STRAY);
1117 }
1118
1119 static bus_dma_tag_t
rk_pcie_get_dma_tag(device_t dev,device_t child)1120 rk_pcie_get_dma_tag(device_t dev, device_t child)
1121 {
1122 struct rk_pcie_softc *sc;
1123
1124 sc = device_get_softc(dev);
1125 return (sc->dmat);
1126 }
1127
1128 static int
rk_pcie_probe(device_t dev)1129 rk_pcie_probe(device_t dev)
1130 {
1131
1132 if (!ofw_bus_status_okay(dev))
1133 return (ENXIO);
1134
1135 if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
1136 return (ENXIO);
1137
1138 device_set_desc(dev, "Rockchip PCIe controller");
1139 return (BUS_PROBE_DEFAULT);
1140 }
1141
1142 static int
rk_pcie_attach(device_t dev)1143 rk_pcie_attach(device_t dev)
1144 {
1145 struct resource_map_request req;
1146 struct resource_map map;
1147 struct rk_pcie_softc *sc;
1148 uint32_t val;
1149 int rv, rid, max_speed;
1150
1151 sc = device_get_softc(dev);
1152 sc->dev = dev;
1153 sc->node = ofw_bus_get_node(dev);
1154
1155 mtx_init(&sc->mtx, "rk_pcie_mtx", NULL, MTX_DEF);
1156
1157 /* XXX Should not be this configurable ? */
1158 sc->bus_start = 0;
1159 sc->bus_end = 0x1F;
1160 sc->root_bus = sc->bus_start;
1161 sc->sub_bus = 1;
1162
1163 /* Read FDT properties */
1164 rv = rk_pcie_parse_fdt_resources(sc);
1165 if (rv != 0)
1166 goto out;
1167
1168 sc->coherent = OF_hasprop(sc->node, "dma-coherent");
1169 sc->no_l0s = OF_hasprop(sc->node, "aspm-no-l0s");
1170 rv = OF_getencprop(sc->node, "num-lanes", &sc->num_lanes,
1171 sizeof(sc->num_lanes));
1172 if (rv != sizeof(sc->num_lanes))
1173 sc->num_lanes = 1;
1174 if (sc->num_lanes != 1 && sc->num_lanes != 2 && sc->num_lanes != 4) {
1175 device_printf(dev,
1176 "invalid number of lanes: %d\n",sc->num_lanes);
1177 sc->num_lanes = 0;
1178 rv = ENXIO;
1179 goto out;
1180 }
1181
1182 rv = OF_getencprop(sc->node, "max-link-speed", &max_speed,
1183 sizeof(max_speed));
1184 if (rv != sizeof(max_speed) || max_speed != 1)
1185 sc->link_is_gen2 = true;
1186 else
1187 sc->link_is_gen2 = false;
1188
1189 rv = ofw_bus_find_string_index(sc->node, "reg-names", "axi-base", &rid);
1190 if (rv != 0) {
1191 device_printf(dev, "Cannot get 'axi-base' memory\n");
1192 rv = ENXIO;
1193 goto out;
1194 }
1195 sc->axi_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1196 RF_ACTIVE | RF_UNMAPPED);
1197 if (sc->axi_mem_res == NULL) {
1198 device_printf(dev, "Cannot allocate 'axi-base' (rid: %d)\n",
1199 rid);
1200 rv = ENXIO;
1201 goto out;
1202 }
1203 resource_init_map_request(&req);
1204 req.memattr = VM_MEMATTR_DEVICE_NP;
1205 rv = bus_map_resource(dev, SYS_RES_MEMORY, sc->axi_mem_res, &req,
1206 &map);
1207 if (rv != 0) {
1208 device_printf(dev, "Cannot map 'axi-base' (rid: %d)\n",
1209 rid);
1210 goto out;
1211 }
1212 rman_set_mapping(sc->axi_mem_res, &map);
1213
1214 rv = ofw_bus_find_string_index(sc->node, "reg-names", "apb-base", &rid);
1215 if (rv != 0) {
1216 device_printf(dev, "Cannot get 'apb-base' memory\n");
1217 rv = ENXIO;
1218 goto out;
1219 }
1220 sc->apb_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1221 RF_ACTIVE);
1222 if (sc->apb_mem_res == NULL) {
1223 device_printf(dev, "Cannot allocate 'apb-base' (rid: %d)\n",
1224 rid);
1225 rv = ENXIO;
1226 goto out;
1227 }
1228
1229 rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1230 "client", &rid);
1231 if (rv != 0) {
1232 device_printf(dev, "Cannot get 'client' IRQ\n");
1233 rv = ENXIO;
1234 goto out;
1235 }
1236 sc->client_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1237 RF_ACTIVE | RF_SHAREABLE);
1238 if (sc->client_irq_res == NULL) {
1239 device_printf(dev, "Cannot allocate 'client' IRQ resource\n");
1240 rv = ENXIO;
1241 goto out;
1242 }
1243
1244 rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1245 "legacy", &rid);
1246 if (rv != 0) {
1247 device_printf(dev, "Cannot get 'legacy' IRQ\n");
1248 rv = ENXIO;
1249 goto out;
1250 }
1251 sc->legacy_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1252 RF_ACTIVE | RF_SHAREABLE);
1253 if (sc->legacy_irq_res == NULL) {
1254 device_printf(dev, "Cannot allocate 'legacy' IRQ resource\n");
1255 rv = ENXIO;
1256 goto out;
1257 }
1258
1259 rv = ofw_bus_find_string_index(sc->node, "interrupt-names",
1260 "sys", &rid);
1261 if (rv != 0) {
1262 device_printf(dev, "Cannot get 'sys' IRQ\n");
1263 rv = ENXIO;
1264 goto out;
1265 }
1266 sc->sys_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1267 RF_ACTIVE | RF_SHAREABLE);
1268 if (sc->sys_irq_res == NULL) {
1269 device_printf(dev, "Cannot allocate 'sys' IRQ resource\n");
1270 rv = ENXIO;
1271 goto out;
1272 }
1273
1274 if (bootverbose)
1275 device_printf(dev, "Bus is%s cache-coherent\n",
1276 sc->coherent ? "" : " not");
1277 rv = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1278 1, 0, /* alignment, bounds */
1279 BUS_SPACE_MAXADDR, /* lowaddr */
1280 BUS_SPACE_MAXADDR, /* highaddr */
1281 NULL, NULL, /* filter, filterarg */
1282 BUS_SPACE_MAXSIZE, /* maxsize */
1283 BUS_SPACE_UNRESTRICTED, /* nsegments */
1284 BUS_SPACE_MAXSIZE, /* maxsegsize */
1285 sc->coherent ? BUS_DMA_COHERENT : 0, /* flags */
1286 NULL, NULL, /* lockfunc, lockarg */
1287 &sc->dmat);
1288 if (rv != 0)
1289 goto out;
1290
1291 rv = ofw_pcib_init(dev);
1292 if (rv != 0)
1293 goto out;
1294
1295 rv = rk_pcie_decode_ranges(sc, sc->ofw_pci.sc_range,
1296 sc->ofw_pci.sc_nrange);
1297 if (rv != 0)
1298 goto out_full;
1299 rv = rk_pcie_setup_hw(sc);
1300 if (rv != 0)
1301 goto out_full;
1302
1303 rv = rk_pcie_setup_sw(sc);
1304 if (rv != 0)
1305 goto out_full;
1306
1307 rv = bus_setup_intr(dev, sc->client_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1308 rk_pcie_client_irq, NULL, sc, &sc->client_irq_cookie);
1309 if (rv != 0) {
1310 device_printf(dev, "cannot setup client interrupt handler\n");
1311 rv = ENXIO;
1312 goto out_full;
1313 }
1314
1315 rv = bus_setup_intr(dev, sc->legacy_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1316 rk_pcie_legacy_irq, NULL, sc, &sc->legacy_irq_cookie);
1317 if (rv != 0) {
1318 device_printf(dev, "cannot setup client interrupt handler\n");
1319 rv = ENXIO;
1320 goto out_full;
1321 }
1322
1323 rv = bus_setup_intr(dev, sc->sys_irq_res, INTR_TYPE_BIO | INTR_MPSAFE,
1324 rk_pcie_sys_irq, NULL, sc, &sc->sys_irq_cookie);
1325 if (rv != 0) {
1326 device_printf(dev, "cannot setup client interrupt handler\n");
1327 rv = ENXIO;
1328 goto out_full;
1329 }
1330
1331 /* Enable interrupts */
1332 val =
1333 PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR |
1334 PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA |
1335 PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG |
1336 PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_INTA |
1337 PCIE_CLIENT_INT_INTB | PCIE_CLIENT_INT_INTC |
1338 PCIE_CLIENT_INT_INTD | PCIE_CLIENT_INT_PHY;
1339
1340 APB_WR4(sc, PCIE_CLIENT_INT_MASK, (val << 16) & ~val);
1341
1342 val =
1343 PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE |
1344 PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO |
1345 PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR |
1346 PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR |
1347 PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE |
1348 PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC |
1349 PCIE_CORE_INT_MMVC;
1350 APB_WR4(sc, PCIE_CORE_INT_MASK, ~(val));
1351
1352 val = APB_RD4(sc, PCIE_RC_CONFIG_LCS);
1353 val |= PCIEM_LINK_CTL_LBMIE | PCIEM_LINK_CTL_LABIE;
1354 APB_WR4(sc, PCIE_RC_CONFIG_LCS, val);
1355
1356 DELAY(250000);
1357 device_add_child(dev, "pci", -1);
1358 return (bus_generic_attach(dev));
1359
1360 out_full:
1361 bus_teardown_intr(dev, sc->sys_irq_res, sc->sys_irq_cookie);
1362 bus_teardown_intr(dev, sc->legacy_irq_res, sc->legacy_irq_cookie);
1363 bus_teardown_intr(dev, sc->client_irq_res, sc->client_irq_cookie);
1364 ofw_pcib_fini(dev);
1365 out:
1366 bus_dma_tag_destroy(sc->dmat);
1367 bus_free_resource(dev, SYS_RES_IRQ, sc->sys_irq_res);
1368 bus_free_resource(dev, SYS_RES_IRQ, sc->legacy_irq_res);
1369 bus_free_resource(dev, SYS_RES_IRQ, sc->client_irq_res);
1370 bus_free_resource(dev, SYS_RES_MEMORY, sc->apb_mem_res);
1371 bus_free_resource(dev, SYS_RES_MEMORY, sc->axi_mem_res);
1372 /* GPIO */
1373 gpio_pin_release(sc->gpio_ep);
1374 /* Phys */
1375 for (int i = 0; i < MAX_LANES; i++) {
1376 phy_release(sc->phys[i]);
1377 }
1378 /* Clocks */
1379 clk_release(sc->clk_aclk);
1380 clk_release(sc->clk_aclk_perf);
1381 clk_release(sc->clk_hclk);
1382 clk_release(sc->clk_pm);
1383 /* Resets */
1384 hwreset_release(sc->hwreset_core);
1385 hwreset_release(sc->hwreset_mgmt);
1386 hwreset_release(sc->hwreset_pipe);
1387 hwreset_release(sc->hwreset_pm);
1388 hwreset_release(sc->hwreset_aclk);
1389 hwreset_release(sc->hwreset_pclk);
1390 /* Regulators */
1391 regulator_release(sc->supply_12v);
1392 regulator_release(sc->supply_3v3);
1393 regulator_release(sc->supply_1v8);
1394 regulator_release(sc->supply_0v9);
1395 return (rv);
1396 }
1397
1398 static device_method_t rk_pcie_methods[] = {
1399 /* Device interface */
1400 DEVMETHOD(device_probe, rk_pcie_probe),
1401 DEVMETHOD(device_attach, rk_pcie_attach),
1402
1403 /* Bus interface */
1404 DEVMETHOD(bus_get_dma_tag, rk_pcie_get_dma_tag),
1405
1406 /* pcib interface */
1407 DEVMETHOD(pcib_read_config, rk_pcie_read_config),
1408 DEVMETHOD(pcib_write_config, rk_pcie_write_config),
1409 DEVMETHOD(pcib_route_interrupt, rk_pcie_route_interrupt),
1410 #ifdef RK_PCIE_ENABLE_MSI
1411 DEVMETHOD(pcib_alloc_msi, rk_pcie_alloc_msi),
1412 DEVMETHOD(pcib_release_msi, rk_pcie_release_msi),
1413 #endif
1414 #ifdef RK_PCIE_ENABLE_MSIX
1415 DEVMETHOD(pcib_alloc_msix, rk_pcie_alloc_msix),
1416 DEVMETHOD(pcib_release_msix, rk_pcie_release_msix),
1417 #endif
1418 DEVMETHOD(pcib_map_msi, rk_pcie_map_msi),
1419 DEVMETHOD(pcib_get_id, rk_pcie_get_id),
1420
1421 /* OFW bus interface */
1422 DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat),
1423 DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model),
1424 DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name),
1425 DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node),
1426 DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type),
1427
1428 DEVMETHOD_END
1429 };
1430
1431 DEFINE_CLASS_1(pcib, rk_pcie_driver, rk_pcie_methods,
1432 sizeof(struct rk_pcie_softc), ofw_pcib_driver);
1433 DRIVER_MODULE( rk_pcie, simplebus, rk_pcie_driver, NULL, NULL);
1434