1 /*-
2 * Copyright (c) 2017 Alexander Motin <[email protected]>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect
29 * two or more systems using a PCI-e links, providing remote memory access.
30 *
31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges.
32 */
33
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36
37 #include <sys/param.h>
38 #include <sys/kernel.h>
39 #include <sys/systm.h>
40 #include <sys/bus.h>
41 #include <sys/interrupt.h>
42 #include <sys/module.h>
43 #include <sys/rman.h>
44 #include <sys/sysctl.h>
45 #include <vm/vm.h>
46 #include <vm/pmap.h>
47 #include <machine/bus.h>
48 #include <machine/intr_machdep.h>
49 #include <machine/resource.h>
50 #include <dev/pci/pcireg.h>
51 #include <dev/pci/pcivar.h>
52
53 #include "../ntb.h"
54
55 #define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */
56 #define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */
57 #define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */
58 #define PLX_NUM_DB 16 /* There are 16 doorbells. */
59
60 struct ntb_plx_mw_info {
61 int mw_bar;
62 int mw_64bit;
63 int mw_rid;
64 struct resource *mw_res;
65 vm_paddr_t mw_pbase;
66 caddr_t mw_vbase;
67 vm_size_t mw_size;
68 vm_memattr_t mw_map_mode;
69 bus_addr_t mw_xlat_addr;
70 size_t mw_xlat_size;
71 };
72
73 struct ntb_plx_softc {
74 /* ntb.c context. Do not move! Must go first! */
75 void *ntb_store;
76
77 device_t dev;
78 struct resource *conf_res;
79 int conf_rid;
80 u_int ntx; /* NTx number within chip. */
81 u_int link; /* Link v/s Virtual side. */
82 u_int port; /* Port number within chip. */
83 u_int alut; /* A-LUT is enabled for NTx */
84
85 int int_rid;
86 struct resource *int_res;
87 void *int_tag;
88
89 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS];
90 int mw_count; /* Number of memory windows. */
91
92 int spad_count1; /* Number of standard spads. */
93 int spad_count2; /* Number of extra spads. */
94 uint32_t spad_off1; /* Offset of our spads. */
95 uint32_t spad_off2; /* Offset of our extra spads. */
96 uint32_t spad_offp1; /* Offset of peer spads. */
97 uint32_t spad_offp2; /* Offset of peer extra spads. */
98
99 /* Parameters of window shared with peer config access in B2B mode. */
100 int b2b_mw; /* Shared window number. */
101 uint64_t b2b_off; /* Offset in shared window. */
102 };
103
104 #define PLX_NT0_BASE 0x3E000
105 #define PLX_NT1_BASE 0x3C000
106 #define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE)
107 #define PLX_NTX_LINK_OFFSET 0x01000
108
109 /* Bases of NTx our/peer interface registers */
110 #define PLX_NTX_OUR_BASE(sc) \
111 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0))
112 #define PLX_NTX_PEER_BASE(sc) \
113 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET))
114
115 /* Read/write NTx our interface registers */
116 #define NTX_READ(sc, reg) \
117 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg))
118 #define NTX_WRITE(sc, reg, val) \
119 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val))
120
121 /* Read/write NTx peer interface registers */
122 #define PNTX_READ(sc, reg) \
123 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg))
124 #define PNTX_WRITE(sc, reg, val) \
125 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val))
126
127 /* Read/write B2B NTx registers */
128 #define BNTX_READ(sc, reg) \
129 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \
130 PLX_NTX_BASE(sc) + (reg))
131 #define BNTX_WRITE(sc, reg, val) \
132 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \
133 PLX_NTX_BASE(sc) + (reg), (val))
134
135 #define PLX_PORT_BASE(p) ((p) << 12)
136 #define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7)
137
138 #define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208)
139
140 static int ntb_plx_init(device_t dev);
141 static int ntb_plx_detach(device_t dev);
142 static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx);
143
144 static int
ntb_plx_probe(device_t dev)145 ntb_plx_probe(device_t dev)
146 {
147
148 switch (pci_get_devid(dev)) {
149 case 0x87a010b5:
150 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link");
151 return (BUS_PROBE_DEFAULT);
152 case 0x87a110b5:
153 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link");
154 return (BUS_PROBE_DEFAULT);
155 case 0x87b010b5:
156 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual");
157 return (BUS_PROBE_DEFAULT);
158 case 0x87b110b5:
159 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual");
160 return (BUS_PROBE_DEFAULT);
161 }
162 return (ENXIO);
163 }
164
165 static int
ntb_plx_init(device_t dev)166 ntb_plx_init(device_t dev)
167 {
168 struct ntb_plx_softc *sc = device_get_softc(dev);
169 struct ntb_plx_mw_info *mw;
170 uint64_t val64;
171 int i;
172 uint32_t val;
173
174 if (sc->b2b_mw >= 0) {
175 /* Set peer BAR0/1 size and address for B2B NTx access. */
176 mw = &sc->mw_info[sc->b2b_mw];
177 if (mw->mw_64bit) {
178 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */
179 val64 = 0x2000000000000000 * mw->mw_bar | 0x4;
180 PNTX_WRITE(sc, PCIR_BAR(0), val64);
181 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32);
182 } else {
183 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */
184 val = 0x20000000 * mw->mw_bar;
185 PNTX_WRITE(sc, PCIR_BAR(0), val);
186 }
187
188 /* Set Virtual to Link address translation for B2B. */
189 for (i = 0; i < sc->mw_count; i++) {
190 mw = &sc->mw_info[i];
191 if (mw->mw_64bit) {
192 val64 = 0x2000000000000000 * mw->mw_bar;
193 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64);
194 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32);
195 } else {
196 val = 0x20000000 * mw->mw_bar;
197 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val);
198 }
199 }
200
201 /* Make sure Virtual to Link A-LUT is disabled. */
202 if (sc->alut)
203 PNTX_WRITE(sc, 0xc94, 0);
204
205 /* Enable all Link Interface LUT entries for peer. */
206 for (i = 0; i < 32; i += 2) {
207 PNTX_WRITE(sc, 0xdb4 + i * 2,
208 0x00010001 | ((i + 1) << 19) | (i << 3));
209 }
210 }
211
212 /*
213 * Enable Virtual Interface LUT entry 0 for 0:0.*.
214 * entry 1 for our Requester ID reported by the chip,
215 * entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines.
216 * XXX: Its a hack, we can't know all DMA engines, but this covers all
217 * I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw.
218 */
219 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001;
220 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val);
221 NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021);
222 NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021);
223
224 /* Set Link to Virtual address translation. */
225 for (i = 0; i < sc->mw_count; i++) {
226 mw = &sc->mw_info[i];
227 if (mw->mw_xlat_size != 0)
228 ntb_plx_mw_set_trans_internal(dev, i);
229 }
230
231 pci_enable_busmaster(dev);
232 if (sc->b2b_mw >= 0)
233 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
234
235 return (0);
236 }
237
238 static void
ntb_plx_isr(void * arg)239 ntb_plx_isr(void *arg)
240 {
241 device_t dev = arg;
242 struct ntb_plx_softc *sc = device_get_softc(dev);
243 uint32_t val;
244
245 ntb_db_event((device_t)arg, 0);
246
247 if (sc->link) /* Link Interface has no Link Error registers. */
248 return;
249
250 val = NTX_READ(sc, 0xfe0);
251 if (val == 0)
252 return;
253 NTX_WRITE(sc, 0xfe0, val);
254 if (val & 1)
255 device_printf(dev, "Correctable Error\n");
256 if (val & 2)
257 device_printf(dev, "Uncorrectable Error\n");
258 if (val & 4) {
259 /* DL_Down resets link side registers, have to reinit. */
260 ntb_plx_init(dev);
261 ntb_link_event(dev);
262 }
263 if (val & 8)
264 device_printf(dev, "Uncorrectable Error Message Drop\n");
265 }
266
267 static int
ntb_plx_setup_intr(device_t dev)268 ntb_plx_setup_intr(device_t dev)
269 {
270 struct ntb_plx_softc *sc = device_get_softc(dev);
271 int error;
272
273 /*
274 * XXX: This hardware supports MSI, but I found it unusable.
275 * It generates new MSI only when doorbell register goes from
276 * zero, but does not generate it when another bit is set or on
277 * partial clear. It makes operation very racy and unreliable.
278 * The data book mentions some mask juggling magic to workaround
279 * that, but I failed to make it work.
280 */
281 sc->int_rid = 0;
282 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
283 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE);
284 if (sc->int_res == NULL) {
285 device_printf(dev, "bus_alloc_resource failed\n");
286 return (ENOMEM);
287 }
288 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC,
289 NULL, ntb_plx_isr, dev, &sc->int_tag);
290 if (error != 0) {
291 device_printf(dev, "bus_setup_intr failed: %d\n", error);
292 return (error);
293 }
294
295 if (!sc->link) { /* Link Interface has no Link Error registers. */
296 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */
297 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */
298 }
299 return (0);
300 }
301
302 static void
ntb_plx_teardown_intr(device_t dev)303 ntb_plx_teardown_intr(device_t dev)
304 {
305 struct ntb_plx_softc *sc = device_get_softc(dev);
306
307 if (!sc->link) /* Link Interface has no Link Error registers. */
308 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */
309
310 if (sc->int_res) {
311 bus_teardown_intr(dev, sc->int_res, sc->int_tag);
312 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid,
313 sc->int_res);
314 }
315 }
316
317 static int
ntb_plx_attach(device_t dev)318 ntb_plx_attach(device_t dev)
319 {
320 struct ntb_plx_softc *sc = device_get_softc(dev);
321 struct ntb_plx_mw_info *mw;
322 int error = 0, i;
323 uint32_t val;
324 char buf[32];
325
326 /* Identify what we are (what side of what NTx). */
327 sc->dev = dev;
328 val = pci_read_config(dev, 0xc8c, 4);
329 sc->ntx = (val & 1) != 0;
330 sc->link = (val & 0x80000000) != 0;
331
332 /* Get access to whole 256KB of chip configuration space via BAR0/1. */
333 sc->conf_rid = PCIR_BAR(0);
334 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
335 &sc->conf_rid, RF_ACTIVE);
336 if (sc->conf_res == NULL) {
337 device_printf(dev, "Can't allocate configuration BAR.\n");
338 return (ENXIO);
339 }
340
341 /* Identify chip port we are connected to. */
342 val = bus_read_4(sc->conf_res, 0x360);
343 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f;
344
345 /* Detect A-LUT enable and size. */
346 val >>= 30;
347 sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0);
348 if (sc->alut)
349 device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut);
350
351 /* Find configured memory windows at BAR2-5. */
352 sc->mw_count = 0;
353 for (i = 2; i <= 5; i++) {
354 mw = &sc->mw_info[sc->mw_count];
355 mw->mw_bar = i;
356 mw->mw_rid = PCIR_BAR(mw->mw_bar);
357 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
358 &mw->mw_rid, RF_ACTIVE);
359 if (mw->mw_res == NULL)
360 continue;
361 mw->mw_pbase = rman_get_start(mw->mw_res);
362 mw->mw_size = rman_get_size(mw->mw_res);
363 mw->mw_vbase = rman_get_virtual(mw->mw_res);
364 mw->mw_map_mode = VM_MEMATTR_UNCACHEABLE;
365 sc->mw_count++;
366
367 /* Skip over adjacent BAR for 64-bit BARs. */
368 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4);
369 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) {
370 mw->mw_64bit = 1;
371 i++;
372 }
373 }
374
375 /* Try to identify B2B mode. */
376 i = 1;
377 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev),
378 device_get_unit(dev));
379 TUNABLE_INT_FETCH(buf, &i);
380 if (sc->link) {
381 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n");
382 sc->b2b_mw = -1;
383 } else if (i == 0) {
384 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n");
385 sc->b2b_mw = -1;
386 } else {
387 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n");
388
389 /* We need at least one memory window for B2B peer access. */
390 if (sc->mw_count == 0) {
391 device_printf(dev, "No memory window BARs enabled.\n");
392 error = ENXIO;
393 goto out;
394 }
395 sc->b2b_mw = sc->mw_count - 1;
396
397 /* Use half of the window for B2B, but no less then 1MB. */
398 mw = &sc->mw_info[sc->b2b_mw];
399 if (mw->mw_size >= 2 * 1024 * 1024)
400 sc->b2b_off = mw->mw_size / 2;
401 else
402 sc->b2b_off = 0;
403 }
404
405 /*
406 * Use Physical Layer User Test Pattern as additional scratchpad.
407 * Make sure they are present and enabled by writing to them.
408 * XXX: Its a hack, but standard 8 registers are not enough.
409 */
410 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c;
411 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c;
412 if (sc->b2b_mw >= 0) {
413 /* In NTB-to-NTB mode each side has own scratchpads. */
414 sc->spad_count1 = PLX_NUM_SPAD;
415 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678);
416 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678)
417 sc->spad_count2 = PLX_NUM_SPAD_PATT;
418 } else {
419 /* Otherwise we have share scratchpads with the peer. */
420 if (sc->link) {
421 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4;
422 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4;
423 } else {
424 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4;
425 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4;
426 }
427 sc->spad_count1 = PLX_NUM_SPAD / 2;
428 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678);
429 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678)
430 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2;
431 }
432
433 /* Apply static part of NTB configuration. */
434 ntb_plx_init(dev);
435
436 /* Allocate and setup interrupts. */
437 error = ntb_plx_setup_intr(dev);
438 if (error)
439 goto out;
440
441 /* Attach children to this controller */
442 error = ntb_register_device(dev);
443
444 out:
445 if (error != 0)
446 ntb_plx_detach(dev);
447 return (error);
448 }
449
450 static int
ntb_plx_detach(device_t dev)451 ntb_plx_detach(device_t dev)
452 {
453 struct ntb_plx_softc *sc = device_get_softc(dev);
454 struct ntb_plx_mw_info *mw;
455 int i;
456
457 /* Detach & delete all children */
458 ntb_unregister_device(dev);
459
460 /* Disable and free interrupts. */
461 ntb_plx_teardown_intr(dev);
462
463 /* Free memory resources. */
464 for (i = 0; i < sc->mw_count; i++) {
465 mw = &sc->mw_info[i];
466 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid,
467 mw->mw_res);
468 }
469 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res);
470 return (0);
471 }
472
473 static int
ntb_plx_port_number(device_t dev)474 ntb_plx_port_number(device_t dev)
475 {
476 struct ntb_plx_softc *sc = device_get_softc(dev);
477
478 return (sc->link ? 1 : 0);
479 }
480
481 static int
ntb_plx_peer_port_count(device_t dev)482 ntb_plx_peer_port_count(device_t dev)
483 {
484
485 return (1);
486 }
487
488 static int
ntb_plx_peer_port_number(device_t dev,int pidx)489 ntb_plx_peer_port_number(device_t dev, int pidx)
490 {
491 struct ntb_plx_softc *sc = device_get_softc(dev);
492
493 if (pidx != 0)
494 return (-EINVAL);
495
496 return (sc->link ? 0 : 1);
497 }
498
499 static int
ntb_plx_peer_port_idx(device_t dev,int port)500 ntb_plx_peer_port_idx(device_t dev, int port)
501 {
502 int peer_port;
503
504 peer_port = ntb_plx_peer_port_number(dev, 0);
505 if (peer_port == -EINVAL || port != peer_port)
506 return (-EINVAL);
507
508 return (0);
509 }
510
511 static bool
ntb_plx_link_is_up(device_t dev,enum ntb_speed * speed,enum ntb_width * width)512 ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width)
513 {
514 uint16_t link;
515
516 link = pcie_read_config(dev, PCIER_LINK_STA, 2);
517 if (speed != NULL)
518 *speed = (link & PCIEM_LINK_STA_SPEED);
519 if (width != NULL)
520 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4;
521 return ((link & PCIEM_LINK_STA_WIDTH) != 0);
522 }
523
524 static int
ntb_plx_link_enable(device_t dev,enum ntb_speed speed __unused,enum ntb_width width __unused)525 ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused,
526 enum ntb_width width __unused)
527 {
528 struct ntb_plx_softc *sc = device_get_softc(dev);
529 uint32_t reg, val;
530
531 /* The fact that we see the Link Interface means link is enabled. */
532 if (sc->link) {
533 ntb_link_event(dev);
534 return (0);
535 }
536
537 reg = PLX_PORT_CONTROL(sc);
538 val = bus_read_4(sc->conf_res, reg);
539 if ((val & (1 << (sc->port & 7))) == 0) {
540 /* If already enabled, generate fake link event and exit. */
541 ntb_link_event(dev);
542 return (0);
543 }
544 val &= ~(1 << (sc->port & 7));
545 bus_write_4(sc->conf_res, reg, val);
546 return (0);
547 }
548
549 static int
ntb_plx_link_disable(device_t dev)550 ntb_plx_link_disable(device_t dev)
551 {
552 struct ntb_plx_softc *sc = device_get_softc(dev);
553 uint32_t reg, val;
554
555 /* Link disable for Link Interface would be suicidal. */
556 if (sc->link)
557 return (0);
558
559 reg = PLX_PORT_CONTROL(sc);
560 val = bus_read_4(sc->conf_res, reg);
561 val |= (1 << (sc->port & 7));
562 bus_write_4(sc->conf_res, reg, val);
563 return (0);
564 }
565
566 static bool
ntb_plx_link_enabled(device_t dev)567 ntb_plx_link_enabled(device_t dev)
568 {
569 struct ntb_plx_softc *sc = device_get_softc(dev);
570 uint32_t reg, val;
571
572 /* The fact that we see the Link Interface means link is enabled. */
573 if (sc->link)
574 return (TRUE);
575
576 reg = PLX_PORT_CONTROL(sc);
577 val = bus_read_4(sc->conf_res, reg);
578 return ((val & (1 << (sc->port & 7))) == 0);
579 }
580
581 static uint8_t
ntb_plx_mw_count(device_t dev)582 ntb_plx_mw_count(device_t dev)
583 {
584 struct ntb_plx_softc *sc = device_get_softc(dev);
585
586 if (sc->b2b_mw >= 0 && sc->b2b_off == 0)
587 return (sc->mw_count - 1); /* B2B consumed whole window. */
588 return (sc->mw_count);
589 }
590
591 static int
ntb_plx_mw_get_range(device_t dev,unsigned mw_idx,vm_paddr_t * base,caddr_t * vbase,size_t * size,size_t * align,size_t * align_size,bus_addr_t * plimit)592 ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base,
593 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
594 bus_addr_t *plimit)
595 {
596 struct ntb_plx_softc *sc = device_get_softc(dev);
597 struct ntb_plx_mw_info *mw;
598 size_t off;
599
600 if (mw_idx >= sc->mw_count)
601 return (EINVAL);
602 off = 0;
603 if (mw_idx == sc->b2b_mw) {
604 KASSERT(sc->b2b_off != 0,
605 ("user shouldn't get non-shared b2b mw"));
606 off = sc->b2b_off;
607 }
608 mw = &sc->mw_info[mw_idx];
609
610 /* Local to remote memory window parameters. */
611 if (base != NULL)
612 *base = mw->mw_pbase + off;
613 if (vbase != NULL)
614 *vbase = mw->mw_vbase + off;
615 if (size != NULL)
616 *size = mw->mw_size - off;
617
618 /*
619 * Remote to local memory window translation address alignment.
620 * Translation address has to be aligned to the BAR size, but A-LUT
621 * entries re-map addresses can be aligned to 1/128 or 1/256 of it.
622 * XXX: In B2B mode we can change BAR size (and so alignmet) live,
623 * but there is no way to report it here, so report safe value.
624 */
625 if (align != NULL) {
626 if (sc->alut && mw->mw_bar == 2)
627 *align = (mw->mw_size - off) / 128 / sc->alut;
628 else
629 *align = mw->mw_size - off;
630 }
631
632 /*
633 * Remote to local memory window size alignment.
634 * The chip has no limit registers, but A-LUT, when available, allows
635 * access control with granularity of 1/128 or 1/256 of the BAR size.
636 * XXX: In B2B case we can change BAR size live, but there is no way
637 * to report it, so report half of the BAR size, that should be safe.
638 * In non-B2B case there is no control at all, so report the BAR size.
639 */
640 if (align_size != NULL) {
641 if (sc->alut && mw->mw_bar == 2)
642 *align_size = (mw->mw_size - off) / 128 / sc->alut;
643 else if (sc->b2b_mw >= 0)
644 *align_size = (mw->mw_size - off) / 2;
645 else
646 *align_size = mw->mw_size - off;
647 }
648
649 /* Remote to local memory window translation address upper limit. */
650 if (plimit != NULL)
651 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR :
652 BUS_SPACE_MAXADDR_32BIT;
653 return (0);
654 }
655
656 static int
ntb_plx_mw_set_trans_internal(device_t dev,unsigned mw_idx)657 ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx)
658 {
659 struct ntb_plx_softc *sc = device_get_softc(dev);
660 struct ntb_plx_mw_info *mw;
661 uint64_t addr, eaddr, off, size, bsize, esize, val64;
662 uint32_t val;
663 int i;
664
665 mw = &sc->mw_info[mw_idx];
666 addr = mw->mw_xlat_addr;
667 size = mw->mw_xlat_size;
668 off = 0;
669 if (mw_idx == sc->b2b_mw) {
670 off = sc->b2b_off;
671 KASSERT(off != 0, ("user shouldn't get non-shared b2b mw"));
672
673 /*
674 * While generally we can set any BAR size on link side,
675 * for B2B shared window we can't go above preconfigured
676 * size due to BAR address alignment requirements.
677 */
678 if (size > mw->mw_size - off)
679 return (EINVAL);
680 }
681
682 if (size > 0) {
683 /* Round BAR size to next power of 2 or at least 1MB. */
684 bsize = size;
685 if (!powerof2(bsize))
686 bsize = 1LL << flsll(bsize);
687 if (bsize < 1024 * 1024)
688 bsize = 1024 * 1024;
689
690 /* A-LUT has 128 or 256 times better granularity. */
691 esize = bsize;
692 if (sc->alut && mw->mw_bar == 2)
693 esize /= 128 * sc->alut;
694
695 /* addr should be aligned to BAR or A-LUT element size. */
696 if ((addr & (esize - 1)) != 0)
697 return (EINVAL);
698 } else
699 esize = bsize = 0;
700
701 if (mw->mw_64bit) {
702 if (sc->b2b_mw >= 0) {
703 /* Set Link Interface BAR size and enable/disable it. */
704 val64 = 0;
705 if (bsize > 0)
706 val64 = (~(bsize - 1) & ~0xfffff);
707 val64 |= 0xc;
708 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64);
709 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32);
710
711 /* Set Link Interface BAR address. */
712 val64 = 0x2000000000000000 * mw->mw_bar + off;
713 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64);
714 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32);
715 }
716
717 /* Set Virtual Interface BARs address translation */
718 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr);
719 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32);
720 } else {
721 /* Make sure we fit into 32-bit address space. */
722 if ((addr & UINT32_MAX) != addr)
723 return (ERANGE);
724 if (((addr + bsize) & UINT32_MAX) != (addr + bsize))
725 return (ERANGE);
726
727 if (sc->b2b_mw >= 0) {
728 /* Set Link Interface BAR size and enable/disable it. */
729 val = 0;
730 if (bsize > 0)
731 val = (~(bsize - 1) & ~0xfffff);
732 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val);
733
734 /* Set Link Interface BAR address. */
735 val64 = 0x20000000 * mw->mw_bar + off;
736 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64);
737 }
738
739 /* Set Virtual Interface BARs address translation */
740 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr);
741 }
742
743 /* Configure and enable Link to Virtual A-LUT if we need it. */
744 if (sc->alut && mw->mw_bar == 2 &&
745 ((addr & (bsize - 1)) != 0 || size != bsize)) {
746 eaddr = addr;
747 for (i = 0; i < 128 * sc->alut; i++) {
748 val = sc->link ? 0 : 1;
749 if (sc->alut == 1)
750 val += 2 * sc->ntx;
751 val *= 0x1000 * sc->alut;
752 val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0);
753 bus_write_4(sc->conf_res, val, eaddr);
754 bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32);
755 bus_write_4(sc->conf_res, val + 0x800,
756 (eaddr < addr + size) ? 0x3 : 0);
757 eaddr += esize;
758 }
759 NTX_WRITE(sc, 0xc94, 0x10000000);
760 } else if (sc->alut && mw->mw_bar == 2)
761 NTX_WRITE(sc, 0xc94, 0);
762
763 return (0);
764 }
765
766 static int
ntb_plx_mw_set_trans(device_t dev,unsigned mw_idx,bus_addr_t addr,size_t size)767 ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size)
768 {
769 struct ntb_plx_softc *sc = device_get_softc(dev);
770 struct ntb_plx_mw_info *mw;
771
772 if (mw_idx >= sc->mw_count)
773 return (EINVAL);
774 mw = &sc->mw_info[mw_idx];
775 mw->mw_xlat_addr = addr;
776 mw->mw_xlat_size = size;
777 return (ntb_plx_mw_set_trans_internal(dev, mw_idx));
778 }
779
780 static int
ntb_plx_mw_clear_trans(device_t dev,unsigned mw_idx)781 ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx)
782 {
783
784 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0));
785 }
786
787 static int
ntb_plx_mw_get_wc(device_t dev,unsigned idx,vm_memattr_t * mode)788 ntb_plx_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode)
789 {
790 struct ntb_plx_softc *sc = device_get_softc(dev);
791 struct ntb_plx_mw_info *mw;
792
793 if (idx >= sc->mw_count)
794 return (EINVAL);
795 mw = &sc->mw_info[idx];
796 *mode = mw->mw_map_mode;
797 return (0);
798 }
799
800 static int
ntb_plx_mw_set_wc(device_t dev,unsigned idx,vm_memattr_t mode)801 ntb_plx_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode)
802 {
803 struct ntb_plx_softc *sc = device_get_softc(dev);
804 struct ntb_plx_mw_info *mw;
805 uint64_t off;
806 int rc;
807
808 if (idx >= sc->mw_count)
809 return (EINVAL);
810 mw = &sc->mw_info[idx];
811 if (mw->mw_map_mode == mode)
812 return (0);
813
814 off = 0;
815 if (idx == sc->b2b_mw) {
816 KASSERT(sc->b2b_off != 0,
817 ("user shouldn't get non-shared b2b mw"));
818 off = sc->b2b_off;
819 }
820
821 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off,
822 mw->mw_size - off, mode);
823 if (rc == 0)
824 mw->mw_map_mode = mode;
825 return (rc);
826 }
827
828 static uint8_t
ntb_plx_spad_count(device_t dev)829 ntb_plx_spad_count(device_t dev)
830 {
831 struct ntb_plx_softc *sc = device_get_softc(dev);
832
833 return (sc->spad_count1 + sc->spad_count2);
834 }
835
836 static int
ntb_plx_spad_write(device_t dev,unsigned int idx,uint32_t val)837 ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val)
838 {
839 struct ntb_plx_softc *sc = device_get_softc(dev);
840 u_int off;
841
842 if (idx >= sc->spad_count1 + sc->spad_count2)
843 return (EINVAL);
844
845 if (idx < sc->spad_count1)
846 off = sc->spad_off1 + idx * 4;
847 else
848 off = sc->spad_off2 + (idx - sc->spad_count1) * 4;
849 bus_write_4(sc->conf_res, off, val);
850 return (0);
851 }
852
853 static void
ntb_plx_spad_clear(device_t dev)854 ntb_plx_spad_clear(device_t dev)
855 {
856 struct ntb_plx_softc *sc = device_get_softc(dev);
857 int i;
858
859 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++)
860 ntb_plx_spad_write(dev, i, 0);
861 }
862
863 static int
ntb_plx_spad_read(device_t dev,unsigned int idx,uint32_t * val)864 ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val)
865 {
866 struct ntb_plx_softc *sc = device_get_softc(dev);
867 u_int off;
868
869 if (idx >= sc->spad_count1 + sc->spad_count2)
870 return (EINVAL);
871
872 if (idx < sc->spad_count1)
873 off = sc->spad_off1 + idx * 4;
874 else
875 off = sc->spad_off2 + (idx - sc->spad_count1) * 4;
876 *val = bus_read_4(sc->conf_res, off);
877 return (0);
878 }
879
880 static int
ntb_plx_peer_spad_write(device_t dev,unsigned int idx,uint32_t val)881 ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val)
882 {
883 struct ntb_plx_softc *sc = device_get_softc(dev);
884 u_int off;
885
886 if (idx >= sc->spad_count1 + sc->spad_count2)
887 return (EINVAL);
888
889 if (idx < sc->spad_count1)
890 off = sc->spad_offp1 + idx * 4;
891 else
892 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4;
893 if (sc->b2b_mw >= 0)
894 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val);
895 else
896 bus_write_4(sc->conf_res, off, val);
897 return (0);
898 }
899
900 static int
ntb_plx_peer_spad_read(device_t dev,unsigned int idx,uint32_t * val)901 ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val)
902 {
903 struct ntb_plx_softc *sc = device_get_softc(dev);
904 u_int off;
905
906 if (idx >= sc->spad_count1 + sc->spad_count2)
907 return (EINVAL);
908
909 if (idx < sc->spad_count1)
910 off = sc->spad_offp1 + idx * 4;
911 else
912 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4;
913 if (sc->b2b_mw >= 0)
914 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off);
915 else
916 *val = bus_read_4(sc->conf_res, off);
917 return (0);
918 }
919
920 static uint64_t
ntb_plx_db_valid_mask(device_t dev)921 ntb_plx_db_valid_mask(device_t dev)
922 {
923
924 return ((1LL << PLX_NUM_DB) - 1);
925 }
926
927 static int
ntb_plx_db_vector_count(device_t dev)928 ntb_plx_db_vector_count(device_t dev)
929 {
930
931 return (1);
932 }
933
934 static uint64_t
ntb_plx_db_vector_mask(device_t dev,uint32_t vector)935 ntb_plx_db_vector_mask(device_t dev, uint32_t vector)
936 {
937
938 if (vector > 0)
939 return (0);
940 return ((1LL << PLX_NUM_DB) - 1);
941 }
942
943 static void
ntb_plx_db_clear(device_t dev,uint64_t bits)944 ntb_plx_db_clear(device_t dev, uint64_t bits)
945 {
946 struct ntb_plx_softc *sc = device_get_softc(dev);
947
948 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits);
949 }
950
951 static void
ntb_plx_db_clear_mask(device_t dev,uint64_t bits)952 ntb_plx_db_clear_mask(device_t dev, uint64_t bits)
953 {
954 struct ntb_plx_softc *sc = device_get_softc(dev);
955
956 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits);
957 }
958
959 static uint64_t
ntb_plx_db_read(device_t dev)960 ntb_plx_db_read(device_t dev)
961 {
962 struct ntb_plx_softc *sc = device_get_softc(dev);
963
964 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c));
965 }
966
967 static void
ntb_plx_db_set_mask(device_t dev,uint64_t bits)968 ntb_plx_db_set_mask(device_t dev, uint64_t bits)
969 {
970 struct ntb_plx_softc *sc = device_get_softc(dev);
971
972 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits);
973 }
974
975 static int
ntb_plx_peer_db_addr(device_t dev,bus_addr_t * db_addr,vm_size_t * db_size)976 ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size)
977 {
978 struct ntb_plx_softc *sc = device_get_softc(dev);
979 struct ntb_plx_mw_info *mw;
980
981 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL"));
982
983 if (sc->b2b_mw >= 0) {
984 mw = &sc->mw_info[sc->b2b_mw];
985 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c;
986 } else {
987 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc);
988 *db_addr += sc->link ? 0xc4c : 0xc5c;
989 }
990 *db_size = 4;
991 return (0);
992 }
993
994 static void
ntb_plx_peer_db_set(device_t dev,uint64_t bit)995 ntb_plx_peer_db_set(device_t dev, uint64_t bit)
996 {
997 struct ntb_plx_softc *sc = device_get_softc(dev);
998
999 if (sc->b2b_mw >= 0)
1000 BNTX_WRITE(sc, 0xc4c, bit);
1001 else
1002 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit);
1003 }
1004
1005 static device_method_t ntb_plx_methods[] = {
1006 /* Device interface */
1007 DEVMETHOD(device_probe, ntb_plx_probe),
1008 DEVMETHOD(device_attach, ntb_plx_attach),
1009 DEVMETHOD(device_detach, ntb_plx_detach),
1010 /* Bus interface */
1011 DEVMETHOD(bus_child_location_str, ntb_child_location_str),
1012 DEVMETHOD(bus_print_child, ntb_print_child),
1013 /* NTB interface */
1014 DEVMETHOD(ntb_port_number, ntb_plx_port_number),
1015 DEVMETHOD(ntb_peer_port_count, ntb_plx_peer_port_count),
1016 DEVMETHOD(ntb_peer_port_number, ntb_plx_peer_port_number),
1017 DEVMETHOD(ntb_peer_port_idx, ntb_plx_peer_port_idx),
1018 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up),
1019 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable),
1020 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable),
1021 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled),
1022 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count),
1023 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range),
1024 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans),
1025 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans),
1026 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc),
1027 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc),
1028 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count),
1029 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear),
1030 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write),
1031 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read),
1032 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write),
1033 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read),
1034 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask),
1035 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count),
1036 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask),
1037 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear),
1038 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask),
1039 DEVMETHOD(ntb_db_read, ntb_plx_db_read),
1040 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask),
1041 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr),
1042 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set),
1043 DEVMETHOD_END
1044 };
1045
1046 static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods,
1047 sizeof(struct ntb_plx_softc));
1048 DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL);
1049 MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1);
1050 MODULE_VERSION(ntb_hw_plx, 1);
1051