1 /*-
2 * Copyright (c) 2016-2017 Alexander Motin <[email protected]>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 #include <sys/param.h>
29 #include <sys/kernel.h>
30 #include <sys/systm.h>
31 #include <sys/bus.h>
32 #include <sys/rmlock.h>
33 #include <sys/malloc.h>
34 #include <sys/module.h>
35 #include <sys/sbuf.h>
36 #include <sys/sysctl.h>
37
38 #include "ntb.h"
39
40 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
41 "NTB sysctls");
42
43 struct ntb_child {
44 device_t dev;
45 int function;
46 int enabled;
47 int mwoff;
48 int mwcnt;
49 int spadoff;
50 int spadcnt;
51 int dboff;
52 int dbcnt;
53 uint64_t dbmask;
54 void *ctx;
55 const struct ntb_ctx_ops *ctx_ops;
56 struct rmlock ctx_lock;
57 struct ntb_child *next;
58 };
59
60 int
ntb_register_device(device_t dev)61 ntb_register_device(device_t dev)
62 {
63 struct ntb_child **cpp = device_get_softc(dev);
64 struct ntb_child *nc;
65 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt;
66 char cfg[128] = "";
67 char buf[32];
68 char *n, *np, *c, *p, *name;
69
70 mwu = 0;
71 mwt = NTB_MW_COUNT(dev);
72 spadu = 0;
73 spadt = NTB_SPAD_COUNT(dev);
74 dbu = 0;
75 dbt = flsll(NTB_DB_VALID_MASK(dev));
76
77 device_printf(dev, "%d memory windows, %d scratchpads, "
78 "%d doorbells\n", mwt, spadt, dbt);
79
80 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev),
81 device_get_unit(dev));
82 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg));
83 n = cfg;
84 i = 0;
85 while ((c = strsep(&n, ",")) != NULL) {
86 np = c;
87 name = strsep(&np, ":");
88 if (name != NULL && name[0] == 0)
89 name = NULL;
90 p = strsep(&np, ":");
91 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu;
92 p = strsep(&np, ":");
93 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu;
94 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu;
95
96 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) {
97 device_printf(dev, "Not enough resources for config\n");
98 break;
99 }
100
101 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO);
102 nc->function = i;
103 nc->mwoff = mwu;
104 nc->mwcnt = mw;
105 nc->spadoff = spadu;
106 nc->spadcnt = spad;
107 nc->dboff = dbu;
108 nc->dbcnt = db;
109 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db));
110 rm_init(&nc->ctx_lock, "ntb ctx");
111 nc->dev = device_add_child(dev, name, -1);
112 if (nc->dev == NULL) {
113 ntb_unregister_device(dev);
114 return (ENOMEM);
115 }
116 device_set_ivars(nc->dev, nc);
117 *cpp = nc;
118 cpp = &nc->next;
119
120 if (bootverbose) {
121 device_printf(dev, "%d \"%s\":", i, name);
122 if (mw > 0) {
123 printf(" memory windows %d", mwu);
124 if (mw > 1)
125 printf("-%d", mwu + mw - 1);
126 }
127 if (spad > 0) {
128 printf(" scratchpads %d", spadu);
129 if (spad > 1)
130 printf("-%d", spadu + spad - 1);
131 }
132 if (db > 0) {
133 printf(" doorbells %d", dbu);
134 if (db > 1)
135 printf("-%d", dbu + db - 1);
136 }
137 printf("\n");
138 }
139
140 mwu += mw;
141 spadu += spad;
142 dbu += db;
143 i++;
144 }
145
146 bus_generic_attach(dev);
147 return (0);
148 }
149
150 int
ntb_unregister_device(device_t dev)151 ntb_unregister_device(device_t dev)
152 {
153 struct ntb_child **cpp = device_get_softc(dev);
154 struct ntb_child *nc;
155 int error = 0;
156
157 while ((nc = *cpp) != NULL) {
158 *cpp = (*cpp)->next;
159 error = device_delete_child(dev, nc->dev);
160 if (error)
161 break;
162 rm_destroy(&nc->ctx_lock);
163 free(nc, M_DEVBUF);
164 }
165 return (error);
166 }
167
168 int
ntb_child_location(device_t dev,device_t child,struct sbuf * sb)169 ntb_child_location(device_t dev, device_t child, struct sbuf *sb)
170 {
171 struct ntb_child *nc = device_get_ivars(child);
172
173 sbuf_printf(sb, "function=%d", nc->function);
174 return (0);
175 }
176
177 int
ntb_print_child(device_t dev,device_t child)178 ntb_print_child(device_t dev, device_t child)
179 {
180 struct ntb_child *nc = device_get_ivars(child);
181 int retval;
182
183 retval = bus_print_child_header(dev, child);
184 if (nc->mwcnt > 0) {
185 printf(" mw %d", nc->mwoff);
186 if (nc->mwcnt > 1)
187 printf("-%d", nc->mwoff + nc->mwcnt - 1);
188 }
189 if (nc->spadcnt > 0) {
190 printf(" spad %d", nc->spadoff);
191 if (nc->spadcnt > 1)
192 printf("-%d", nc->spadoff + nc->spadcnt - 1);
193 }
194 if (nc->dbcnt > 0) {
195 printf(" db %d", nc->dboff);
196 if (nc->dbcnt > 1)
197 printf("-%d", nc->dboff + nc->dbcnt - 1);
198 }
199 retval += printf(" at function %d", nc->function);
200 retval += bus_print_child_domain(dev, child);
201 retval += bus_print_child_footer(dev, child);
202
203 return (retval);
204 }
205
206 bus_dma_tag_t
ntb_get_dma_tag(device_t bus,device_t child)207 ntb_get_dma_tag(device_t bus, device_t child)
208 {
209
210 return (bus_get_dma_tag(bus));
211 }
212
213 void
ntb_link_event(device_t dev)214 ntb_link_event(device_t dev)
215 {
216 struct ntb_child **cpp = device_get_softc(dev);
217 struct ntb_child *nc;
218 struct rm_priotracker ctx_tracker;
219 enum ntb_speed speed;
220 enum ntb_width width;
221
222 if (NTB_LINK_IS_UP(dev, &speed, &width)) {
223 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n",
224 (int)speed, (int)width);
225 } else {
226 device_printf(dev, "Link is down\n");
227 }
228 for (nc = *cpp; nc != NULL; nc = nc->next) {
229 rm_rlock(&nc->ctx_lock, &ctx_tracker);
230 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL)
231 nc->ctx_ops->link_event(nc->ctx);
232 rm_runlock(&nc->ctx_lock, &ctx_tracker);
233 }
234 }
235
236 void
ntb_db_event(device_t dev,uint32_t vec)237 ntb_db_event(device_t dev, uint32_t vec)
238 {
239 struct ntb_child **cpp = device_get_softc(dev);
240 struct ntb_child *nc;
241 struct rm_priotracker ctx_tracker;
242
243 for (nc = *cpp; nc != NULL; nc = nc->next) {
244 rm_rlock(&nc->ctx_lock, &ctx_tracker);
245 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL)
246 nc->ctx_ops->db_event(nc->ctx, vec);
247 rm_runlock(&nc->ctx_lock, &ctx_tracker);
248 }
249 }
250
251 int
ntb_port_number(device_t ntb)252 ntb_port_number(device_t ntb)
253 {
254 return (NTB_PORT_NUMBER(device_get_parent(ntb)));
255 }
256
257 int
ntb_peer_port_count(device_t ntb)258 ntb_peer_port_count(device_t ntb)
259 {
260 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb)));
261 }
262
263 int
ntb_peer_port_number(device_t ntb,int pidx)264 ntb_peer_port_number(device_t ntb, int pidx)
265 {
266 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx));
267 }
268
269 int
ntb_peer_port_idx(device_t ntb,int port)270 ntb_peer_port_idx(device_t ntb, int port)
271 {
272 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port));
273 }
274
275 bool
ntb_link_is_up(device_t ntb,enum ntb_speed * speed,enum ntb_width * width)276 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width)
277 {
278
279 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width));
280 }
281
282 int
ntb_link_enable(device_t ntb,enum ntb_speed speed,enum ntb_width width)283 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width)
284 {
285 struct ntb_child *nc = device_get_ivars(ntb);
286 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
287 struct ntb_child *nc1;
288
289 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
290 if (nc1->enabled) {
291 nc->enabled = 1;
292 return (0);
293 }
294 }
295 nc->enabled = 1;
296 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width));
297 }
298
299 int
ntb_link_disable(device_t ntb)300 ntb_link_disable(device_t ntb)
301 {
302 struct ntb_child *nc = device_get_ivars(ntb);
303 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
304 struct ntb_child *nc1;
305
306 if (!nc->enabled)
307 return (0);
308 nc->enabled = 0;
309 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
310 if (nc1->enabled)
311 return (0);
312 }
313 return (NTB_LINK_DISABLE(device_get_parent(ntb)));
314 }
315
316 bool
ntb_link_enabled(device_t ntb)317 ntb_link_enabled(device_t ntb)
318 {
319 struct ntb_child *nc = device_get_ivars(ntb);
320
321 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb)));
322 }
323
324 int
ntb_set_ctx(device_t ntb,void * ctx,const struct ntb_ctx_ops * ctx_ops)325 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops)
326 {
327 struct ntb_child *nc = device_get_ivars(ntb);
328
329 if (ctx == NULL || ctx_ops == NULL)
330 return (EINVAL);
331
332 rm_wlock(&nc->ctx_lock);
333 if (nc->ctx_ops != NULL) {
334 rm_wunlock(&nc->ctx_lock);
335 return (EINVAL);
336 }
337 nc->ctx = ctx;
338 nc->ctx_ops = ctx_ops;
339
340 /*
341 * If applicaiton driver asks for link events, generate fake one now
342 * to let it update link state without races while we hold the lock.
343 */
344 if (ctx_ops->link_event != NULL)
345 ctx_ops->link_event(ctx);
346 rm_wunlock(&nc->ctx_lock);
347
348 return (0);
349 }
350
351 void *
ntb_get_ctx(device_t ntb,const struct ntb_ctx_ops ** ctx_ops)352 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops)
353 {
354 struct ntb_child *nc = device_get_ivars(ntb);
355
356 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus"));
357 if (ctx_ops != NULL)
358 *ctx_ops = nc->ctx_ops;
359 return (nc->ctx);
360 }
361
362 void
ntb_clear_ctx(device_t ntb)363 ntb_clear_ctx(device_t ntb)
364 {
365 struct ntb_child *nc = device_get_ivars(ntb);
366
367 rm_wlock(&nc->ctx_lock);
368 nc->ctx = NULL;
369 nc->ctx_ops = NULL;
370 rm_wunlock(&nc->ctx_lock);
371 }
372
373 uint8_t
ntb_mw_count(device_t ntb)374 ntb_mw_count(device_t ntb)
375 {
376 struct ntb_child *nc = device_get_ivars(ntb);
377
378 return (nc->mwcnt);
379 }
380
381 int
ntb_mw_get_range(device_t ntb,unsigned mw_idx,vm_paddr_t * base,caddr_t * vbase,size_t * size,size_t * align,size_t * align_size,bus_addr_t * plimit)382 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base,
383 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
384 bus_addr_t *plimit)
385 {
386 struct ntb_child *nc = device_get_ivars(ntb);
387
388 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff,
389 base, vbase, size, align, align_size, plimit));
390 }
391
392 int
ntb_mw_set_trans(device_t ntb,unsigned mw_idx,bus_addr_t addr,size_t size)393 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size)
394 {
395 struct ntb_child *nc = device_get_ivars(ntb);
396
397 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff,
398 addr, size));
399 }
400
401 int
ntb_mw_clear_trans(device_t ntb,unsigned mw_idx)402 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx)
403 {
404 struct ntb_child *nc = device_get_ivars(ntb);
405
406 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff));
407 }
408
409 int
ntb_mw_get_wc(device_t ntb,unsigned mw_idx,vm_memattr_t * mode)410 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode)
411 {
412 struct ntb_child *nc = device_get_ivars(ntb);
413
414 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
415 }
416
417 int
ntb_mw_set_wc(device_t ntb,unsigned mw_idx,vm_memattr_t mode)418 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode)
419 {
420 struct ntb_child *nc = device_get_ivars(ntb);
421
422 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
423 }
424
425 uint8_t
ntb_spad_count(device_t ntb)426 ntb_spad_count(device_t ntb)
427 {
428 struct ntb_child *nc = device_get_ivars(ntb);
429
430 return (nc->spadcnt);
431 }
432
433 void
ntb_spad_clear(device_t ntb)434 ntb_spad_clear(device_t ntb)
435 {
436 struct ntb_child *nc = device_get_ivars(ntb);
437 unsigned i;
438
439 for (i = 0; i < nc->spadcnt; i++)
440 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0);
441 }
442
443 int
ntb_spad_write(device_t ntb,unsigned int idx,uint32_t val)444 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val)
445 {
446 struct ntb_child *nc = device_get_ivars(ntb);
447
448 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val));
449 }
450
451 int
ntb_spad_read(device_t ntb,unsigned int idx,uint32_t * val)452 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
453 {
454 struct ntb_child *nc = device_get_ivars(ntb);
455
456 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val));
457 }
458
459 int
ntb_peer_spad_write(device_t ntb,unsigned int idx,uint32_t val)460 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val)
461 {
462 struct ntb_child *nc = device_get_ivars(ntb);
463
464 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff,
465 val));
466 }
467
468 int
ntb_peer_spad_read(device_t ntb,unsigned int idx,uint32_t * val)469 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
470 {
471 struct ntb_child *nc = device_get_ivars(ntb);
472
473 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff,
474 val));
475 }
476
477 uint64_t
ntb_db_valid_mask(device_t ntb)478 ntb_db_valid_mask(device_t ntb)
479 {
480 struct ntb_child *nc = device_get_ivars(ntb);
481
482 return (nc->dbmask);
483 }
484
485 int
ntb_db_vector_count(device_t ntb)486 ntb_db_vector_count(device_t ntb)
487 {
488
489 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb)));
490 }
491
492 uint64_t
ntb_db_vector_mask(device_t ntb,uint32_t vector)493 ntb_db_vector_mask(device_t ntb, uint32_t vector)
494 {
495 struct ntb_child *nc = device_get_ivars(ntb);
496
497 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector)
498 >> nc->dboff) & nc->dbmask);
499 }
500
501 int
ntb_peer_db_addr(device_t ntb,bus_addr_t * db_addr,vm_size_t * db_size)502 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size)
503 {
504
505 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size));
506 }
507
508 void
ntb_db_clear(device_t ntb,uint64_t bits)509 ntb_db_clear(device_t ntb, uint64_t bits)
510 {
511 struct ntb_child *nc = device_get_ivars(ntb);
512
513 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff));
514 }
515
516 void
ntb_db_clear_mask(device_t ntb,uint64_t bits)517 ntb_db_clear_mask(device_t ntb, uint64_t bits)
518 {
519 struct ntb_child *nc = device_get_ivars(ntb);
520
521 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff));
522 }
523
524 uint64_t
ntb_db_read(device_t ntb)525 ntb_db_read(device_t ntb)
526 {
527 struct ntb_child *nc = device_get_ivars(ntb);
528
529 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff)
530 & nc->dbmask);
531 }
532
533 void
ntb_db_set_mask(device_t ntb,uint64_t bits)534 ntb_db_set_mask(device_t ntb, uint64_t bits)
535 {
536 struct ntb_child *nc = device_get_ivars(ntb);
537
538 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff));
539 }
540
541 void
ntb_peer_db_set(device_t ntb,uint64_t bits)542 ntb_peer_db_set(device_t ntb, uint64_t bits)
543 {
544 struct ntb_child *nc = device_get_ivars(ntb);
545
546 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff));
547 }
548
549 MODULE_VERSION(ntb, 1);
550