1 /*-
2 * Copyright (c) 2016-2017 Alexander Motin <[email protected]>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include <sys/param.h>
31 #include <sys/kernel.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/rmlock.h>
35 #include <sys/malloc.h>
36 #include <sys/module.h>
37 #include <sys/sysctl.h>
38
39 #include "ntb.h"
40
41 devclass_t ntb_hw_devclass;
42 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls");
43
44 struct ntb_child {
45 device_t dev;
46 int function;
47 int enabled;
48 int mwoff;
49 int mwcnt;
50 int spadoff;
51 int spadcnt;
52 int dboff;
53 int dbcnt;
54 uint64_t dbmask;
55 void *ctx;
56 const struct ntb_ctx_ops *ctx_ops;
57 struct rmlock ctx_lock;
58 struct ntb_child *next;
59 };
60
61 int
ntb_register_device(device_t dev)62 ntb_register_device(device_t dev)
63 {
64 struct ntb_child **cpp = device_get_softc(dev);
65 struct ntb_child *nc;
66 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt;
67 char cfg[128] = "";
68 char buf[32];
69 char *n, *np, *c, *p, *name;
70
71 mwu = 0;
72 mwt = NTB_MW_COUNT(dev);
73 spadu = 0;
74 spadt = NTB_SPAD_COUNT(dev);
75 dbu = 0;
76 dbt = flsll(NTB_DB_VALID_MASK(dev));
77
78 device_printf(dev, "%d memory windows, %d scratchpads, "
79 "%d doorbells\n", mwt, spadt, dbt);
80
81 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev),
82 device_get_unit(dev));
83 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg));
84 n = cfg;
85 i = 0;
86 while ((c = strsep(&n, ",")) != NULL) {
87 np = c;
88 name = strsep(&np, ":");
89 if (name != NULL && name[0] == 0)
90 name = NULL;
91 p = strsep(&np, ":");
92 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu;
93 p = strsep(&np, ":");
94 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu;
95 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu;
96
97 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) {
98 device_printf(dev, "Not enough resources for config\n");
99 break;
100 }
101
102 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO);
103 nc->function = i;
104 nc->mwoff = mwu;
105 nc->mwcnt = mw;
106 nc->spadoff = spadu;
107 nc->spadcnt = spad;
108 nc->dboff = dbu;
109 nc->dbcnt = db;
110 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db));
111 rm_init(&nc->ctx_lock, "ntb ctx");
112 nc->dev = device_add_child(dev, name, -1);
113 if (nc->dev == NULL) {
114 ntb_unregister_device(dev);
115 return (ENOMEM);
116 }
117 device_set_ivars(nc->dev, nc);
118 *cpp = nc;
119 cpp = &nc->next;
120
121 if (bootverbose) {
122 device_printf(dev, "%d \"%s\":", i, name);
123 if (mw > 0) {
124 printf(" memory windows %d", mwu);
125 if (mw > 1)
126 printf("-%d", mwu + mw - 1);
127 }
128 if (spad > 0) {
129 printf(" scratchpads %d", spadu);
130 if (spad > 1)
131 printf("-%d", spadu + spad - 1);
132 }
133 if (db > 0) {
134 printf(" doorbells %d", dbu);
135 if (db > 1)
136 printf("-%d", dbu + db - 1);
137 }
138 printf("\n");
139 }
140
141 mwu += mw;
142 spadu += spad;
143 dbu += db;
144 i++;
145 }
146
147 bus_generic_attach(dev);
148 return (0);
149 }
150
151 int
ntb_unregister_device(device_t dev)152 ntb_unregister_device(device_t dev)
153 {
154 struct ntb_child **cpp = device_get_softc(dev);
155 struct ntb_child *nc;
156 int error = 0;
157
158 while ((nc = *cpp) != NULL) {
159 *cpp = (*cpp)->next;
160 error = device_delete_child(dev, nc->dev);
161 if (error)
162 break;
163 rm_destroy(&nc->ctx_lock);
164 free(nc, M_DEVBUF);
165 }
166 return (error);
167 }
168
169 int
ntb_child_location_str(device_t dev,device_t child,char * buf,size_t buflen)170 ntb_child_location_str(device_t dev, device_t child, char *buf,
171 size_t buflen)
172 {
173 struct ntb_child *nc = device_get_ivars(child);
174
175 snprintf(buf, buflen, "function=%d", nc->function);
176 return (0);
177 }
178
179 int
ntb_print_child(device_t dev,device_t child)180 ntb_print_child(device_t dev, device_t child)
181 {
182 struct ntb_child *nc = device_get_ivars(child);
183 int retval;
184
185 retval = bus_print_child_header(dev, child);
186 if (nc->mwcnt > 0) {
187 printf(" mw %d", nc->mwoff);
188 if (nc->mwcnt > 1)
189 printf("-%d", nc->mwoff + nc->mwcnt - 1);
190 }
191 if (nc->spadcnt > 0) {
192 printf(" spad %d", nc->spadoff);
193 if (nc->spadcnt > 1)
194 printf("-%d", nc->spadoff + nc->spadcnt - 1);
195 }
196 if (nc->dbcnt > 0) {
197 printf(" db %d", nc->dboff);
198 if (nc->dbcnt > 1)
199 printf("-%d", nc->dboff + nc->dbcnt - 1);
200 }
201 retval += printf(" at function %d", nc->function);
202 retval += bus_print_child_domain(dev, child);
203 retval += bus_print_child_footer(dev, child);
204
205 return (retval);
206 }
207
208 void
ntb_link_event(device_t dev)209 ntb_link_event(device_t dev)
210 {
211 struct ntb_child **cpp = device_get_softc(dev);
212 struct ntb_child *nc;
213 struct rm_priotracker ctx_tracker;
214 enum ntb_speed speed;
215 enum ntb_width width;
216
217 if (NTB_LINK_IS_UP(dev, &speed, &width)) {
218 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n",
219 (int)speed, (int)width);
220 } else {
221 device_printf(dev, "Link is down\n");
222 }
223 for (nc = *cpp; nc != NULL; nc = nc->next) {
224 rm_rlock(&nc->ctx_lock, &ctx_tracker);
225 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL)
226 nc->ctx_ops->link_event(nc->ctx);
227 rm_runlock(&nc->ctx_lock, &ctx_tracker);
228 }
229 }
230
231 void
ntb_db_event(device_t dev,uint32_t vec)232 ntb_db_event(device_t dev, uint32_t vec)
233 {
234 struct ntb_child **cpp = device_get_softc(dev);
235 struct ntb_child *nc;
236 struct rm_priotracker ctx_tracker;
237
238 for (nc = *cpp; nc != NULL; nc = nc->next) {
239 rm_rlock(&nc->ctx_lock, &ctx_tracker);
240 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL)
241 nc->ctx_ops->db_event(nc->ctx, vec);
242 rm_runlock(&nc->ctx_lock, &ctx_tracker);
243 }
244 }
245
246 int
ntb_port_number(device_t ntb)247 ntb_port_number(device_t ntb)
248 {
249 return (NTB_PORT_NUMBER(device_get_parent(ntb)));
250 }
251
252 int
ntb_peer_port_count(device_t ntb)253 ntb_peer_port_count(device_t ntb)
254 {
255 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb)));
256 }
257
258 int
ntb_peer_port_number(device_t ntb,int pidx)259 ntb_peer_port_number(device_t ntb, int pidx)
260 {
261 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx));
262 }
263
264 int
ntb_peer_port_idx(device_t ntb,int port)265 ntb_peer_port_idx(device_t ntb, int port)
266 {
267 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port));
268 }
269
270 bool
ntb_link_is_up(device_t ntb,enum ntb_speed * speed,enum ntb_width * width)271 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width)
272 {
273
274 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width));
275 }
276
277 int
ntb_link_enable(device_t ntb,enum ntb_speed speed,enum ntb_width width)278 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width)
279 {
280 struct ntb_child *nc = device_get_ivars(ntb);
281 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
282 struct ntb_child *nc1;
283
284 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
285 if (nc1->enabled) {
286 nc->enabled = 1;
287 return (0);
288 }
289 }
290 nc->enabled = 1;
291 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width));
292 }
293
294 int
ntb_link_disable(device_t ntb)295 ntb_link_disable(device_t ntb)
296 {
297 struct ntb_child *nc = device_get_ivars(ntb);
298 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev));
299 struct ntb_child *nc1;
300
301 if (!nc->enabled)
302 return (0);
303 nc->enabled = 0;
304 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) {
305 if (nc1->enabled)
306 return (0);
307 }
308 return (NTB_LINK_DISABLE(device_get_parent(ntb)));
309 }
310
311 bool
ntb_link_enabled(device_t ntb)312 ntb_link_enabled(device_t ntb)
313 {
314 struct ntb_child *nc = device_get_ivars(ntb);
315
316 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb)));
317 }
318
319 int
ntb_set_ctx(device_t ntb,void * ctx,const struct ntb_ctx_ops * ctx_ops)320 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops)
321 {
322 struct ntb_child *nc = device_get_ivars(ntb);
323
324 if (ctx == NULL || ctx_ops == NULL)
325 return (EINVAL);
326
327 rm_wlock(&nc->ctx_lock);
328 if (nc->ctx_ops != NULL) {
329 rm_wunlock(&nc->ctx_lock);
330 return (EINVAL);
331 }
332 nc->ctx = ctx;
333 nc->ctx_ops = ctx_ops;
334
335 /*
336 * If applicaiton driver asks for link events, generate fake one now
337 * to let it update link state without races while we hold the lock.
338 */
339 if (ctx_ops->link_event != NULL)
340 ctx_ops->link_event(ctx);
341 rm_wunlock(&nc->ctx_lock);
342
343 return (0);
344 }
345
346 void *
ntb_get_ctx(device_t ntb,const struct ntb_ctx_ops ** ctx_ops)347 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops)
348 {
349 struct ntb_child *nc = device_get_ivars(ntb);
350
351 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus"));
352 if (ctx_ops != NULL)
353 *ctx_ops = nc->ctx_ops;
354 return (nc->ctx);
355 }
356
357 void
ntb_clear_ctx(device_t ntb)358 ntb_clear_ctx(device_t ntb)
359 {
360 struct ntb_child *nc = device_get_ivars(ntb);
361
362 rm_wlock(&nc->ctx_lock);
363 nc->ctx = NULL;
364 nc->ctx_ops = NULL;
365 rm_wunlock(&nc->ctx_lock);
366 }
367
368 uint8_t
ntb_mw_count(device_t ntb)369 ntb_mw_count(device_t ntb)
370 {
371 struct ntb_child *nc = device_get_ivars(ntb);
372
373 return (nc->mwcnt);
374 }
375
376 int
ntb_mw_get_range(device_t ntb,unsigned mw_idx,vm_paddr_t * base,caddr_t * vbase,size_t * size,size_t * align,size_t * align_size,bus_addr_t * plimit)377 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base,
378 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
379 bus_addr_t *plimit)
380 {
381 struct ntb_child *nc = device_get_ivars(ntb);
382
383 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff,
384 base, vbase, size, align, align_size, plimit));
385 }
386
387 int
ntb_mw_set_trans(device_t ntb,unsigned mw_idx,bus_addr_t addr,size_t size)388 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size)
389 {
390 struct ntb_child *nc = device_get_ivars(ntb);
391
392 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff,
393 addr, size));
394 }
395
396 int
ntb_mw_clear_trans(device_t ntb,unsigned mw_idx)397 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx)
398 {
399 struct ntb_child *nc = device_get_ivars(ntb);
400
401 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff));
402 }
403
404 int
ntb_mw_get_wc(device_t ntb,unsigned mw_idx,vm_memattr_t * mode)405 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode)
406 {
407 struct ntb_child *nc = device_get_ivars(ntb);
408
409 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
410 }
411
412 int
ntb_mw_set_wc(device_t ntb,unsigned mw_idx,vm_memattr_t mode)413 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode)
414 {
415 struct ntb_child *nc = device_get_ivars(ntb);
416
417 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode));
418 }
419
420 uint8_t
ntb_spad_count(device_t ntb)421 ntb_spad_count(device_t ntb)
422 {
423 struct ntb_child *nc = device_get_ivars(ntb);
424
425 return (nc->spadcnt);
426 }
427
428 void
ntb_spad_clear(device_t ntb)429 ntb_spad_clear(device_t ntb)
430 {
431 struct ntb_child *nc = device_get_ivars(ntb);
432 unsigned i;
433
434 for (i = 0; i < nc->spadcnt; i++)
435 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0);
436 }
437
438 int
ntb_spad_write(device_t ntb,unsigned int idx,uint32_t val)439 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val)
440 {
441 struct ntb_child *nc = device_get_ivars(ntb);
442
443 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val));
444 }
445
446 int
ntb_spad_read(device_t ntb,unsigned int idx,uint32_t * val)447 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
448 {
449 struct ntb_child *nc = device_get_ivars(ntb);
450
451 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val));
452 }
453
454 int
ntb_peer_spad_write(device_t ntb,unsigned int idx,uint32_t val)455 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val)
456 {
457 struct ntb_child *nc = device_get_ivars(ntb);
458
459 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff,
460 val));
461 }
462
463 int
ntb_peer_spad_read(device_t ntb,unsigned int idx,uint32_t * val)464 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val)
465 {
466 struct ntb_child *nc = device_get_ivars(ntb);
467
468 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff,
469 val));
470 }
471
472 uint64_t
ntb_db_valid_mask(device_t ntb)473 ntb_db_valid_mask(device_t ntb)
474 {
475 struct ntb_child *nc = device_get_ivars(ntb);
476
477 return (nc->dbmask);
478 }
479
480 int
ntb_db_vector_count(device_t ntb)481 ntb_db_vector_count(device_t ntb)
482 {
483
484 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb)));
485 }
486
487 uint64_t
ntb_db_vector_mask(device_t ntb,uint32_t vector)488 ntb_db_vector_mask(device_t ntb, uint32_t vector)
489 {
490 struct ntb_child *nc = device_get_ivars(ntb);
491
492 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector)
493 >> nc->dboff) & nc->dbmask);
494 }
495
496 int
ntb_peer_db_addr(device_t ntb,bus_addr_t * db_addr,vm_size_t * db_size)497 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size)
498 {
499
500 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size));
501 }
502
503 void
ntb_db_clear(device_t ntb,uint64_t bits)504 ntb_db_clear(device_t ntb, uint64_t bits)
505 {
506 struct ntb_child *nc = device_get_ivars(ntb);
507
508 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff));
509 }
510
511 void
ntb_db_clear_mask(device_t ntb,uint64_t bits)512 ntb_db_clear_mask(device_t ntb, uint64_t bits)
513 {
514 struct ntb_child *nc = device_get_ivars(ntb);
515
516 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff));
517 }
518
519 uint64_t
ntb_db_read(device_t ntb)520 ntb_db_read(device_t ntb)
521 {
522 struct ntb_child *nc = device_get_ivars(ntb);
523
524 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff)
525 & nc->dbmask);
526 }
527
528 void
ntb_db_set_mask(device_t ntb,uint64_t bits)529 ntb_db_set_mask(device_t ntb, uint64_t bits)
530 {
531 struct ntb_child *nc = device_get_ivars(ntb);
532
533 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff));
534 }
535
536 void
ntb_peer_db_set(device_t ntb,uint64_t bits)537 ntb_peer_db_set(device_t ntb, uint64_t bits)
538 {
539 struct ntb_child *nc = device_get_ivars(ntb);
540
541 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff));
542 }
543
544 MODULE_VERSION(ntb, 1);
545