1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2017 Chelsio Communications, Inc.
5 * Copyright (c) 2017 Conrad Meyer <[email protected]>
6 * All rights reserved.
7 * Largely borrowed from ccr(4), Written by: John Baldwin <[email protected]>
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include "opt_ddb.h"
35
36 #include <sys/param.h>
37 #include <sys/bus.h>
38 #include <sys/lock.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/module.h>
43 #include <sys/random.h>
44 #include <sys/sglist.h>
45 #include <sys/sysctl.h>
46
47 #ifdef DDB
48 #include <ddb/ddb.h>
49 #endif
50
51 #include <dev/pci/pcivar.h>
52
53 #include <dev/random/randomdev.h>
54
55 #include <opencrypto/cryptodev.h>
56 #include <opencrypto/xform.h>
57
58 #include "cryptodev_if.h"
59
60 #include "ccp.h"
61 #include "ccp_hardware.h"
62
63 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
64
65 /*
66 * Need a global softc available for garbage random_source API, which lacks any
67 * context pointer. It's also handy for debugging.
68 */
69 struct ccp_softc *g_ccp_softc;
70
71 bool g_debug_print = false;
72 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
73 "Set to enable debugging log messages");
74
75 static struct pciid {
76 uint32_t devid;
77 const char *desc;
78 } ccp_ids[] = {
79 { 0x14561022, "AMD CCP-5a" },
80 { 0x14681022, "AMD CCP-5b" },
81 { 0x15df1022, "AMD CCP-5a" },
82 };
83
84 static struct random_source random_ccp = {
85 .rs_ident = "AMD CCP TRNG",
86 .rs_source = RANDOM_PURE_CCP,
87 .rs_read = random_ccp_read,
88 };
89
90 /*
91 * ccp_populate_sglist() generates a scatter/gather list that covers the entire
92 * crypto operation buffer.
93 */
94 static int
ccp_populate_sglist(struct sglist * sg,struct crypto_buffer * cb)95 ccp_populate_sglist(struct sglist *sg, struct crypto_buffer *cb)
96 {
97 int error;
98
99 sglist_reset(sg);
100 switch (cb->cb_type) {
101 case CRYPTO_BUF_MBUF:
102 error = sglist_append_mbuf(sg, cb->cb_mbuf);
103 break;
104 case CRYPTO_BUF_SINGLE_MBUF:
105 error = sglist_append_single_mbuf(sg, cb->cb_mbuf);
106 break;
107 case CRYPTO_BUF_UIO:
108 error = sglist_append_uio(sg, cb->cb_uio);
109 break;
110 case CRYPTO_BUF_CONTIG:
111 error = sglist_append(sg, cb->cb_buf, cb->cb_buf_len);
112 break;
113 case CRYPTO_BUF_VMPAGE:
114 error = sglist_append_vmpages(sg, cb->cb_vm_page,
115 cb->cb_vm_page_len, cb->cb_vm_page_offset);
116 break;
117 default:
118 error = EINVAL;
119 }
120 return (error);
121 }
122
123 static int
ccp_probe(device_t dev)124 ccp_probe(device_t dev)
125 {
126 struct pciid *ip;
127 uint32_t id;
128
129 id = pci_get_devid(dev);
130 for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
131 if (id == ip->devid) {
132 device_set_desc(dev, ip->desc);
133 return (0);
134 }
135 }
136 return (ENXIO);
137 }
138
139 static void
ccp_initialize_queues(struct ccp_softc * sc)140 ccp_initialize_queues(struct ccp_softc *sc)
141 {
142 struct ccp_queue *qp;
143 size_t i;
144
145 for (i = 0; i < nitems(sc->queues); i++) {
146 qp = &sc->queues[i];
147
148 qp->cq_softc = sc;
149 qp->cq_qindex = i;
150 mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
151 /* XXX - arbitrarily chosen sizes */
152 qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
153 /* Two more SGEs than sg_crp to accommodate ipad. */
154 qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
155 qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
156 }
157 }
158
159 static void
ccp_free_queues(struct ccp_softc * sc)160 ccp_free_queues(struct ccp_softc *sc)
161 {
162 struct ccp_queue *qp;
163 size_t i;
164
165 for (i = 0; i < nitems(sc->queues); i++) {
166 qp = &sc->queues[i];
167
168 mtx_destroy(&qp->cq_lock);
169 sglist_free(qp->cq_sg_crp);
170 sglist_free(qp->cq_sg_ulptx);
171 sglist_free(qp->cq_sg_dst);
172 }
173 }
174
175 static int
ccp_attach(device_t dev)176 ccp_attach(device_t dev)
177 {
178 struct ccp_softc *sc;
179 int error;
180
181 sc = device_get_softc(dev);
182 sc->dev = dev;
183
184 sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
185 CRYPTOCAP_F_HARDWARE);
186 if (sc->cid < 0) {
187 device_printf(dev, "could not get crypto driver id\n");
188 return (ENXIO);
189 }
190
191 error = ccp_hw_attach(dev);
192 if (error != 0)
193 return (error);
194
195 mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
196
197 ccp_initialize_queues(sc);
198
199 if (g_ccp_softc == NULL) {
200 g_ccp_softc = sc;
201 if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
202 random_source_register(&random_ccp);
203 }
204
205 return (0);
206 }
207
208 static int
ccp_detach(device_t dev)209 ccp_detach(device_t dev)
210 {
211 struct ccp_softc *sc;
212
213 sc = device_get_softc(dev);
214
215 mtx_lock(&sc->lock);
216 sc->detaching = true;
217 mtx_unlock(&sc->lock);
218
219 crypto_unregister_all(sc->cid);
220 if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
221 random_source_deregister(&random_ccp);
222
223 ccp_hw_detach(dev);
224 ccp_free_queues(sc);
225
226 if (g_ccp_softc == sc)
227 g_ccp_softc = NULL;
228
229 mtx_destroy(&sc->lock);
230 return (0);
231 }
232
233 static void
ccp_init_hmac_digest(struct ccp_session * s,const char * key,int klen)234 ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
235 {
236 union authctx auth_ctx;
237 struct auth_hash *axf;
238 u_int i;
239
240 /*
241 * If the key is larger than the block size, use the digest of
242 * the key as the key instead.
243 */
244 axf = s->hmac.auth_hash;
245 if (klen > axf->blocksize) {
246 axf->Init(&auth_ctx);
247 axf->Update(&auth_ctx, key, klen);
248 axf->Final(s->hmac.ipad, &auth_ctx);
249 explicit_bzero(&auth_ctx, sizeof(auth_ctx));
250 klen = axf->hashsize;
251 } else
252 memcpy(s->hmac.ipad, key, klen);
253
254 memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
255 memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
256
257 for (i = 0; i < axf->blocksize; i++) {
258 s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
259 s->hmac.opad[i] ^= HMAC_OPAD_VAL;
260 }
261 }
262
263 static bool
ccp_aes_check_keylen(int alg,int klen)264 ccp_aes_check_keylen(int alg, int klen)
265 {
266
267 switch (klen * 8) {
268 case 128:
269 case 192:
270 if (alg == CRYPTO_AES_XTS)
271 return (false);
272 break;
273 case 256:
274 break;
275 case 512:
276 if (alg != CRYPTO_AES_XTS)
277 return (false);
278 break;
279 default:
280 return (false);
281 }
282 return (true);
283 }
284
285 static void
ccp_aes_setkey(struct ccp_session * s,int alg,const void * key,int klen)286 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
287 {
288 unsigned kbits;
289
290 if (alg == CRYPTO_AES_XTS)
291 kbits = (klen / 2) * 8;
292 else
293 kbits = klen * 8;
294
295 switch (kbits) {
296 case 128:
297 s->blkcipher.cipher_type = CCP_AES_TYPE_128;
298 break;
299 case 192:
300 s->blkcipher.cipher_type = CCP_AES_TYPE_192;
301 break;
302 case 256:
303 s->blkcipher.cipher_type = CCP_AES_TYPE_256;
304 break;
305 default:
306 panic("should not get here");
307 }
308
309 s->blkcipher.key_len = klen;
310 memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
311 }
312
313 static bool
ccp_auth_supported(struct ccp_softc * sc,const struct crypto_session_params * csp)314 ccp_auth_supported(struct ccp_softc *sc,
315 const struct crypto_session_params *csp)
316 {
317
318 if ((sc->hw_features & VERSION_CAP_SHA) == 0)
319 return (false);
320 switch (csp->csp_auth_alg) {
321 case CRYPTO_SHA1_HMAC:
322 case CRYPTO_SHA2_256_HMAC:
323 case CRYPTO_SHA2_384_HMAC:
324 case CRYPTO_SHA2_512_HMAC:
325 if (csp->csp_auth_key == NULL)
326 return (false);
327 break;
328 default:
329 return (false);
330 }
331 return (true);
332 }
333
334 static bool
ccp_cipher_supported(struct ccp_softc * sc,const struct crypto_session_params * csp)335 ccp_cipher_supported(struct ccp_softc *sc,
336 const struct crypto_session_params *csp)
337 {
338
339 if ((sc->hw_features & VERSION_CAP_AES) == 0)
340 return (false);
341 switch (csp->csp_cipher_alg) {
342 case CRYPTO_AES_CBC:
343 if (csp->csp_ivlen != AES_BLOCK_LEN)
344 return (false);
345 break;
346 case CRYPTO_AES_ICM:
347 if (csp->csp_ivlen != AES_BLOCK_LEN)
348 return (false);
349 break;
350 case CRYPTO_AES_XTS:
351 if (csp->csp_ivlen != AES_XTS_IV_LEN)
352 return (false);
353 break;
354 default:
355 return (false);
356 }
357 return (ccp_aes_check_keylen(csp->csp_cipher_alg,
358 csp->csp_cipher_klen));
359 }
360
361 static int
ccp_probesession(device_t dev,const struct crypto_session_params * csp)362 ccp_probesession(device_t dev, const struct crypto_session_params *csp)
363 {
364 struct ccp_softc *sc;
365
366 if (csp->csp_flags != 0)
367 return (EINVAL);
368 sc = device_get_softc(dev);
369 switch (csp->csp_mode) {
370 case CSP_MODE_DIGEST:
371 if (!ccp_auth_supported(sc, csp))
372 return (EINVAL);
373 break;
374 case CSP_MODE_CIPHER:
375 if (!ccp_cipher_supported(sc, csp))
376 return (EINVAL);
377 break;
378 case CSP_MODE_AEAD:
379 switch (csp->csp_cipher_alg) {
380 case CRYPTO_AES_NIST_GCM_16:
381 if (csp->csp_ivlen != AES_GCM_IV_LEN)
382 return (EINVAL);
383 if (csp->csp_auth_mlen < 0 ||
384 csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
385 return (EINVAL);
386 if ((sc->hw_features & VERSION_CAP_AES) == 0)
387 return (EINVAL);
388 break;
389 default:
390 return (EINVAL);
391 }
392 break;
393 case CSP_MODE_ETA:
394 if (!ccp_auth_supported(sc, csp) ||
395 !ccp_cipher_supported(sc, csp))
396 return (EINVAL);
397 break;
398 default:
399 return (EINVAL);
400 }
401
402 return (CRYPTODEV_PROBE_HARDWARE);
403 }
404
405 static int
ccp_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)406 ccp_newsession(device_t dev, crypto_session_t cses,
407 const struct crypto_session_params *csp)
408 {
409 struct ccp_softc *sc;
410 struct ccp_session *s;
411 struct auth_hash *auth_hash;
412 enum ccp_aes_mode cipher_mode;
413 unsigned auth_mode;
414 unsigned q;
415
416 /* XXX reconcile auth_mode with use by ccp_sha */
417 switch (csp->csp_auth_alg) {
418 case CRYPTO_SHA1_HMAC:
419 auth_hash = &auth_hash_hmac_sha1;
420 auth_mode = SHA1;
421 break;
422 case CRYPTO_SHA2_256_HMAC:
423 auth_hash = &auth_hash_hmac_sha2_256;
424 auth_mode = SHA2_256;
425 break;
426 case CRYPTO_SHA2_384_HMAC:
427 auth_hash = &auth_hash_hmac_sha2_384;
428 auth_mode = SHA2_384;
429 break;
430 case CRYPTO_SHA2_512_HMAC:
431 auth_hash = &auth_hash_hmac_sha2_512;
432 auth_mode = SHA2_512;
433 break;
434 default:
435 auth_hash = NULL;
436 auth_mode = 0;
437 break;
438 }
439
440 switch (csp->csp_cipher_alg) {
441 case CRYPTO_AES_CBC:
442 cipher_mode = CCP_AES_MODE_CBC;
443 break;
444 case CRYPTO_AES_ICM:
445 cipher_mode = CCP_AES_MODE_CTR;
446 break;
447 case CRYPTO_AES_NIST_GCM_16:
448 cipher_mode = CCP_AES_MODE_GCTR;
449 break;
450 case CRYPTO_AES_XTS:
451 cipher_mode = CCP_AES_MODE_XTS;
452 break;
453 default:
454 cipher_mode = CCP_AES_MODE_ECB;
455 break;
456 }
457
458 sc = device_get_softc(dev);
459 mtx_lock(&sc->lock);
460 if (sc->detaching) {
461 mtx_unlock(&sc->lock);
462 return (ENXIO);
463 }
464
465 s = crypto_get_driver_session(cses);
466
467 /* Just grab the first usable queue for now. */
468 for (q = 0; q < nitems(sc->queues); q++)
469 if ((sc->valid_queues & (1 << q)) != 0)
470 break;
471 if (q == nitems(sc->queues)) {
472 mtx_unlock(&sc->lock);
473 return (ENXIO);
474 }
475 s->queue = q;
476
477 switch (csp->csp_mode) {
478 case CSP_MODE_AEAD:
479 s->mode = GCM;
480 break;
481 case CSP_MODE_ETA:
482 s->mode = AUTHENC;
483 break;
484 case CSP_MODE_DIGEST:
485 s->mode = HMAC;
486 break;
487 case CSP_MODE_CIPHER:
488 s->mode = BLKCIPHER;
489 break;
490 }
491
492 if (s->mode == GCM) {
493 if (csp->csp_auth_mlen == 0)
494 s->gmac.hash_len = AES_GMAC_HASH_LEN;
495 else
496 s->gmac.hash_len = csp->csp_auth_mlen;
497 } else if (auth_hash != NULL) {
498 s->hmac.auth_hash = auth_hash;
499 s->hmac.auth_mode = auth_mode;
500 if (csp->csp_auth_mlen == 0)
501 s->hmac.hash_len = auth_hash->hashsize;
502 else
503 s->hmac.hash_len = csp->csp_auth_mlen;
504 ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
505 }
506 if (cipher_mode != CCP_AES_MODE_ECB) {
507 s->blkcipher.cipher_mode = cipher_mode;
508 if (csp->csp_cipher_key != NULL)
509 ccp_aes_setkey(s, csp->csp_cipher_alg,
510 csp->csp_cipher_key, csp->csp_cipher_klen);
511 }
512
513 s->active = true;
514 mtx_unlock(&sc->lock);
515
516 return (0);
517 }
518
519 static void
ccp_freesession(device_t dev,crypto_session_t cses)520 ccp_freesession(device_t dev, crypto_session_t cses)
521 {
522 struct ccp_session *s;
523
524 s = crypto_get_driver_session(cses);
525
526 if (s->pending != 0)
527 device_printf(dev,
528 "session %p freed with %d pending requests\n", s,
529 s->pending);
530 s->active = false;
531 }
532
533 static int
ccp_process(device_t dev,struct cryptop * crp,int hint)534 ccp_process(device_t dev, struct cryptop *crp, int hint)
535 {
536 const struct crypto_session_params *csp;
537 struct ccp_softc *sc;
538 struct ccp_queue *qp;
539 struct ccp_session *s;
540 int error;
541 bool qpheld;
542
543 qpheld = false;
544 qp = NULL;
545
546 csp = crypto_get_params(crp->crp_session);
547 s = crypto_get_driver_session(crp->crp_session);
548 sc = device_get_softc(dev);
549 mtx_lock(&sc->lock);
550 qp = &sc->queues[s->queue];
551 mtx_unlock(&sc->lock);
552 error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
553 if (error != 0)
554 goto out;
555 qpheld = true;
556
557 error = ccp_populate_sglist(qp->cq_sg_crp, &crp->crp_buf);
558 if (error != 0)
559 goto out;
560
561 if (crp->crp_auth_key != NULL) {
562 KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
563 ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
564 }
565 if (crp->crp_cipher_key != NULL)
566 ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
567 csp->csp_cipher_klen);
568
569 switch (s->mode) {
570 case HMAC:
571 if (s->pending != 0) {
572 error = EAGAIN;
573 break;
574 }
575 error = ccp_hmac(qp, s, crp);
576 break;
577 case BLKCIPHER:
578 if (s->pending != 0) {
579 error = EAGAIN;
580 break;
581 }
582 error = ccp_blkcipher(qp, s, crp);
583 break;
584 case AUTHENC:
585 if (s->pending != 0) {
586 error = EAGAIN;
587 break;
588 }
589 error = ccp_authenc(qp, s, crp);
590 break;
591 case GCM:
592 if (s->pending != 0) {
593 error = EAGAIN;
594 break;
595 }
596 error = ccp_gcm(qp, s, crp);
597 break;
598 }
599
600 if (error == 0)
601 s->pending++;
602
603 out:
604 if (qpheld) {
605 if (error != 0) {
606 /*
607 * Squash EAGAIN so callers don't uselessly and
608 * expensively retry if the ring was full.
609 */
610 if (error == EAGAIN)
611 error = ENOMEM;
612 ccp_queue_abort(qp);
613 } else
614 ccp_queue_release(qp);
615 }
616
617 if (error != 0) {
618 DPRINTF(dev, "%s: early error:%d\n", __func__, error);
619 crp->crp_etype = error;
620 crypto_done(crp);
621 }
622 return (0);
623 }
624
625 static device_method_t ccp_methods[] = {
626 DEVMETHOD(device_probe, ccp_probe),
627 DEVMETHOD(device_attach, ccp_attach),
628 DEVMETHOD(device_detach, ccp_detach),
629
630 DEVMETHOD(cryptodev_probesession, ccp_probesession),
631 DEVMETHOD(cryptodev_newsession, ccp_newsession),
632 DEVMETHOD(cryptodev_freesession, ccp_freesession),
633 DEVMETHOD(cryptodev_process, ccp_process),
634
635 DEVMETHOD_END
636 };
637
638 static driver_t ccp_driver = {
639 "ccp",
640 ccp_methods,
641 sizeof(struct ccp_softc)
642 };
643
644 static devclass_t ccp_devclass;
645 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
646 MODULE_VERSION(ccp, 1);
647 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
648 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
649 #if 0 /* There are enough known issues that we shouldn't load automatically */
650 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
651 nitems(ccp_ids));
652 #endif
653
654 static int
ccp_queue_reserve_space(struct ccp_queue * qp,unsigned n,int mflags)655 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
656 {
657 struct ccp_softc *sc;
658
659 mtx_assert(&qp->cq_lock, MA_OWNED);
660 sc = qp->cq_softc;
661
662 if (n < 1 || n >= (1 << sc->ring_size_order))
663 return (EINVAL);
664
665 while (true) {
666 if (ccp_queue_get_ring_space(qp) >= n)
667 return (0);
668 if ((mflags & M_WAITOK) == 0)
669 return (EAGAIN);
670 qp->cq_waiting = true;
671 msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
672 }
673 }
674
675 int
ccp_queue_acquire_reserve(struct ccp_queue * qp,unsigned n,int mflags)676 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
677 {
678 int error;
679
680 mtx_lock(&qp->cq_lock);
681 qp->cq_acq_tail = qp->cq_tail;
682 error = ccp_queue_reserve_space(qp, n, mflags);
683 if (error != 0)
684 mtx_unlock(&qp->cq_lock);
685 return (error);
686 }
687
688 void
ccp_queue_release(struct ccp_queue * qp)689 ccp_queue_release(struct ccp_queue *qp)
690 {
691
692 mtx_assert(&qp->cq_lock, MA_OWNED);
693 if (qp->cq_tail != qp->cq_acq_tail) {
694 wmb();
695 ccp_queue_write_tail(qp);
696 }
697 mtx_unlock(&qp->cq_lock);
698 }
699
700 void
ccp_queue_abort(struct ccp_queue * qp)701 ccp_queue_abort(struct ccp_queue *qp)
702 {
703 unsigned i;
704
705 mtx_assert(&qp->cq_lock, MA_OWNED);
706
707 /* Wipe out any descriptors associated with this aborted txn. */
708 for (i = qp->cq_acq_tail; i != qp->cq_tail;
709 i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
710 memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
711 }
712 qp->cq_tail = qp->cq_acq_tail;
713
714 mtx_unlock(&qp->cq_lock);
715 }
716
717 #ifdef DDB
718 #define _db_show_lock(lo) LOCK_CLASS(lo)->lc_ddb_show(lo)
719 #define db_show_lock(lk) _db_show_lock(&(lk)->lock_object)
720 static void
db_show_ccp_sc(struct ccp_softc * sc)721 db_show_ccp_sc(struct ccp_softc *sc)
722 {
723
724 db_printf("ccp softc at %p\n", sc);
725 db_printf(" cid: %d\n", (int)sc->cid);
726
727 db_printf(" lock: ");
728 db_show_lock(&sc->lock);
729
730 db_printf(" detaching: %d\n", (int)sc->detaching);
731 db_printf(" ring_size_order: %u\n", sc->ring_size_order);
732
733 db_printf(" hw_version: %d\n", (int)sc->hw_version);
734 db_printf(" hw_features: %b\n", (int)sc->hw_features,
735 "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
736 "\11SHA\0103DES\07AES");
737
738 db_printf(" hw status:\n");
739 db_ccp_show_hw(sc);
740 }
741
742 static void
db_show_ccp_qp(struct ccp_queue * qp)743 db_show_ccp_qp(struct ccp_queue *qp)
744 {
745
746 db_printf(" lock: ");
747 db_show_lock(&qp->cq_lock);
748
749 db_printf(" cq_qindex: %u\n", qp->cq_qindex);
750 db_printf(" cq_softc: %p\n", qp->cq_softc);
751
752 db_printf(" head: %u\n", qp->cq_head);
753 db_printf(" tail: %u\n", qp->cq_tail);
754 db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
755 db_printf(" desc_ring: %p\n", qp->desc_ring);
756 db_printf(" completions_ring: %p\n", qp->completions_ring);
757 db_printf(" descriptors (phys): 0x%jx\n",
758 (uintmax_t)qp->desc_ring_bus_addr);
759
760 db_printf(" hw status:\n");
761 db_ccp_show_queue_hw(qp);
762 }
763
DB_SHOW_COMMAND(ccp,db_show_ccp)764 DB_SHOW_COMMAND(ccp, db_show_ccp)
765 {
766 struct ccp_softc *sc;
767 unsigned unit, qindex;
768
769 if (!have_addr)
770 goto usage;
771
772 unit = (unsigned)addr;
773
774 sc = devclass_get_softc(ccp_devclass, unit);
775 if (sc == NULL) {
776 db_printf("No such device ccp%u\n", unit);
777 goto usage;
778 }
779
780 if (count == -1) {
781 db_show_ccp_sc(sc);
782 return;
783 }
784
785 qindex = (unsigned)count;
786 if (qindex >= nitems(sc->queues)) {
787 db_printf("No such queue %u\n", qindex);
788 goto usage;
789 }
790 db_show_ccp_qp(&sc->queues[qindex]);
791 return;
792
793 usage:
794 db_printf("usage: show ccp <unit>[,<qindex>]\n");
795 return;
796 }
797 #endif /* DDB */
798