1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */
2
3 /*-
4 * Copyright (c) 2001 Theo de Raadt
5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6 * Copyright (c) 2014-2021 The FreeBSD Foundation
7 * All rights reserved.
8 *
9 * Portions of this software were developed by John-Mark Gurney
10 * under sponsorship of the FreeBSD Foundation and
11 * Rubicon Communications, LLC (Netgate).
12 *
13 * Portions of this software were developed by Ararat River
14 * Consulting, LLC under sponsorship of the FreeBSD Foundation.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 *
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
25 * 3. The name of the author may not be used to endorse or promote products
26 * derived from this software without specific prior written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 *
39 * Effort sponsored in part by the Defense Advanced Research Projects
40 * Agency (DARPA) and Air Force Research Laboratory, Air Force
41 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
42 */
43
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/malloc.h>
50 #include <sys/mbuf.h>
51 #include <sys/lock.h>
52 #include <sys/mutex.h>
53 #include <sys/proc.h>
54 #include <sys/sysctl.h>
55 #include <sys/errno.h>
56 #include <sys/random.h>
57 #include <sys/conf.h>
58 #include <sys/kernel.h>
59 #include <sys/module.h>
60 #include <sys/fcntl.h>
61 #include <sys/bus.h>
62 #include <sys/sdt.h>
63 #include <sys/syscallsubr.h>
64
65 #include <opencrypto/cryptodev.h>
66 #include <opencrypto/xform.h>
67
68 SDT_PROVIDER_DECLARE(opencrypto);
69
70 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
71
72 #ifdef COMPAT_FREEBSD12
73 /*
74 * Previously, most ioctls were performed against a cloned descriptor
75 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed
76 * against /dev/crypto directly.
77 */
78 #define CRIOGET _IOWR('c', 100, uint32_t)
79 #endif
80
81 /* the following are done against the cloned descriptor */
82
83 #ifdef COMPAT_FREEBSD32
84 #include <sys/mount.h>
85 #include <compat/freebsd32/freebsd32.h>
86
87 struct session_op32 {
88 uint32_t cipher;
89 uint32_t mac;
90 uint32_t keylen;
91 uint32_t key;
92 int mackeylen;
93 uint32_t mackey;
94 uint32_t ses;
95 };
96
97 struct session2_op32 {
98 uint32_t cipher;
99 uint32_t mac;
100 uint32_t keylen;
101 uint32_t key;
102 int mackeylen;
103 uint32_t mackey;
104 uint32_t ses;
105 int crid;
106 int ivlen;
107 int maclen;
108 int pad[2];
109 };
110
111 struct crypt_op32 {
112 uint32_t ses;
113 uint16_t op;
114 uint16_t flags;
115 u_int len;
116 uint32_t src, dst;
117 uint32_t mac;
118 uint32_t iv;
119 };
120
121 struct crypt_aead32 {
122 uint32_t ses;
123 uint16_t op;
124 uint16_t flags;
125 u_int len;
126 u_int aadlen;
127 u_int ivlen;
128 uint32_t src;
129 uint32_t dst;
130 uint32_t aad;
131 uint32_t tag;
132 uint32_t iv;
133 };
134
135 struct crparam32 {
136 uint32_t crp_p;
137 u_int crp_nbits;
138 };
139
140 struct crypt_kop32 {
141 u_int crk_op;
142 u_int crk_status;
143 u_short crk_iparams;
144 u_short crk_oparams;
145 u_int crk_crid;
146 struct crparam32 crk_param[CRK_MAXPARAM];
147 };
148
149 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32)
150 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32)
151 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32)
152 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32)
153 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32)
154 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32)
155
156 static void
session_op_from_32(const struct session_op32 * from,struct session2_op * to)157 session_op_from_32(const struct session_op32 *from, struct session2_op *to)
158 {
159
160 memset(to, 0, sizeof(*to));
161 CP(*from, *to, cipher);
162 CP(*from, *to, mac);
163 CP(*from, *to, keylen);
164 PTRIN_CP(*from, *to, key);
165 CP(*from, *to, mackeylen);
166 PTRIN_CP(*from, *to, mackey);
167 CP(*from, *to, ses);
168 to->crid = CRYPTOCAP_F_HARDWARE;
169 }
170
171 static void
session2_op_from_32(const struct session2_op32 * from,struct session2_op * to)172 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
173 {
174
175 session_op_from_32((const struct session_op32 *)from, to);
176 CP(*from, *to, crid);
177 CP(*from, *to, ivlen);
178 CP(*from, *to, maclen);
179 }
180
181 static void
session_op_to_32(const struct session2_op * from,struct session_op32 * to)182 session_op_to_32(const struct session2_op *from, struct session_op32 *to)
183 {
184
185 CP(*from, *to, cipher);
186 CP(*from, *to, mac);
187 CP(*from, *to, keylen);
188 PTROUT_CP(*from, *to, key);
189 CP(*from, *to, mackeylen);
190 PTROUT_CP(*from, *to, mackey);
191 CP(*from, *to, ses);
192 }
193
194 static void
session2_op_to_32(const struct session2_op * from,struct session2_op32 * to)195 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
196 {
197
198 session_op_to_32(from, (struct session_op32 *)to);
199 CP(*from, *to, crid);
200 }
201
202 static void
crypt_op_from_32(const struct crypt_op32 * from,struct crypt_op * to)203 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
204 {
205
206 CP(*from, *to, ses);
207 CP(*from, *to, op);
208 CP(*from, *to, flags);
209 CP(*from, *to, len);
210 PTRIN_CP(*from, *to, src);
211 PTRIN_CP(*from, *to, dst);
212 PTRIN_CP(*from, *to, mac);
213 PTRIN_CP(*from, *to, iv);
214 }
215
216 static void
crypt_op_to_32(const struct crypt_op * from,struct crypt_op32 * to)217 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
218 {
219
220 CP(*from, *to, ses);
221 CP(*from, *to, op);
222 CP(*from, *to, flags);
223 CP(*from, *to, len);
224 PTROUT_CP(*from, *to, src);
225 PTROUT_CP(*from, *to, dst);
226 PTROUT_CP(*from, *to, mac);
227 PTROUT_CP(*from, *to, iv);
228 }
229
230 static void
crypt_aead_from_32(const struct crypt_aead32 * from,struct crypt_aead * to)231 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to)
232 {
233
234 CP(*from, *to, ses);
235 CP(*from, *to, op);
236 CP(*from, *to, flags);
237 CP(*from, *to, len);
238 CP(*from, *to, aadlen);
239 CP(*from, *to, ivlen);
240 PTRIN_CP(*from, *to, src);
241 PTRIN_CP(*from, *to, dst);
242 PTRIN_CP(*from, *to, aad);
243 PTRIN_CP(*from, *to, tag);
244 PTRIN_CP(*from, *to, iv);
245 }
246
247 static void
crypt_aead_to_32(const struct crypt_aead * from,struct crypt_aead32 * to)248 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to)
249 {
250
251 CP(*from, *to, ses);
252 CP(*from, *to, op);
253 CP(*from, *to, flags);
254 CP(*from, *to, len);
255 CP(*from, *to, aadlen);
256 CP(*from, *to, ivlen);
257 PTROUT_CP(*from, *to, src);
258 PTROUT_CP(*from, *to, dst);
259 PTROUT_CP(*from, *to, aad);
260 PTROUT_CP(*from, *to, tag);
261 PTROUT_CP(*from, *to, iv);
262 }
263
264 static void
crparam_from_32(const struct crparam32 * from,struct crparam * to)265 crparam_from_32(const struct crparam32 *from, struct crparam *to)
266 {
267
268 PTRIN_CP(*from, *to, crp_p);
269 CP(*from, *to, crp_nbits);
270 }
271
272 static void
crparam_to_32(const struct crparam * from,struct crparam32 * to)273 crparam_to_32(const struct crparam *from, struct crparam32 *to)
274 {
275
276 PTROUT_CP(*from, *to, crp_p);
277 CP(*from, *to, crp_nbits);
278 }
279
280 static void
crypt_kop_from_32(const struct crypt_kop32 * from,struct crypt_kop * to)281 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to)
282 {
283 int i;
284
285 CP(*from, *to, crk_op);
286 CP(*from, *to, crk_status);
287 CP(*from, *to, crk_iparams);
288 CP(*from, *to, crk_oparams);
289 CP(*from, *to, crk_crid);
290 for (i = 0; i < CRK_MAXPARAM; i++)
291 crparam_from_32(&from->crk_param[i], &to->crk_param[i]);
292 }
293
294 static void
crypt_kop_to_32(const struct crypt_kop * from,struct crypt_kop32 * to)295 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to)
296 {
297 int i;
298
299 CP(*from, *to, crk_op);
300 CP(*from, *to, crk_status);
301 CP(*from, *to, crk_iparams);
302 CP(*from, *to, crk_oparams);
303 CP(*from, *to, crk_crid);
304 for (i = 0; i < CRK_MAXPARAM; i++)
305 crparam_to_32(&from->crk_param[i], &to->crk_param[i]);
306 }
307 #endif
308
309 static void
session2_op_from_op(const struct session_op * from,struct session2_op * to)310 session2_op_from_op(const struct session_op *from, struct session2_op *to)
311 {
312
313 memset(to, 0, sizeof(*to));
314 memcpy(to, from, sizeof(*from));
315 to->crid = CRYPTOCAP_F_HARDWARE;
316 }
317
318 static void
session2_op_to_op(const struct session2_op * from,struct session_op * to)319 session2_op_to_op(const struct session2_op *from, struct session_op *to)
320 {
321
322 memcpy(to, from, sizeof(*to));
323 }
324
325 struct csession {
326 TAILQ_ENTRY(csession) next;
327 crypto_session_t cses;
328 volatile u_int refs;
329 uint32_t ses;
330 struct mtx lock; /* for op submission */
331
332 struct enc_xform *txform;
333 int hashsize;
334 int ivsize;
335
336 void *key;
337 void *mackey;
338 };
339
340 struct cryptop_data {
341 struct csession *cse;
342
343 char *buf;
344 char *obuf;
345 char *aad;
346 bool done;
347 };
348
349 struct fcrypt {
350 TAILQ_HEAD(csessionlist, csession) csessions;
351 int sesn;
352 struct mtx lock;
353 };
354
355 static bool use_outputbuffers;
356 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
357 &use_outputbuffers, 0,
358 "Use separate output buffers for /dev/crypto requests.");
359
360 static bool use_separate_aad;
361 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
362 &use_separate_aad, 0,
363 "Use separate AAD buffer for /dev/crypto requests.");
364
365 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 };
366 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW,
367 &warninterval,
368 "Delay in seconds between warnings of deprecated /dev/crypto algorithms");
369
370 /*
371 * Check a crypto identifier to see if it requested
372 * a software device/driver. This can be done either
373 * by device name/class or through search constraints.
374 */
375 static int
checkforsoftware(int * cridp)376 checkforsoftware(int *cridp)
377 {
378 int crid;
379
380 crid = *cridp;
381
382 if (!crypto_devallowsoft) {
383 if (crid & CRYPTOCAP_F_SOFTWARE) {
384 if (crid & CRYPTOCAP_F_HARDWARE) {
385 *cridp = CRYPTOCAP_F_HARDWARE;
386 return 0;
387 }
388 return EINVAL;
389 }
390 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
391 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
392 return EINVAL;
393 }
394 return 0;
395 }
396
397 static int
cse_create(struct fcrypt * fcr,struct session2_op * sop)398 cse_create(struct fcrypt *fcr, struct session2_op *sop)
399 {
400 struct crypto_session_params csp;
401 struct csession *cse;
402 struct enc_xform *txform;
403 struct auth_hash *thash;
404 void *key = NULL;
405 void *mackey = NULL;
406 crypto_session_t cses;
407 int crid, error;
408
409 switch (sop->cipher) {
410 case 0:
411 txform = NULL;
412 break;
413 case CRYPTO_AES_CBC:
414 txform = &enc_xform_rijndael128;
415 break;
416 case CRYPTO_AES_XTS:
417 txform = &enc_xform_aes_xts;
418 break;
419 case CRYPTO_NULL_CBC:
420 txform = &enc_xform_null;
421 break;
422 case CRYPTO_CAMELLIA_CBC:
423 txform = &enc_xform_camellia;
424 break;
425 case CRYPTO_AES_ICM:
426 txform = &enc_xform_aes_icm;
427 break;
428 case CRYPTO_AES_NIST_GCM_16:
429 txform = &enc_xform_aes_nist_gcm;
430 break;
431 case CRYPTO_CHACHA20:
432 txform = &enc_xform_chacha20;
433 break;
434 case CRYPTO_AES_CCM_16:
435 txform = &enc_xform_ccm;
436 break;
437 case CRYPTO_CHACHA20_POLY1305:
438 txform = &enc_xform_chacha20_poly1305;
439 break;
440 default:
441 CRYPTDEB("invalid cipher");
442 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
443 return (EINVAL);
444 }
445
446 switch (sop->mac) {
447 case 0:
448 thash = NULL;
449 break;
450 case CRYPTO_POLY1305:
451 thash = &auth_hash_poly1305;
452 break;
453 case CRYPTO_SHA1_HMAC:
454 thash = &auth_hash_hmac_sha1;
455 break;
456 case CRYPTO_SHA2_224_HMAC:
457 thash = &auth_hash_hmac_sha2_224;
458 break;
459 case CRYPTO_SHA2_256_HMAC:
460 thash = &auth_hash_hmac_sha2_256;
461 break;
462 case CRYPTO_SHA2_384_HMAC:
463 thash = &auth_hash_hmac_sha2_384;
464 break;
465 case CRYPTO_SHA2_512_HMAC:
466 thash = &auth_hash_hmac_sha2_512;
467 break;
468 case CRYPTO_RIPEMD160_HMAC:
469 thash = &auth_hash_hmac_ripemd_160;
470 break;
471 #ifdef COMPAT_FREEBSD12
472 case CRYPTO_AES_128_NIST_GMAC:
473 case CRYPTO_AES_192_NIST_GMAC:
474 case CRYPTO_AES_256_NIST_GMAC:
475 /* Should always be paired with GCM. */
476 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
477 CRYPTDEB("GMAC without GCM");
478 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
479 return (EINVAL);
480 }
481 break;
482 #endif
483 case CRYPTO_AES_NIST_GMAC:
484 switch (sop->mackeylen * 8) {
485 case 128:
486 thash = &auth_hash_nist_gmac_aes_128;
487 break;
488 case 192:
489 thash = &auth_hash_nist_gmac_aes_192;
490 break;
491 case 256:
492 thash = &auth_hash_nist_gmac_aes_256;
493 break;
494 default:
495 CRYPTDEB("invalid GMAC key length");
496 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
497 return (EINVAL);
498 }
499 break;
500 case CRYPTO_AES_CCM_CBC_MAC:
501 switch (sop->mackeylen) {
502 case 16:
503 thash = &auth_hash_ccm_cbc_mac_128;
504 break;
505 case 24:
506 thash = &auth_hash_ccm_cbc_mac_192;
507 break;
508 case 32:
509 thash = &auth_hash_ccm_cbc_mac_256;
510 break;
511 default:
512 CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen);
513 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
514 return (EINVAL);
515 }
516 break;
517 case CRYPTO_SHA1:
518 thash = &auth_hash_sha1;
519 break;
520 case CRYPTO_SHA2_224:
521 thash = &auth_hash_sha2_224;
522 break;
523 case CRYPTO_SHA2_256:
524 thash = &auth_hash_sha2_256;
525 break;
526 case CRYPTO_SHA2_384:
527 thash = &auth_hash_sha2_384;
528 break;
529 case CRYPTO_SHA2_512:
530 thash = &auth_hash_sha2_512;
531 break;
532
533 case CRYPTO_NULL_HMAC:
534 thash = &auth_hash_null;
535 break;
536
537 case CRYPTO_BLAKE2B:
538 thash = &auth_hash_blake2b;
539 break;
540 case CRYPTO_BLAKE2S:
541 thash = &auth_hash_blake2s;
542 break;
543
544 default:
545 CRYPTDEB("invalid mac");
546 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
547 return (EINVAL);
548 }
549
550 if (txform == NULL && thash == NULL) {
551 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
552 return (EINVAL);
553 }
554
555 memset(&csp, 0, sizeof(csp));
556 if (use_outputbuffers)
557 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
558
559 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
560 switch (sop->mac) {
561 #ifdef COMPAT_FREEBSD12
562 case CRYPTO_AES_128_NIST_GMAC:
563 case CRYPTO_AES_192_NIST_GMAC:
564 case CRYPTO_AES_256_NIST_GMAC:
565 if (sop->keylen != sop->mackeylen) {
566 SDT_PROBE1(opencrypto, dev, ioctl, error,
567 __LINE__);
568 return (EINVAL);
569 }
570 break;
571 #endif
572 case 0:
573 break;
574 default:
575 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
576 return (EINVAL);
577 }
578 csp.csp_mode = CSP_MODE_AEAD;
579 } else if (sop->cipher == CRYPTO_AES_CCM_16) {
580 switch (sop->mac) {
581 #ifdef COMPAT_FREEBSD12
582 case CRYPTO_AES_CCM_CBC_MAC:
583 if (sop->keylen != sop->mackeylen) {
584 SDT_PROBE1(opencrypto, dev, ioctl, error,
585 __LINE__);
586 return (EINVAL);
587 }
588 thash = NULL;
589 break;
590 #endif
591 case 0:
592 break;
593 default:
594 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
595 return (EINVAL);
596 }
597 csp.csp_mode = CSP_MODE_AEAD;
598 } else if (sop->cipher == CRYPTO_CHACHA20_POLY1305) {
599 if (sop->mac != 0) {
600 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
601 return (EINVAL);
602 }
603 csp.csp_mode = CSP_MODE_AEAD;
604 } else if (txform != NULL && thash != NULL)
605 csp.csp_mode = CSP_MODE_ETA;
606 else if (txform != NULL)
607 csp.csp_mode = CSP_MODE_CIPHER;
608 else
609 csp.csp_mode = CSP_MODE_DIGEST;
610
611 switch (csp.csp_mode) {
612 case CSP_MODE_AEAD:
613 case CSP_MODE_ETA:
614 if (use_separate_aad)
615 csp.csp_flags |= CSP_F_SEPARATE_AAD;
616 break;
617 }
618
619 if (txform != NULL) {
620 csp.csp_cipher_alg = txform->type;
621 csp.csp_cipher_klen = sop->keylen;
622 if (sop->keylen > txform->maxkey ||
623 sop->keylen < txform->minkey) {
624 CRYPTDEB("invalid cipher parameters");
625 error = EINVAL;
626 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
627 goto bail;
628 }
629
630 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
631 error = copyin(sop->key, key, csp.csp_cipher_klen);
632 if (error) {
633 CRYPTDEB("invalid key");
634 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
635 goto bail;
636 }
637 csp.csp_cipher_key = key;
638 csp.csp_ivlen = txform->ivsize;
639 }
640
641 if (thash != NULL) {
642 csp.csp_auth_alg = thash->type;
643 csp.csp_auth_klen = sop->mackeylen;
644 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) {
645 CRYPTDEB("invalid mac key length");
646 error = EINVAL;
647 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
648 goto bail;
649 }
650
651 if (csp.csp_auth_klen != 0) {
652 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK);
653 error = copyin(sop->mackey, mackey, csp.csp_auth_klen);
654 if (error) {
655 CRYPTDEB("invalid mac key");
656 SDT_PROBE1(opencrypto, dev, ioctl, error,
657 __LINE__);
658 goto bail;
659 }
660 csp.csp_auth_key = mackey;
661 }
662
663 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
664 csp.csp_ivlen = AES_GCM_IV_LEN;
665 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
666 csp.csp_ivlen = AES_CCM_IV_LEN;
667 }
668
669 if (sop->ivlen != 0) {
670 if (csp.csp_ivlen == 0) {
671 CRYPTDEB("does not support an IV");
672 error = EINVAL;
673 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
674 goto bail;
675 }
676 csp.csp_ivlen = sop->ivlen;
677 }
678 if (sop->maclen != 0) {
679 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) {
680 CRYPTDEB("does not support a MAC");
681 error = EINVAL;
682 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
683 goto bail;
684 }
685 csp.csp_auth_mlen = sop->maclen;
686 }
687
688 crid = sop->crid;
689 error = checkforsoftware(&crid);
690 if (error) {
691 CRYPTDEB("checkforsoftware");
692 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
693 goto bail;
694 }
695 error = crypto_newsession(&cses, &csp, crid);
696 if (error) {
697 CRYPTDEB("crypto_newsession");
698 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
699 goto bail;
700 }
701
702 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO);
703 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
704 refcount_init(&cse->refs, 1);
705 cse->key = key;
706 cse->mackey = mackey;
707 cse->cses = cses;
708 cse->txform = txform;
709 if (sop->maclen != 0)
710 cse->hashsize = sop->maclen;
711 else if (thash != NULL)
712 cse->hashsize = thash->hashsize;
713 else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
714 cse->hashsize = AES_GMAC_HASH_LEN;
715 else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16)
716 cse->hashsize = AES_CBC_MAC_HASH_LEN;
717 else if (csp.csp_cipher_alg == CRYPTO_CHACHA20_POLY1305)
718 cse->hashsize = POLY1305_HASH_LEN;
719 cse->ivsize = csp.csp_ivlen;
720
721 mtx_lock(&fcr->lock);
722 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
723 cse->ses = fcr->sesn++;
724 mtx_unlock(&fcr->lock);
725
726 sop->ses = cse->ses;
727
728 /* return hardware/driver id */
729 sop->crid = crypto_ses2hid(cse->cses);
730 bail:
731 if (error) {
732 free(key, M_XDATA);
733 free(mackey, M_XDATA);
734 }
735 return (error);
736 }
737
738 static struct csession *
cse_find(struct fcrypt * fcr,u_int ses)739 cse_find(struct fcrypt *fcr, u_int ses)
740 {
741 struct csession *cse;
742
743 mtx_lock(&fcr->lock);
744 TAILQ_FOREACH(cse, &fcr->csessions, next) {
745 if (cse->ses == ses) {
746 refcount_acquire(&cse->refs);
747 mtx_unlock(&fcr->lock);
748 return (cse);
749 }
750 }
751 mtx_unlock(&fcr->lock);
752 return (NULL);
753 }
754
755 static void
cse_free(struct csession * cse)756 cse_free(struct csession *cse)
757 {
758
759 if (!refcount_release(&cse->refs))
760 return;
761 crypto_freesession(cse->cses);
762 mtx_destroy(&cse->lock);
763 if (cse->key)
764 free(cse->key, M_XDATA);
765 if (cse->mackey)
766 free(cse->mackey, M_XDATA);
767 free(cse, M_XDATA);
768 }
769
770 static bool
cse_delete(struct fcrypt * fcr,u_int ses)771 cse_delete(struct fcrypt *fcr, u_int ses)
772 {
773 struct csession *cse;
774
775 mtx_lock(&fcr->lock);
776 TAILQ_FOREACH(cse, &fcr->csessions, next) {
777 if (cse->ses == ses) {
778 TAILQ_REMOVE(&fcr->csessions, cse, next);
779 mtx_unlock(&fcr->lock);
780 cse_free(cse);
781 return (true);
782 }
783 }
784 mtx_unlock(&fcr->lock);
785 return (false);
786 }
787
788 static struct cryptop_data *
cod_alloc(struct csession * cse,size_t aad_len,size_t len)789 cod_alloc(struct csession *cse, size_t aad_len, size_t len)
790 {
791 struct cryptop_data *cod;
792
793 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
794
795 cod->cse = cse;
796 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
797 if (aad_len != 0)
798 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK);
799 cod->buf = malloc(len, M_XDATA, M_WAITOK);
800 } else
801 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK);
802 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
803 cod->obuf = malloc(len, M_XDATA, M_WAITOK);
804 return (cod);
805 }
806
807 static void
cod_free(struct cryptop_data * cod)808 cod_free(struct cryptop_data *cod)
809 {
810
811 free(cod->aad, M_XDATA);
812 free(cod->obuf, M_XDATA);
813 free(cod->buf, M_XDATA);
814 free(cod, M_XDATA);
815 }
816
817 static int
cryptodev_cb(struct cryptop * crp)818 cryptodev_cb(struct cryptop *crp)
819 {
820 struct cryptop_data *cod = crp->crp_opaque;
821
822 /*
823 * Lock to ensure the wakeup() is not missed by the loops
824 * waiting on cod->done in cryptodev_op() and
825 * cryptodev_aead().
826 */
827 mtx_lock(&cod->cse->lock);
828 cod->done = true;
829 mtx_unlock(&cod->cse->lock);
830 wakeup(cod);
831 return (0);
832 }
833
834 static int
cryptodev_op(struct csession * cse,const struct crypt_op * cop)835 cryptodev_op(struct csession *cse, const struct crypt_op *cop)
836 {
837 const struct crypto_session_params *csp;
838 struct cryptop_data *cod = NULL;
839 struct cryptop *crp = NULL;
840 char *dst;
841 int error;
842
843 if (cop->len > 256*1024-4) {
844 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
845 return (E2BIG);
846 }
847
848 if (cse->txform) {
849 if ((cop->len % cse->txform->blocksize) != 0) {
850 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
851 return (EINVAL);
852 }
853 }
854
855 if (cop->mac && cse->hashsize == 0) {
856 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
857 return (EINVAL);
858 }
859
860 /*
861 * The COP_F_CIPHER_FIRST flag predates explicit session
862 * modes, but the only way it was used was for EtA so allow it
863 * as long as it is consistent with EtA.
864 */
865 if (cop->flags & COP_F_CIPHER_FIRST) {
866 if (cop->op != COP_ENCRYPT) {
867 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
868 return (EINVAL);
869 }
870 }
871
872 cod = cod_alloc(cse, 0, cop->len + cse->hashsize);
873 dst = cop->dst;
874
875 crp = crypto_getreq(cse->cses, M_WAITOK);
876
877 error = copyin(cop->src, cod->buf, cop->len);
878 if (error) {
879 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
880 goto bail;
881 }
882 crp->crp_payload_start = 0;
883 crp->crp_payload_length = cop->len;
884 if (cse->hashsize)
885 crp->crp_digest_start = cop->len;
886
887 csp = crypto_get_params(cse->cses);
888 switch (csp->csp_mode) {
889 case CSP_MODE_COMPRESS:
890 switch (cop->op) {
891 case COP_ENCRYPT:
892 crp->crp_op = CRYPTO_OP_COMPRESS;
893 break;
894 case COP_DECRYPT:
895 crp->crp_op = CRYPTO_OP_DECOMPRESS;
896 break;
897 default:
898 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
899 error = EINVAL;
900 goto bail;
901 }
902 break;
903 case CSP_MODE_CIPHER:
904 if (cop->len == 0 ||
905 (cop->iv == NULL && cop->len == cse->ivsize)) {
906 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
907 error = EINVAL;
908 goto bail;
909 }
910 switch (cop->op) {
911 case COP_ENCRYPT:
912 crp->crp_op = CRYPTO_OP_ENCRYPT;
913 break;
914 case COP_DECRYPT:
915 crp->crp_op = CRYPTO_OP_DECRYPT;
916 break;
917 default:
918 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
919 error = EINVAL;
920 goto bail;
921 }
922 break;
923 case CSP_MODE_DIGEST:
924 switch (cop->op) {
925 case 0:
926 case COP_ENCRYPT:
927 case COP_DECRYPT:
928 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
929 if (cod->obuf != NULL)
930 crp->crp_digest_start = 0;
931 break;
932 default:
933 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
934 error = EINVAL;
935 goto bail;
936 }
937 break;
938 case CSP_MODE_AEAD:
939 if (cse->ivsize != 0 && cop->iv == NULL) {
940 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
941 error = EINVAL;
942 goto bail;
943 }
944 /* FALLTHROUGH */
945 case CSP_MODE_ETA:
946 switch (cop->op) {
947 case COP_ENCRYPT:
948 crp->crp_op = CRYPTO_OP_ENCRYPT |
949 CRYPTO_OP_COMPUTE_DIGEST;
950 break;
951 case COP_DECRYPT:
952 crp->crp_op = CRYPTO_OP_DECRYPT |
953 CRYPTO_OP_VERIFY_DIGEST;
954 break;
955 default:
956 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
957 error = EINVAL;
958 goto bail;
959 }
960 break;
961 default:
962 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
963 error = EINVAL;
964 goto bail;
965 }
966
967 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
968 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
969 if (cod->obuf)
970 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
971 crp->crp_callback = cryptodev_cb;
972 crp->crp_opaque = cod;
973
974 if (cop->iv) {
975 if (cse->ivsize == 0) {
976 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
977 error = EINVAL;
978 goto bail;
979 }
980 error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
981 if (error) {
982 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
983 goto bail;
984 }
985 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
986 } else if (cse->ivsize != 0) {
987 if (crp->crp_payload_length < cse->ivsize) {
988 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
989 error = EINVAL;
990 goto bail;
991 }
992 crp->crp_iv_start = 0;
993 crp->crp_payload_length -= cse->ivsize;
994 if (crp->crp_payload_length != 0)
995 crp->crp_payload_start = cse->ivsize;
996 dst += cse->ivsize;
997 }
998
999 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1000 error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
1001 cse->hashsize);
1002 if (error) {
1003 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1004 goto bail;
1005 }
1006 }
1007 again:
1008 /*
1009 * Let the dispatch run unlocked, then, interlock against the
1010 * callback before checking if the operation completed and going
1011 * to sleep. This insures drivers don't inherit our lock which
1012 * results in a lock order reversal between crypto_dispatch forced
1013 * entry and the crypto_done callback into us.
1014 */
1015 error = crypto_dispatch(crp);
1016 if (error != 0) {
1017 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1018 goto bail;
1019 }
1020
1021 mtx_lock(&cse->lock);
1022 while (!cod->done)
1023 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1024 mtx_unlock(&cse->lock);
1025
1026 if (crp->crp_etype == EAGAIN) {
1027 crp->crp_etype = 0;
1028 crp->crp_flags &= ~CRYPTO_F_DONE;
1029 cod->done = false;
1030 goto again;
1031 }
1032
1033 if (crp->crp_etype != 0) {
1034 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1035 error = crp->crp_etype;
1036 goto bail;
1037 }
1038
1039 if (cop->dst != NULL) {
1040 error = copyout(cod->obuf != NULL ? cod->obuf :
1041 cod->buf + crp->crp_payload_start, dst,
1042 crp->crp_payload_length);
1043 if (error) {
1044 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1045 goto bail;
1046 }
1047 }
1048
1049 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1050 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1051 crp->crp_digest_start, cop->mac, cse->hashsize);
1052 if (error) {
1053 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1054 goto bail;
1055 }
1056 }
1057
1058 bail:
1059 crypto_freereq(crp);
1060 cod_free(cod);
1061
1062 return (error);
1063 }
1064
1065 static int
cryptodev_aead(struct csession * cse,struct crypt_aead * caead)1066 cryptodev_aead(struct csession *cse, struct crypt_aead *caead)
1067 {
1068 const struct crypto_session_params *csp;
1069 struct cryptop_data *cod = NULL;
1070 struct cryptop *crp = NULL;
1071 char *dst;
1072 int error;
1073
1074 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
1075 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1076 return (E2BIG);
1077 }
1078
1079 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
1080 (caead->len % cse->txform->blocksize) != 0) {
1081 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1082 return (EINVAL);
1083 }
1084
1085 /*
1086 * The COP_F_CIPHER_FIRST flag predates explicit session
1087 * modes, but the only way it was used was for EtA so allow it
1088 * as long as it is consistent with EtA.
1089 */
1090 if (caead->flags & COP_F_CIPHER_FIRST) {
1091 if (caead->op != COP_ENCRYPT) {
1092 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1093 return (EINVAL);
1094 }
1095 }
1096
1097 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize);
1098 dst = caead->dst;
1099
1100 crp = crypto_getreq(cse->cses, M_WAITOK);
1101
1102 if (cod->aad != NULL)
1103 error = copyin(caead->aad, cod->aad, caead->aadlen);
1104 else
1105 error = copyin(caead->aad, cod->buf, caead->aadlen);
1106 if (error) {
1107 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1108 goto bail;
1109 }
1110 crp->crp_aad = cod->aad;
1111 crp->crp_aad_start = 0;
1112 crp->crp_aad_length = caead->aadlen;
1113
1114 if (cod->aad != NULL)
1115 crp->crp_payload_start = 0;
1116 else
1117 crp->crp_payload_start = caead->aadlen;
1118 error = copyin(caead->src, cod->buf + crp->crp_payload_start,
1119 caead->len);
1120 if (error) {
1121 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1122 goto bail;
1123 }
1124 crp->crp_payload_length = caead->len;
1125 if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
1126 crp->crp_digest_start = crp->crp_payload_output_start +
1127 caead->len;
1128 else
1129 crp->crp_digest_start = crp->crp_payload_start + caead->len;
1130
1131 csp = crypto_get_params(cse->cses);
1132 switch (csp->csp_mode) {
1133 case CSP_MODE_AEAD:
1134 case CSP_MODE_ETA:
1135 switch (caead->op) {
1136 case COP_ENCRYPT:
1137 crp->crp_op = CRYPTO_OP_ENCRYPT |
1138 CRYPTO_OP_COMPUTE_DIGEST;
1139 break;
1140 case COP_DECRYPT:
1141 crp->crp_op = CRYPTO_OP_DECRYPT |
1142 CRYPTO_OP_VERIFY_DIGEST;
1143 break;
1144 default:
1145 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1146 error = EINVAL;
1147 goto bail;
1148 }
1149 break;
1150 default:
1151 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1152 error = EINVAL;
1153 goto bail;
1154 }
1155
1156 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
1157 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
1158 cse->hashsize);
1159 if (cod->obuf != NULL)
1160 crypto_use_output_buf(crp, cod->obuf, caead->len +
1161 cse->hashsize);
1162 crp->crp_callback = cryptodev_cb;
1163 crp->crp_opaque = cod;
1164
1165 if (caead->iv) {
1166 /*
1167 * Permit a 16-byte IV for AES-XTS, but only use the
1168 * first 8 bytes as a block number.
1169 */
1170 if (csp->csp_mode == CSP_MODE_ETA &&
1171 csp->csp_cipher_alg == CRYPTO_AES_XTS &&
1172 caead->ivlen == AES_BLOCK_LEN)
1173 caead->ivlen = AES_XTS_IV_LEN;
1174
1175 if (cse->ivsize == 0) {
1176 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1177 error = EINVAL;
1178 goto bail;
1179 }
1180 if (caead->ivlen != cse->ivsize) {
1181 error = EINVAL;
1182 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1183 goto bail;
1184 }
1185
1186 error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
1187 if (error) {
1188 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1189 goto bail;
1190 }
1191 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1192 } else {
1193 error = EINVAL;
1194 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1195 goto bail;
1196 }
1197
1198 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1199 error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1200 cse->hashsize);
1201 if (error) {
1202 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1203 goto bail;
1204 }
1205 }
1206 again:
1207 /*
1208 * Let the dispatch run unlocked, then, interlock against the
1209 * callback before checking if the operation completed and going
1210 * to sleep. This insures drivers don't inherit our lock which
1211 * results in a lock order reversal between crypto_dispatch forced
1212 * entry and the crypto_done callback into us.
1213 */
1214 error = crypto_dispatch(crp);
1215 if (error != 0) {
1216 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1217 goto bail;
1218 }
1219
1220 mtx_lock(&cse->lock);
1221 while (!cod->done)
1222 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1223 mtx_unlock(&cse->lock);
1224
1225 if (crp->crp_etype == EAGAIN) {
1226 crp->crp_etype = 0;
1227 crp->crp_flags &= ~CRYPTO_F_DONE;
1228 cod->done = false;
1229 goto again;
1230 }
1231
1232 if (crp->crp_etype != 0) {
1233 error = crp->crp_etype;
1234 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1235 goto bail;
1236 }
1237
1238 if (caead->dst != NULL) {
1239 error = copyout(cod->obuf != NULL ? cod->obuf :
1240 cod->buf + crp->crp_payload_start, dst,
1241 crp->crp_payload_length);
1242 if (error) {
1243 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1244 goto bail;
1245 }
1246 }
1247
1248 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1249 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1250 crp->crp_digest_start, caead->tag, cse->hashsize);
1251 if (error) {
1252 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1253 goto bail;
1254 }
1255 }
1256
1257 bail:
1258 crypto_freereq(crp);
1259 cod_free(cod);
1260
1261 return (error);
1262 }
1263
1264 static void
cryptodevkey_cb(struct cryptkop * krp)1265 cryptodevkey_cb(struct cryptkop *krp)
1266 {
1267
1268 wakeup_one(krp);
1269 }
1270
1271 static int
cryptodev_key(struct crypt_kop * kop)1272 cryptodev_key(struct crypt_kop *kop)
1273 {
1274 struct cryptkop *krp = NULL;
1275 int error = EINVAL;
1276 int in, out, size, i;
1277
1278 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) {
1279 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1280 return (EFBIG);
1281 }
1282
1283 in = kop->crk_iparams;
1284 out = kop->crk_oparams;
1285 switch (kop->crk_op) {
1286 case CRK_MOD_EXP:
1287 if (in == 3 && out == 1)
1288 break;
1289 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1290 return (EINVAL);
1291 case CRK_MOD_EXP_CRT:
1292 if (in == 6 && out == 1)
1293 break;
1294 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1295 return (EINVAL);
1296 case CRK_DSA_SIGN:
1297 if (in == 5 && out == 2)
1298 break;
1299 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1300 return (EINVAL);
1301 case CRK_DSA_VERIFY:
1302 if (in == 7 && out == 0)
1303 break;
1304 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1305 return (EINVAL);
1306 case CRK_DH_COMPUTE_KEY:
1307 if (in == 3 && out == 1)
1308 break;
1309 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1310 return (EINVAL);
1311 default:
1312 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1313 return (EINVAL);
1314 }
1315
1316 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO);
1317 krp->krp_op = kop->crk_op;
1318 krp->krp_status = kop->crk_status;
1319 krp->krp_iparams = kop->crk_iparams;
1320 krp->krp_oparams = kop->crk_oparams;
1321 krp->krp_crid = kop->crk_crid;
1322 krp->krp_status = 0;
1323 krp->krp_callback = cryptodevkey_cb;
1324
1325 for (i = 0; i < CRK_MAXPARAM; i++) {
1326 if (kop->crk_param[i].crp_nbits > 65536) {
1327 /* Limit is the same as in OpenBSD */
1328 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1329 goto fail;
1330 }
1331 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
1332 }
1333 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
1334 size = (krp->krp_param[i].crp_nbits + 7) / 8;
1335 if (size == 0)
1336 continue;
1337 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK);
1338 if (i >= krp->krp_iparams)
1339 continue;
1340 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size);
1341 if (error) {
1342 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1343 goto fail;
1344 }
1345 }
1346
1347 error = crypto_kdispatch(krp);
1348 if (error) {
1349 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1350 goto fail;
1351 }
1352 error = tsleep(krp, PSOCK, "crydev", 0);
1353 if (error) {
1354 /* XXX can this happen? if so, how do we recover? */
1355 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1356 goto fail;
1357 }
1358
1359 kop->crk_crid = krp->krp_hid; /* device that did the work */
1360 if (krp->krp_status != 0) {
1361 error = krp->krp_status;
1362 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1363 goto fail;
1364 }
1365
1366 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) {
1367 size = (krp->krp_param[i].crp_nbits + 7) / 8;
1368 if (size == 0)
1369 continue;
1370 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size);
1371 if (error) {
1372 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1373 goto fail;
1374 }
1375 }
1376
1377 fail:
1378 if (krp) {
1379 kop->crk_status = krp->krp_status;
1380 for (i = 0; i < CRK_MAXPARAM; i++) {
1381 if (krp->krp_param[i].crp_p)
1382 free(krp->krp_param[i].crp_p, M_XDATA);
1383 }
1384 free(krp, M_XDATA);
1385 }
1386 return (error);
1387 }
1388
1389 static int
cryptodev_find(struct crypt_find_op * find)1390 cryptodev_find(struct crypt_find_op *find)
1391 {
1392 device_t dev;
1393 size_t fnlen = sizeof find->name;
1394
1395 if (find->crid != -1) {
1396 dev = crypto_find_device_byhid(find->crid);
1397 if (dev == NULL)
1398 return (ENOENT);
1399 strncpy(find->name, device_get_nameunit(dev), fnlen);
1400 find->name[fnlen - 1] = '\x0';
1401 } else {
1402 find->name[fnlen - 1] = '\x0';
1403 find->crid = crypto_find_driver(find->name);
1404 if (find->crid == -1)
1405 return (ENOENT);
1406 }
1407 return (0);
1408 }
1409
1410 static void
fcrypt_dtor(void * data)1411 fcrypt_dtor(void *data)
1412 {
1413 struct fcrypt *fcr = data;
1414 struct csession *cse;
1415
1416 while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1417 TAILQ_REMOVE(&fcr->csessions, cse, next);
1418 KASSERT(refcount_load(&cse->refs) == 1,
1419 ("%s: crypto session %p with %d refs", __func__, cse,
1420 refcount_load(&cse->refs)));
1421 cse_free(cse);
1422 }
1423 mtx_destroy(&fcr->lock);
1424 free(fcr, M_XDATA);
1425 }
1426
1427 static int
crypto_open(struct cdev * dev,int oflags,int devtype,struct thread * td)1428 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
1429 {
1430 struct fcrypt *fcr;
1431 int error;
1432
1433 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO);
1434 TAILQ_INIT(&fcr->csessions);
1435 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1436 error = devfs_set_cdevpriv(fcr, fcrypt_dtor);
1437 if (error)
1438 fcrypt_dtor(fcr);
1439 return (error);
1440 }
1441
1442 static int
crypto_ioctl(struct cdev * dev,u_long cmd,caddr_t data,int flag,struct thread * td)1443 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
1444 struct thread *td)
1445 {
1446 static struct timeval keywarn, featwarn;
1447 struct fcrypt *fcr;
1448 struct csession *cse;
1449 struct session2_op *sop;
1450 struct crypt_op *cop;
1451 struct crypt_aead *caead;
1452 struct crypt_kop *kop;
1453 uint32_t ses;
1454 int error = 0;
1455 union {
1456 struct session2_op sopc;
1457 #ifdef COMPAT_FREEBSD32
1458 struct crypt_op copc;
1459 struct crypt_aead aeadc;
1460 struct crypt_kop kopc;
1461 #endif
1462 } thunk;
1463 #ifdef COMPAT_FREEBSD32
1464 u_long cmd32;
1465 void *data32;
1466
1467 cmd32 = 0;
1468 data32 = NULL;
1469 switch (cmd) {
1470 case CIOCGSESSION32:
1471 cmd32 = cmd;
1472 data32 = data;
1473 cmd = CIOCGSESSION;
1474 data = (void *)&thunk.sopc;
1475 session_op_from_32((struct session_op32 *)data32, &thunk.sopc);
1476 break;
1477 case CIOCGSESSION232:
1478 cmd32 = cmd;
1479 data32 = data;
1480 cmd = CIOCGSESSION2;
1481 data = (void *)&thunk.sopc;
1482 session2_op_from_32((struct session2_op32 *)data32,
1483 &thunk.sopc);
1484 break;
1485 case CIOCCRYPT32:
1486 cmd32 = cmd;
1487 data32 = data;
1488 cmd = CIOCCRYPT;
1489 data = (void *)&thunk.copc;
1490 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc);
1491 break;
1492 case CIOCCRYPTAEAD32:
1493 cmd32 = cmd;
1494 data32 = data;
1495 cmd = CIOCCRYPTAEAD;
1496 data = (void *)&thunk.aeadc;
1497 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc);
1498 break;
1499 case CIOCKEY32:
1500 case CIOCKEY232:
1501 cmd32 = cmd;
1502 data32 = data;
1503 if (cmd == CIOCKEY32)
1504 cmd = CIOCKEY;
1505 else
1506 cmd = CIOCKEY2;
1507 data = (void *)&thunk.kopc;
1508 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc);
1509 break;
1510 }
1511 #endif
1512
1513 devfs_get_cdevpriv((void **)&fcr);
1514
1515 switch (cmd) {
1516 #ifdef COMPAT_FREEBSD12
1517 case CRIOGET:
1518 /*
1519 * NB: This may fail in cases that the old
1520 * implementation did not if the current process has
1521 * restricted filesystem access (e.g. running in a
1522 * jail that does not expose /dev/crypto or in
1523 * capability mode).
1524 */
1525 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE,
1526 O_RDWR, 0);
1527 if (error == 0)
1528 *(uint32_t *)data = td->td_retval[0];
1529 break;
1530 #endif
1531 case CIOCGSESSION:
1532 case CIOCGSESSION2:
1533 if (cmd == CIOCGSESSION) {
1534 session2_op_from_op((void *)data, &thunk.sopc);
1535 sop = &thunk.sopc;
1536 } else
1537 sop = (struct session2_op *)data;
1538
1539 error = cse_create(fcr, sop);
1540 if (cmd == CIOCGSESSION && error == 0)
1541 session2_op_to_op(sop, (void *)data);
1542 break;
1543 case CIOCFSESSION:
1544 ses = *(uint32_t *)data;
1545 if (!cse_delete(fcr, ses)) {
1546 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1547 return (EINVAL);
1548 }
1549 break;
1550 case CIOCCRYPT:
1551 cop = (struct crypt_op *)data;
1552 cse = cse_find(fcr, cop->ses);
1553 if (cse == NULL) {
1554 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1555 return (EINVAL);
1556 }
1557 error = cryptodev_op(cse, cop);
1558 cse_free(cse);
1559 break;
1560 case CIOCKEY:
1561 case CIOCKEY2:
1562 if (ratecheck(&keywarn, &warninterval))
1563 gone_in(14,
1564 "Asymmetric crypto operations via /dev/crypto");
1565
1566 if (!crypto_userasymcrypto) {
1567 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1568 return (EPERM); /* XXX compat? */
1569 }
1570 kop = (struct crypt_kop *)data;
1571 if (cmd == CIOCKEY) {
1572 /* NB: crypto core enforces s/w driver use */
1573 kop->crk_crid =
1574 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
1575 }
1576 mtx_lock(&Giant);
1577 error = cryptodev_key(kop);
1578 mtx_unlock(&Giant);
1579 break;
1580 case CIOCASYMFEAT:
1581 if (ratecheck(&featwarn, &warninterval))
1582 gone_in(14,
1583 "Asymmetric crypto features via /dev/crypto");
1584
1585 if (!crypto_userasymcrypto) {
1586 /*
1587 * NB: if user asym crypto operations are
1588 * not permitted return "no algorithms"
1589 * so well-behaved applications will just
1590 * fallback to doing them in software.
1591 */
1592 *(int *)data = 0;
1593 } else {
1594 error = crypto_getfeat((int *)data);
1595 if (error)
1596 SDT_PROBE1(opencrypto, dev, ioctl, error,
1597 __LINE__);
1598 }
1599 break;
1600 case CIOCFINDDEV:
1601 error = cryptodev_find((struct crypt_find_op *)data);
1602 break;
1603 case CIOCCRYPTAEAD:
1604 caead = (struct crypt_aead *)data;
1605 cse = cse_find(fcr, caead->ses);
1606 if (cse == NULL) {
1607 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1608 return (EINVAL);
1609 }
1610 error = cryptodev_aead(cse, caead);
1611 cse_free(cse);
1612 break;
1613 default:
1614 error = EINVAL;
1615 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1616 break;
1617 }
1618
1619 #ifdef COMPAT_FREEBSD32
1620 switch (cmd32) {
1621 case CIOCGSESSION32:
1622 if (error == 0)
1623 session_op_to_32((void *)data, data32);
1624 break;
1625 case CIOCGSESSION232:
1626 if (error == 0)
1627 session2_op_to_32((void *)data, data32);
1628 break;
1629 case CIOCCRYPT32:
1630 if (error == 0)
1631 crypt_op_to_32((void *)data, data32);
1632 break;
1633 case CIOCCRYPTAEAD32:
1634 if (error == 0)
1635 crypt_aead_to_32((void *)data, data32);
1636 break;
1637 case CIOCKEY32:
1638 case CIOCKEY232:
1639 crypt_kop_to_32((void *)data, data32);
1640 break;
1641 }
1642 #endif
1643 return (error);
1644 }
1645
1646 static struct cdevsw crypto_cdevsw = {
1647 .d_version = D_VERSION,
1648 .d_open = crypto_open,
1649 .d_ioctl = crypto_ioctl,
1650 .d_name = "crypto",
1651 };
1652 static struct cdev *crypto_dev;
1653
1654 /*
1655 * Initialization code, both for static and dynamic loading.
1656 */
1657 static int
cryptodev_modevent(module_t mod,int type,void * unused)1658 cryptodev_modevent(module_t mod, int type, void *unused)
1659 {
1660 switch (type) {
1661 case MOD_LOAD:
1662 if (bootverbose)
1663 printf("crypto: <crypto device>\n");
1664 crypto_dev = make_dev(&crypto_cdevsw, 0,
1665 UID_ROOT, GID_WHEEL, 0666,
1666 "crypto");
1667 return 0;
1668 case MOD_UNLOAD:
1669 /*XXX disallow if active sessions */
1670 destroy_dev(crypto_dev);
1671 return 0;
1672 }
1673 return EINVAL;
1674 }
1675
1676 static moduledata_t cryptodev_mod = {
1677 "cryptodev",
1678 cryptodev_modevent,
1679 0
1680 };
1681 MODULE_VERSION(cryptodev, 1);
1682 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1683 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1684 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
1685