1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */ 2 3 /*- 4 * Copyright (c) 2001 Theo de Raadt 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * Copyright (c) 2014-2021 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * Portions of this software were developed by John-Mark Gurney 10 * under sponsorship of the FreeBSD Foundation and 11 * Rubicon Communications, LLC (Netgate). 12 * 13 * Portions of this software were developed by Ararat River 14 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 3. The name of the author may not be used to endorse or promote products 26 * derived from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 33 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 37 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * Effort sponsored in part by the Defense Advanced Research Projects 40 * Agency (DARPA) and Air Force Research Laboratory, Air Force 41 * Materiel Command, USAF, under agreement number F30602-01-2-0537. 42 */ 43 44 #include <sys/cdefs.h> 45 __FBSDID("$FreeBSD$"); 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/malloc.h> 50 #include <sys/mbuf.h> 51 #include <sys/lock.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sysctl.h> 55 #include <sys/errno.h> 56 #include <sys/random.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/module.h> 60 #include <sys/fcntl.h> 61 #include <sys/bus.h> 62 #include <sys/sdt.h> 63 #include <sys/syscallsubr.h> 64 65 #include <opencrypto/cryptodev.h> 66 #include <opencrypto/xform.h> 67 68 SDT_PROVIDER_DECLARE(opencrypto); 69 70 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/); 71 72 #ifdef COMPAT_FREEBSD12 73 /* 74 * Previously, most ioctls were performed against a cloned descriptor 75 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed 76 * against /dev/crypto directly. 77 */ 78 #define CRIOGET _IOWR('c', 100, uint32_t) 79 #endif 80 81 /* the following are done against the cloned descriptor */ 82 83 #ifdef COMPAT_FREEBSD32 84 #include <sys/mount.h> 85 #include <compat/freebsd32/freebsd32.h> 86 87 struct session_op32 { 88 uint32_t cipher; 89 uint32_t mac; 90 uint32_t keylen; 91 uint32_t key; 92 int mackeylen; 93 uint32_t mackey; 94 uint32_t ses; 95 }; 96 97 struct session2_op32 { 98 uint32_t cipher; 99 uint32_t mac; 100 uint32_t keylen; 101 uint32_t key; 102 int mackeylen; 103 uint32_t mackey; 104 uint32_t ses; 105 int crid; 106 int ivlen; 107 int maclen; 108 int pad[2]; 109 }; 110 111 struct crypt_op32 { 112 uint32_t ses; 113 uint16_t op; 114 uint16_t flags; 115 u_int len; 116 uint32_t src, dst; 117 uint32_t mac; 118 uint32_t iv; 119 }; 120 121 struct crypt_aead32 { 122 uint32_t ses; 123 uint16_t op; 124 uint16_t flags; 125 u_int len; 126 u_int aadlen; 127 u_int ivlen; 128 uint32_t src; 129 uint32_t dst; 130 uint32_t aad; 131 uint32_t tag; 132 uint32_t iv; 133 }; 134 135 struct crparam32 { 136 uint32_t crp_p; 137 u_int crp_nbits; 138 }; 139 140 struct crypt_kop32 { 141 u_int crk_op; 142 u_int crk_status; 143 u_short crk_iparams; 144 u_short crk_oparams; 145 u_int crk_crid; 146 struct crparam32 crk_param[CRK_MAXPARAM]; 147 }; 148 149 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32) 150 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32) 151 #define CIOCKEY32 _IOWR('c', 104, struct crypt_kop32) 152 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32) 153 #define CIOCKEY232 _IOWR('c', 107, struct crypt_kop32) 154 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32) 155 156 static void 157 session_op_from_32(const struct session_op32 *from, struct session2_op *to) 158 { 159 160 memset(to, 0, sizeof(*to)); 161 CP(*from, *to, cipher); 162 CP(*from, *to, mac); 163 CP(*from, *to, keylen); 164 PTRIN_CP(*from, *to, key); 165 CP(*from, *to, mackeylen); 166 PTRIN_CP(*from, *to, mackey); 167 CP(*from, *to, ses); 168 to->crid = CRYPTOCAP_F_HARDWARE; 169 } 170 171 static void 172 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to) 173 { 174 175 session_op_from_32((const struct session_op32 *)from, to); 176 CP(*from, *to, crid); 177 CP(*from, *to, ivlen); 178 CP(*from, *to, maclen); 179 } 180 181 static void 182 session_op_to_32(const struct session2_op *from, struct session_op32 *to) 183 { 184 185 CP(*from, *to, cipher); 186 CP(*from, *to, mac); 187 CP(*from, *to, keylen); 188 PTROUT_CP(*from, *to, key); 189 CP(*from, *to, mackeylen); 190 PTROUT_CP(*from, *to, mackey); 191 CP(*from, *to, ses); 192 } 193 194 static void 195 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to) 196 { 197 198 session_op_to_32(from, (struct session_op32 *)to); 199 CP(*from, *to, crid); 200 } 201 202 static void 203 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to) 204 { 205 206 CP(*from, *to, ses); 207 CP(*from, *to, op); 208 CP(*from, *to, flags); 209 CP(*from, *to, len); 210 PTRIN_CP(*from, *to, src); 211 PTRIN_CP(*from, *to, dst); 212 PTRIN_CP(*from, *to, mac); 213 PTRIN_CP(*from, *to, iv); 214 } 215 216 static void 217 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to) 218 { 219 220 CP(*from, *to, ses); 221 CP(*from, *to, op); 222 CP(*from, *to, flags); 223 CP(*from, *to, len); 224 PTROUT_CP(*from, *to, src); 225 PTROUT_CP(*from, *to, dst); 226 PTROUT_CP(*from, *to, mac); 227 PTROUT_CP(*from, *to, iv); 228 } 229 230 static void 231 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to) 232 { 233 234 CP(*from, *to, ses); 235 CP(*from, *to, op); 236 CP(*from, *to, flags); 237 CP(*from, *to, len); 238 CP(*from, *to, aadlen); 239 CP(*from, *to, ivlen); 240 PTRIN_CP(*from, *to, src); 241 PTRIN_CP(*from, *to, dst); 242 PTRIN_CP(*from, *to, aad); 243 PTRIN_CP(*from, *to, tag); 244 PTRIN_CP(*from, *to, iv); 245 } 246 247 static void 248 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to) 249 { 250 251 CP(*from, *to, ses); 252 CP(*from, *to, op); 253 CP(*from, *to, flags); 254 CP(*from, *to, len); 255 CP(*from, *to, aadlen); 256 CP(*from, *to, ivlen); 257 PTROUT_CP(*from, *to, src); 258 PTROUT_CP(*from, *to, dst); 259 PTROUT_CP(*from, *to, aad); 260 PTROUT_CP(*from, *to, tag); 261 PTROUT_CP(*from, *to, iv); 262 } 263 264 static void 265 crparam_from_32(const struct crparam32 *from, struct crparam *to) 266 { 267 268 PTRIN_CP(*from, *to, crp_p); 269 CP(*from, *to, crp_nbits); 270 } 271 272 static void 273 crparam_to_32(const struct crparam *from, struct crparam32 *to) 274 { 275 276 PTROUT_CP(*from, *to, crp_p); 277 CP(*from, *to, crp_nbits); 278 } 279 280 static void 281 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to) 282 { 283 int i; 284 285 CP(*from, *to, crk_op); 286 CP(*from, *to, crk_status); 287 CP(*from, *to, crk_iparams); 288 CP(*from, *to, crk_oparams); 289 CP(*from, *to, crk_crid); 290 for (i = 0; i < CRK_MAXPARAM; i++) 291 crparam_from_32(&from->crk_param[i], &to->crk_param[i]); 292 } 293 294 static void 295 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to) 296 { 297 int i; 298 299 CP(*from, *to, crk_op); 300 CP(*from, *to, crk_status); 301 CP(*from, *to, crk_iparams); 302 CP(*from, *to, crk_oparams); 303 CP(*from, *to, crk_crid); 304 for (i = 0; i < CRK_MAXPARAM; i++) 305 crparam_to_32(&from->crk_param[i], &to->crk_param[i]); 306 } 307 #endif 308 309 static void 310 session2_op_from_op(const struct session_op *from, struct session2_op *to) 311 { 312 313 memset(to, 0, sizeof(*to)); 314 memcpy(to, from, sizeof(*from)); 315 to->crid = CRYPTOCAP_F_HARDWARE; 316 } 317 318 static void 319 session2_op_to_op(const struct session2_op *from, struct session_op *to) 320 { 321 322 memcpy(to, from, sizeof(*to)); 323 } 324 325 struct csession { 326 TAILQ_ENTRY(csession) next; 327 crypto_session_t cses; 328 volatile u_int refs; 329 uint32_t ses; 330 struct mtx lock; /* for op submission */ 331 332 struct enc_xform *txform; 333 int hashsize; 334 int ivsize; 335 336 void *key; 337 void *mackey; 338 }; 339 340 struct cryptop_data { 341 struct csession *cse; 342 343 char *buf; 344 char *obuf; 345 char *aad; 346 bool done; 347 }; 348 349 struct fcrypt { 350 TAILQ_HEAD(csessionlist, csession) csessions; 351 int sesn; 352 struct mtx lock; 353 }; 354 355 static bool use_outputbuffers; 356 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW, 357 &use_outputbuffers, 0, 358 "Use separate output buffers for /dev/crypto requests."); 359 360 static bool use_separate_aad; 361 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW, 362 &use_separate_aad, 0, 363 "Use separate AAD buffer for /dev/crypto requests."); 364 365 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 }; 366 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW, 367 &warninterval, 368 "Delay in seconds between warnings of deprecated /dev/crypto algorithms"); 369 370 /* 371 * Check a crypto identifier to see if it requested 372 * a software device/driver. This can be done either 373 * by device name/class or through search constraints. 374 */ 375 static int 376 checkforsoftware(int *cridp) 377 { 378 int crid; 379 380 crid = *cridp; 381 382 if (!crypto_devallowsoft) { 383 if (crid & CRYPTOCAP_F_SOFTWARE) { 384 if (crid & CRYPTOCAP_F_HARDWARE) { 385 *cridp = CRYPTOCAP_F_HARDWARE; 386 return 0; 387 } 388 return EINVAL; 389 } 390 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 && 391 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0) 392 return EINVAL; 393 } 394 return 0; 395 } 396 397 static int 398 cse_create(struct fcrypt *fcr, struct session2_op *sop) 399 { 400 struct crypto_session_params csp; 401 struct csession *cse; 402 struct enc_xform *txform; 403 struct auth_hash *thash; 404 void *key = NULL; 405 void *mackey = NULL; 406 crypto_session_t cses; 407 int crid, error; 408 409 switch (sop->cipher) { 410 case 0: 411 txform = NULL; 412 break; 413 case CRYPTO_AES_CBC: 414 txform = &enc_xform_rijndael128; 415 break; 416 case CRYPTO_AES_XTS: 417 txform = &enc_xform_aes_xts; 418 break; 419 case CRYPTO_NULL_CBC: 420 txform = &enc_xform_null; 421 break; 422 case CRYPTO_CAMELLIA_CBC: 423 txform = &enc_xform_camellia; 424 break; 425 case CRYPTO_AES_ICM: 426 txform = &enc_xform_aes_icm; 427 break; 428 case CRYPTO_AES_NIST_GCM_16: 429 txform = &enc_xform_aes_nist_gcm; 430 break; 431 case CRYPTO_CHACHA20: 432 txform = &enc_xform_chacha20; 433 break; 434 case CRYPTO_AES_CCM_16: 435 txform = &enc_xform_ccm; 436 break; 437 case CRYPTO_CHACHA20_POLY1305: 438 txform = &enc_xform_chacha20_poly1305; 439 break; 440 default: 441 CRYPTDEB("invalid cipher"); 442 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 443 return (EINVAL); 444 } 445 446 switch (sop->mac) { 447 case 0: 448 thash = NULL; 449 break; 450 case CRYPTO_POLY1305: 451 thash = &auth_hash_poly1305; 452 break; 453 case CRYPTO_SHA1_HMAC: 454 thash = &auth_hash_hmac_sha1; 455 break; 456 case CRYPTO_SHA2_224_HMAC: 457 thash = &auth_hash_hmac_sha2_224; 458 break; 459 case CRYPTO_SHA2_256_HMAC: 460 thash = &auth_hash_hmac_sha2_256; 461 break; 462 case CRYPTO_SHA2_384_HMAC: 463 thash = &auth_hash_hmac_sha2_384; 464 break; 465 case CRYPTO_SHA2_512_HMAC: 466 thash = &auth_hash_hmac_sha2_512; 467 break; 468 case CRYPTO_RIPEMD160_HMAC: 469 thash = &auth_hash_hmac_ripemd_160; 470 break; 471 #ifdef COMPAT_FREEBSD12 472 case CRYPTO_AES_128_NIST_GMAC: 473 case CRYPTO_AES_192_NIST_GMAC: 474 case CRYPTO_AES_256_NIST_GMAC: 475 /* Should always be paired with GCM. */ 476 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) { 477 CRYPTDEB("GMAC without GCM"); 478 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 479 return (EINVAL); 480 } 481 break; 482 #endif 483 case CRYPTO_AES_NIST_GMAC: 484 switch (sop->mackeylen * 8) { 485 case 128: 486 thash = &auth_hash_nist_gmac_aes_128; 487 break; 488 case 192: 489 thash = &auth_hash_nist_gmac_aes_192; 490 break; 491 case 256: 492 thash = &auth_hash_nist_gmac_aes_256; 493 break; 494 default: 495 CRYPTDEB("invalid GMAC key length"); 496 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 497 return (EINVAL); 498 } 499 break; 500 case CRYPTO_AES_CCM_CBC_MAC: 501 switch (sop->mackeylen) { 502 case 16: 503 thash = &auth_hash_ccm_cbc_mac_128; 504 break; 505 case 24: 506 thash = &auth_hash_ccm_cbc_mac_192; 507 break; 508 case 32: 509 thash = &auth_hash_ccm_cbc_mac_256; 510 break; 511 default: 512 CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen); 513 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 514 return (EINVAL); 515 } 516 break; 517 case CRYPTO_SHA1: 518 thash = &auth_hash_sha1; 519 break; 520 case CRYPTO_SHA2_224: 521 thash = &auth_hash_sha2_224; 522 break; 523 case CRYPTO_SHA2_256: 524 thash = &auth_hash_sha2_256; 525 break; 526 case CRYPTO_SHA2_384: 527 thash = &auth_hash_sha2_384; 528 break; 529 case CRYPTO_SHA2_512: 530 thash = &auth_hash_sha2_512; 531 break; 532 533 case CRYPTO_NULL_HMAC: 534 thash = &auth_hash_null; 535 break; 536 537 case CRYPTO_BLAKE2B: 538 thash = &auth_hash_blake2b; 539 break; 540 case CRYPTO_BLAKE2S: 541 thash = &auth_hash_blake2s; 542 break; 543 544 default: 545 CRYPTDEB("invalid mac"); 546 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 547 return (EINVAL); 548 } 549 550 if (txform == NULL && thash == NULL) { 551 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 552 return (EINVAL); 553 } 554 555 memset(&csp, 0, sizeof(csp)); 556 if (use_outputbuffers) 557 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT; 558 559 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) { 560 switch (sop->mac) { 561 #ifdef COMPAT_FREEBSD12 562 case CRYPTO_AES_128_NIST_GMAC: 563 case CRYPTO_AES_192_NIST_GMAC: 564 case CRYPTO_AES_256_NIST_GMAC: 565 if (sop->keylen != sop->mackeylen) { 566 SDT_PROBE1(opencrypto, dev, ioctl, error, 567 __LINE__); 568 return (EINVAL); 569 } 570 break; 571 #endif 572 case 0: 573 break; 574 default: 575 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 576 return (EINVAL); 577 } 578 csp.csp_mode = CSP_MODE_AEAD; 579 } else if (sop->cipher == CRYPTO_AES_CCM_16) { 580 switch (sop->mac) { 581 #ifdef COMPAT_FREEBSD12 582 case CRYPTO_AES_CCM_CBC_MAC: 583 if (sop->keylen != sop->mackeylen) { 584 SDT_PROBE1(opencrypto, dev, ioctl, error, 585 __LINE__); 586 return (EINVAL); 587 } 588 thash = NULL; 589 break; 590 #endif 591 case 0: 592 break; 593 default: 594 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 595 return (EINVAL); 596 } 597 csp.csp_mode = CSP_MODE_AEAD; 598 } else if (sop->cipher == CRYPTO_CHACHA20_POLY1305) { 599 if (sop->mac != 0) { 600 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 601 return (EINVAL); 602 } 603 csp.csp_mode = CSP_MODE_AEAD; 604 } else if (txform != NULL && thash != NULL) 605 csp.csp_mode = CSP_MODE_ETA; 606 else if (txform != NULL) 607 csp.csp_mode = CSP_MODE_CIPHER; 608 else 609 csp.csp_mode = CSP_MODE_DIGEST; 610 611 switch (csp.csp_mode) { 612 case CSP_MODE_AEAD: 613 case CSP_MODE_ETA: 614 if (use_separate_aad) 615 csp.csp_flags |= CSP_F_SEPARATE_AAD; 616 break; 617 } 618 619 if (txform != NULL) { 620 csp.csp_cipher_alg = txform->type; 621 csp.csp_cipher_klen = sop->keylen; 622 if (sop->keylen > txform->maxkey || 623 sop->keylen < txform->minkey) { 624 CRYPTDEB("invalid cipher parameters"); 625 error = EINVAL; 626 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 627 goto bail; 628 } 629 630 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK); 631 error = copyin(sop->key, key, csp.csp_cipher_klen); 632 if (error) { 633 CRYPTDEB("invalid key"); 634 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 635 goto bail; 636 } 637 csp.csp_cipher_key = key; 638 csp.csp_ivlen = txform->ivsize; 639 } 640 641 if (thash != NULL) { 642 csp.csp_auth_alg = thash->type; 643 csp.csp_auth_klen = sop->mackeylen; 644 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) { 645 CRYPTDEB("invalid mac key length"); 646 error = EINVAL; 647 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 648 goto bail; 649 } 650 651 if (csp.csp_auth_klen != 0) { 652 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK); 653 error = copyin(sop->mackey, mackey, csp.csp_auth_klen); 654 if (error) { 655 CRYPTDEB("invalid mac key"); 656 SDT_PROBE1(opencrypto, dev, ioctl, error, 657 __LINE__); 658 goto bail; 659 } 660 csp.csp_auth_key = mackey; 661 } 662 663 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC) 664 csp.csp_ivlen = AES_GCM_IV_LEN; 665 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC) 666 csp.csp_ivlen = AES_CCM_IV_LEN; 667 } 668 669 if (sop->ivlen != 0) { 670 if (csp.csp_ivlen == 0) { 671 CRYPTDEB("does not support an IV"); 672 error = EINVAL; 673 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 674 goto bail; 675 } 676 csp.csp_ivlen = sop->ivlen; 677 } 678 if (sop->maclen != 0) { 679 if (!(thash != NULL || csp.csp_mode == CSP_MODE_AEAD)) { 680 CRYPTDEB("does not support a MAC"); 681 error = EINVAL; 682 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 683 goto bail; 684 } 685 csp.csp_auth_mlen = sop->maclen; 686 } 687 688 crid = sop->crid; 689 error = checkforsoftware(&crid); 690 if (error) { 691 CRYPTDEB("checkforsoftware"); 692 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 693 goto bail; 694 } 695 error = crypto_newsession(&cses, &csp, crid); 696 if (error) { 697 CRYPTDEB("crypto_newsession"); 698 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 699 goto bail; 700 } 701 702 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO); 703 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF); 704 refcount_init(&cse->refs, 1); 705 cse->key = key; 706 cse->mackey = mackey; 707 cse->cses = cses; 708 cse->txform = txform; 709 if (sop->maclen != 0) 710 cse->hashsize = sop->maclen; 711 else if (thash != NULL) 712 cse->hashsize = thash->hashsize; 713 else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) 714 cse->hashsize = AES_GMAC_HASH_LEN; 715 else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16) 716 cse->hashsize = AES_CBC_MAC_HASH_LEN; 717 else if (csp.csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) 718 cse->hashsize = POLY1305_HASH_LEN; 719 cse->ivsize = csp.csp_ivlen; 720 721 mtx_lock(&fcr->lock); 722 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next); 723 cse->ses = fcr->sesn++; 724 mtx_unlock(&fcr->lock); 725 726 sop->ses = cse->ses; 727 728 /* return hardware/driver id */ 729 sop->crid = crypto_ses2hid(cse->cses); 730 bail: 731 if (error) { 732 free(key, M_XDATA); 733 free(mackey, M_XDATA); 734 } 735 return (error); 736 } 737 738 static struct csession * 739 cse_find(struct fcrypt *fcr, u_int ses) 740 { 741 struct csession *cse; 742 743 mtx_lock(&fcr->lock); 744 TAILQ_FOREACH(cse, &fcr->csessions, next) { 745 if (cse->ses == ses) { 746 refcount_acquire(&cse->refs); 747 mtx_unlock(&fcr->lock); 748 return (cse); 749 } 750 } 751 mtx_unlock(&fcr->lock); 752 return (NULL); 753 } 754 755 static void 756 cse_free(struct csession *cse) 757 { 758 759 if (!refcount_release(&cse->refs)) 760 return; 761 crypto_freesession(cse->cses); 762 mtx_destroy(&cse->lock); 763 if (cse->key) 764 free(cse->key, M_XDATA); 765 if (cse->mackey) 766 free(cse->mackey, M_XDATA); 767 free(cse, M_XDATA); 768 } 769 770 static bool 771 cse_delete(struct fcrypt *fcr, u_int ses) 772 { 773 struct csession *cse; 774 775 mtx_lock(&fcr->lock); 776 TAILQ_FOREACH(cse, &fcr->csessions, next) { 777 if (cse->ses == ses) { 778 TAILQ_REMOVE(&fcr->csessions, cse, next); 779 mtx_unlock(&fcr->lock); 780 cse_free(cse); 781 return (true); 782 } 783 } 784 mtx_unlock(&fcr->lock); 785 return (false); 786 } 787 788 static struct cryptop_data * 789 cod_alloc(struct csession *cse, size_t aad_len, size_t len) 790 { 791 struct cryptop_data *cod; 792 793 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO); 794 795 cod->cse = cse; 796 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) { 797 if (aad_len != 0) 798 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK); 799 cod->buf = malloc(len, M_XDATA, M_WAITOK); 800 } else 801 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK); 802 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT) 803 cod->obuf = malloc(len, M_XDATA, M_WAITOK); 804 return (cod); 805 } 806 807 static void 808 cod_free(struct cryptop_data *cod) 809 { 810 811 free(cod->aad, M_XDATA); 812 free(cod->obuf, M_XDATA); 813 free(cod->buf, M_XDATA); 814 free(cod, M_XDATA); 815 } 816 817 static int 818 cryptodev_cb(struct cryptop *crp) 819 { 820 struct cryptop_data *cod = crp->crp_opaque; 821 822 /* 823 * Lock to ensure the wakeup() is not missed by the loops 824 * waiting on cod->done in cryptodev_op() and 825 * cryptodev_aead(). 826 */ 827 mtx_lock(&cod->cse->lock); 828 cod->done = true; 829 mtx_unlock(&cod->cse->lock); 830 wakeup(cod); 831 return (0); 832 } 833 834 static int 835 cryptodev_op(struct csession *cse, const struct crypt_op *cop) 836 { 837 const struct crypto_session_params *csp; 838 struct cryptop_data *cod = NULL; 839 struct cryptop *crp = NULL; 840 char *dst; 841 int error; 842 843 if (cop->len > 256*1024-4) { 844 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 845 return (E2BIG); 846 } 847 848 if (cse->txform) { 849 if ((cop->len % cse->txform->blocksize) != 0) { 850 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 851 return (EINVAL); 852 } 853 } 854 855 if (cop->mac && cse->hashsize == 0) { 856 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 857 return (EINVAL); 858 } 859 860 /* 861 * The COP_F_CIPHER_FIRST flag predates explicit session 862 * modes, but the only way it was used was for EtA so allow it 863 * as long as it is consistent with EtA. 864 */ 865 if (cop->flags & COP_F_CIPHER_FIRST) { 866 if (cop->op != COP_ENCRYPT) { 867 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 868 return (EINVAL); 869 } 870 } 871 872 cod = cod_alloc(cse, 0, cop->len + cse->hashsize); 873 dst = cop->dst; 874 875 crp = crypto_getreq(cse->cses, M_WAITOK); 876 877 error = copyin(cop->src, cod->buf, cop->len); 878 if (error) { 879 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 880 goto bail; 881 } 882 crp->crp_payload_start = 0; 883 crp->crp_payload_length = cop->len; 884 if (cse->hashsize) 885 crp->crp_digest_start = cop->len; 886 887 csp = crypto_get_params(cse->cses); 888 switch (csp->csp_mode) { 889 case CSP_MODE_COMPRESS: 890 switch (cop->op) { 891 case COP_ENCRYPT: 892 crp->crp_op = CRYPTO_OP_COMPRESS; 893 break; 894 case COP_DECRYPT: 895 crp->crp_op = CRYPTO_OP_DECOMPRESS; 896 break; 897 default: 898 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 899 error = EINVAL; 900 goto bail; 901 } 902 break; 903 case CSP_MODE_CIPHER: 904 if (cop->len == 0 || 905 (cop->iv == NULL && cop->len == cse->ivsize)) { 906 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 907 error = EINVAL; 908 goto bail; 909 } 910 switch (cop->op) { 911 case COP_ENCRYPT: 912 crp->crp_op = CRYPTO_OP_ENCRYPT; 913 break; 914 case COP_DECRYPT: 915 crp->crp_op = CRYPTO_OP_DECRYPT; 916 break; 917 default: 918 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 919 error = EINVAL; 920 goto bail; 921 } 922 break; 923 case CSP_MODE_DIGEST: 924 switch (cop->op) { 925 case 0: 926 case COP_ENCRYPT: 927 case COP_DECRYPT: 928 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST; 929 if (cod->obuf != NULL) 930 crp->crp_digest_start = 0; 931 break; 932 default: 933 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 934 error = EINVAL; 935 goto bail; 936 } 937 break; 938 case CSP_MODE_AEAD: 939 if (cse->ivsize != 0 && cop->iv == NULL) { 940 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 941 error = EINVAL; 942 goto bail; 943 } 944 /* FALLTHROUGH */ 945 case CSP_MODE_ETA: 946 switch (cop->op) { 947 case COP_ENCRYPT: 948 crp->crp_op = CRYPTO_OP_ENCRYPT | 949 CRYPTO_OP_COMPUTE_DIGEST; 950 break; 951 case COP_DECRYPT: 952 crp->crp_op = CRYPTO_OP_DECRYPT | 953 CRYPTO_OP_VERIFY_DIGEST; 954 break; 955 default: 956 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 957 error = EINVAL; 958 goto bail; 959 } 960 break; 961 default: 962 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 963 error = EINVAL; 964 goto bail; 965 } 966 967 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH); 968 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize); 969 if (cod->obuf) 970 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize); 971 crp->crp_callback = cryptodev_cb; 972 crp->crp_opaque = cod; 973 974 if (cop->iv) { 975 if (cse->ivsize == 0) { 976 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 977 error = EINVAL; 978 goto bail; 979 } 980 error = copyin(cop->iv, crp->crp_iv, cse->ivsize); 981 if (error) { 982 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 983 goto bail; 984 } 985 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 986 } else if (cse->ivsize != 0) { 987 if (crp->crp_payload_length < cse->ivsize) { 988 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 989 error = EINVAL; 990 goto bail; 991 } 992 crp->crp_iv_start = 0; 993 crp->crp_payload_length -= cse->ivsize; 994 if (crp->crp_payload_length != 0) 995 crp->crp_payload_start = cse->ivsize; 996 dst += cse->ivsize; 997 } 998 999 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1000 error = copyin(cop->mac, cod->buf + crp->crp_digest_start, 1001 cse->hashsize); 1002 if (error) { 1003 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1004 goto bail; 1005 } 1006 } 1007 again: 1008 /* 1009 * Let the dispatch run unlocked, then, interlock against the 1010 * callback before checking if the operation completed and going 1011 * to sleep. This insures drivers don't inherit our lock which 1012 * results in a lock order reversal between crypto_dispatch forced 1013 * entry and the crypto_done callback into us. 1014 */ 1015 error = crypto_dispatch(crp); 1016 if (error != 0) { 1017 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1018 goto bail; 1019 } 1020 1021 mtx_lock(&cse->lock); 1022 while (!cod->done) 1023 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1024 mtx_unlock(&cse->lock); 1025 1026 if (crp->crp_etype == EAGAIN) { 1027 crp->crp_etype = 0; 1028 crp->crp_flags &= ~CRYPTO_F_DONE; 1029 cod->done = false; 1030 goto again; 1031 } 1032 1033 if (crp->crp_etype != 0) { 1034 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1035 error = crp->crp_etype; 1036 goto bail; 1037 } 1038 1039 if (cop->dst != NULL) { 1040 error = copyout(cod->obuf != NULL ? cod->obuf : 1041 cod->buf + crp->crp_payload_start, dst, 1042 crp->crp_payload_length); 1043 if (error) { 1044 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1045 goto bail; 1046 } 1047 } 1048 1049 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1050 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1051 crp->crp_digest_start, cop->mac, cse->hashsize); 1052 if (error) { 1053 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1054 goto bail; 1055 } 1056 } 1057 1058 bail: 1059 crypto_freereq(crp); 1060 cod_free(cod); 1061 1062 return (error); 1063 } 1064 1065 static int 1066 cryptodev_aead(struct csession *cse, struct crypt_aead *caead) 1067 { 1068 const struct crypto_session_params *csp; 1069 struct cryptop_data *cod = NULL; 1070 struct cryptop *crp = NULL; 1071 char *dst; 1072 int error; 1073 1074 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) { 1075 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1076 return (E2BIG); 1077 } 1078 1079 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL || 1080 (caead->len % cse->txform->blocksize) != 0) { 1081 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1082 return (EINVAL); 1083 } 1084 1085 /* 1086 * The COP_F_CIPHER_FIRST flag predates explicit session 1087 * modes, but the only way it was used was for EtA so allow it 1088 * as long as it is consistent with EtA. 1089 */ 1090 if (caead->flags & COP_F_CIPHER_FIRST) { 1091 if (caead->op != COP_ENCRYPT) { 1092 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1093 return (EINVAL); 1094 } 1095 } 1096 1097 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize); 1098 dst = caead->dst; 1099 1100 crp = crypto_getreq(cse->cses, M_WAITOK); 1101 1102 if (cod->aad != NULL) 1103 error = copyin(caead->aad, cod->aad, caead->aadlen); 1104 else 1105 error = copyin(caead->aad, cod->buf, caead->aadlen); 1106 if (error) { 1107 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1108 goto bail; 1109 } 1110 crp->crp_aad = cod->aad; 1111 crp->crp_aad_start = 0; 1112 crp->crp_aad_length = caead->aadlen; 1113 1114 if (cod->aad != NULL) 1115 crp->crp_payload_start = 0; 1116 else 1117 crp->crp_payload_start = caead->aadlen; 1118 error = copyin(caead->src, cod->buf + crp->crp_payload_start, 1119 caead->len); 1120 if (error) { 1121 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1122 goto bail; 1123 } 1124 crp->crp_payload_length = caead->len; 1125 if (caead->op == COP_ENCRYPT && cod->obuf != NULL) 1126 crp->crp_digest_start = crp->crp_payload_output_start + 1127 caead->len; 1128 else 1129 crp->crp_digest_start = crp->crp_payload_start + caead->len; 1130 1131 csp = crypto_get_params(cse->cses); 1132 switch (csp->csp_mode) { 1133 case CSP_MODE_AEAD: 1134 case CSP_MODE_ETA: 1135 switch (caead->op) { 1136 case COP_ENCRYPT: 1137 crp->crp_op = CRYPTO_OP_ENCRYPT | 1138 CRYPTO_OP_COMPUTE_DIGEST; 1139 break; 1140 case COP_DECRYPT: 1141 crp->crp_op = CRYPTO_OP_DECRYPT | 1142 CRYPTO_OP_VERIFY_DIGEST; 1143 break; 1144 default: 1145 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1146 error = EINVAL; 1147 goto bail; 1148 } 1149 break; 1150 default: 1151 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1152 error = EINVAL; 1153 goto bail; 1154 } 1155 1156 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH); 1157 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len + 1158 cse->hashsize); 1159 if (cod->obuf != NULL) 1160 crypto_use_output_buf(crp, cod->obuf, caead->len + 1161 cse->hashsize); 1162 crp->crp_callback = cryptodev_cb; 1163 crp->crp_opaque = cod; 1164 1165 if (caead->iv) { 1166 /* 1167 * Permit a 16-byte IV for AES-XTS, but only use the 1168 * first 8 bytes as a block number. 1169 */ 1170 if (csp->csp_mode == CSP_MODE_ETA && 1171 csp->csp_cipher_alg == CRYPTO_AES_XTS && 1172 caead->ivlen == AES_BLOCK_LEN) 1173 caead->ivlen = AES_XTS_IV_LEN; 1174 1175 if (cse->ivsize == 0) { 1176 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1177 error = EINVAL; 1178 goto bail; 1179 } 1180 if (caead->ivlen != cse->ivsize) { 1181 error = EINVAL; 1182 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1183 goto bail; 1184 } 1185 1186 error = copyin(caead->iv, crp->crp_iv, cse->ivsize); 1187 if (error) { 1188 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1189 goto bail; 1190 } 1191 crp->crp_flags |= CRYPTO_F_IV_SEPARATE; 1192 } else { 1193 error = EINVAL; 1194 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1195 goto bail; 1196 } 1197 1198 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 1199 error = copyin(caead->tag, cod->buf + crp->crp_digest_start, 1200 cse->hashsize); 1201 if (error) { 1202 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1203 goto bail; 1204 } 1205 } 1206 again: 1207 /* 1208 * Let the dispatch run unlocked, then, interlock against the 1209 * callback before checking if the operation completed and going 1210 * to sleep. This insures drivers don't inherit our lock which 1211 * results in a lock order reversal between crypto_dispatch forced 1212 * entry and the crypto_done callback into us. 1213 */ 1214 error = crypto_dispatch(crp); 1215 if (error != 0) { 1216 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1217 goto bail; 1218 } 1219 1220 mtx_lock(&cse->lock); 1221 while (!cod->done) 1222 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0); 1223 mtx_unlock(&cse->lock); 1224 1225 if (crp->crp_etype == EAGAIN) { 1226 crp->crp_etype = 0; 1227 crp->crp_flags &= ~CRYPTO_F_DONE; 1228 cod->done = false; 1229 goto again; 1230 } 1231 1232 if (crp->crp_etype != 0) { 1233 error = crp->crp_etype; 1234 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1235 goto bail; 1236 } 1237 1238 if (caead->dst != NULL) { 1239 error = copyout(cod->obuf != NULL ? cod->obuf : 1240 cod->buf + crp->crp_payload_start, dst, 1241 crp->crp_payload_length); 1242 if (error) { 1243 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1244 goto bail; 1245 } 1246 } 1247 1248 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) { 1249 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) + 1250 crp->crp_digest_start, caead->tag, cse->hashsize); 1251 if (error) { 1252 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1253 goto bail; 1254 } 1255 } 1256 1257 bail: 1258 crypto_freereq(crp); 1259 cod_free(cod); 1260 1261 return (error); 1262 } 1263 1264 static void 1265 cryptodevkey_cb(struct cryptkop *krp) 1266 { 1267 1268 wakeup_one(krp); 1269 } 1270 1271 static int 1272 cryptodev_key(struct crypt_kop *kop) 1273 { 1274 struct cryptkop *krp = NULL; 1275 int error = EINVAL; 1276 int in, out, size, i; 1277 1278 if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) { 1279 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1280 return (EFBIG); 1281 } 1282 1283 in = kop->crk_iparams; 1284 out = kop->crk_oparams; 1285 switch (kop->crk_op) { 1286 case CRK_MOD_EXP: 1287 if (in == 3 && out == 1) 1288 break; 1289 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1290 return (EINVAL); 1291 case CRK_MOD_EXP_CRT: 1292 if (in == 6 && out == 1) 1293 break; 1294 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1295 return (EINVAL); 1296 case CRK_DSA_SIGN: 1297 if (in == 5 && out == 2) 1298 break; 1299 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1300 return (EINVAL); 1301 case CRK_DSA_VERIFY: 1302 if (in == 7 && out == 0) 1303 break; 1304 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1305 return (EINVAL); 1306 case CRK_DH_COMPUTE_KEY: 1307 if (in == 3 && out == 1) 1308 break; 1309 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1310 return (EINVAL); 1311 default: 1312 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1313 return (EINVAL); 1314 } 1315 1316 krp = malloc(sizeof(*krp), M_XDATA, M_WAITOK | M_ZERO); 1317 krp->krp_op = kop->crk_op; 1318 krp->krp_status = kop->crk_status; 1319 krp->krp_iparams = kop->crk_iparams; 1320 krp->krp_oparams = kop->crk_oparams; 1321 krp->krp_crid = kop->crk_crid; 1322 krp->krp_status = 0; 1323 krp->krp_callback = cryptodevkey_cb; 1324 1325 for (i = 0; i < CRK_MAXPARAM; i++) { 1326 if (kop->crk_param[i].crp_nbits > 65536) { 1327 /* Limit is the same as in OpenBSD */ 1328 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1329 goto fail; 1330 } 1331 krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits; 1332 } 1333 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) { 1334 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1335 if (size == 0) 1336 continue; 1337 krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK); 1338 if (i >= krp->krp_iparams) 1339 continue; 1340 error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size); 1341 if (error) { 1342 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1343 goto fail; 1344 } 1345 } 1346 1347 error = crypto_kdispatch(krp); 1348 if (error) { 1349 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1350 goto fail; 1351 } 1352 error = tsleep(krp, PSOCK, "crydev", 0); 1353 if (error) { 1354 /* XXX can this happen? if so, how do we recover? */ 1355 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1356 goto fail; 1357 } 1358 1359 kop->crk_crid = krp->krp_hid; /* device that did the work */ 1360 if (krp->krp_status != 0) { 1361 error = krp->krp_status; 1362 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1363 goto fail; 1364 } 1365 1366 for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) { 1367 size = (krp->krp_param[i].crp_nbits + 7) / 8; 1368 if (size == 0) 1369 continue; 1370 error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size); 1371 if (error) { 1372 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1373 goto fail; 1374 } 1375 } 1376 1377 fail: 1378 if (krp) { 1379 kop->crk_status = krp->krp_status; 1380 for (i = 0; i < CRK_MAXPARAM; i++) { 1381 if (krp->krp_param[i].crp_p) 1382 free(krp->krp_param[i].crp_p, M_XDATA); 1383 } 1384 free(krp, M_XDATA); 1385 } 1386 return (error); 1387 } 1388 1389 static int 1390 cryptodev_find(struct crypt_find_op *find) 1391 { 1392 device_t dev; 1393 size_t fnlen = sizeof find->name; 1394 1395 if (find->crid != -1) { 1396 dev = crypto_find_device_byhid(find->crid); 1397 if (dev == NULL) 1398 return (ENOENT); 1399 strncpy(find->name, device_get_nameunit(dev), fnlen); 1400 find->name[fnlen - 1] = '\x0'; 1401 } else { 1402 find->name[fnlen - 1] = '\x0'; 1403 find->crid = crypto_find_driver(find->name); 1404 if (find->crid == -1) 1405 return (ENOENT); 1406 } 1407 return (0); 1408 } 1409 1410 static void 1411 fcrypt_dtor(void *data) 1412 { 1413 struct fcrypt *fcr = data; 1414 struct csession *cse; 1415 1416 while ((cse = TAILQ_FIRST(&fcr->csessions))) { 1417 TAILQ_REMOVE(&fcr->csessions, cse, next); 1418 KASSERT(refcount_load(&cse->refs) == 1, 1419 ("%s: crypto session %p with %d refs", __func__, cse, 1420 refcount_load(&cse->refs))); 1421 cse_free(cse); 1422 } 1423 mtx_destroy(&fcr->lock); 1424 free(fcr, M_XDATA); 1425 } 1426 1427 static int 1428 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 1429 { 1430 struct fcrypt *fcr; 1431 int error; 1432 1433 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO); 1434 TAILQ_INIT(&fcr->csessions); 1435 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF); 1436 error = devfs_set_cdevpriv(fcr, fcrypt_dtor); 1437 if (error) 1438 fcrypt_dtor(fcr); 1439 return (error); 1440 } 1441 1442 static int 1443 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, 1444 struct thread *td) 1445 { 1446 static struct timeval keywarn, featwarn; 1447 struct fcrypt *fcr; 1448 struct csession *cse; 1449 struct session2_op *sop; 1450 struct crypt_op *cop; 1451 struct crypt_aead *caead; 1452 struct crypt_kop *kop; 1453 uint32_t ses; 1454 int error = 0; 1455 union { 1456 struct session2_op sopc; 1457 #ifdef COMPAT_FREEBSD32 1458 struct crypt_op copc; 1459 struct crypt_aead aeadc; 1460 struct crypt_kop kopc; 1461 #endif 1462 } thunk; 1463 #ifdef COMPAT_FREEBSD32 1464 u_long cmd32; 1465 void *data32; 1466 1467 cmd32 = 0; 1468 data32 = NULL; 1469 switch (cmd) { 1470 case CIOCGSESSION32: 1471 cmd32 = cmd; 1472 data32 = data; 1473 cmd = CIOCGSESSION; 1474 data = (void *)&thunk.sopc; 1475 session_op_from_32((struct session_op32 *)data32, &thunk.sopc); 1476 break; 1477 case CIOCGSESSION232: 1478 cmd32 = cmd; 1479 data32 = data; 1480 cmd = CIOCGSESSION2; 1481 data = (void *)&thunk.sopc; 1482 session2_op_from_32((struct session2_op32 *)data32, 1483 &thunk.sopc); 1484 break; 1485 case CIOCCRYPT32: 1486 cmd32 = cmd; 1487 data32 = data; 1488 cmd = CIOCCRYPT; 1489 data = (void *)&thunk.copc; 1490 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc); 1491 break; 1492 case CIOCCRYPTAEAD32: 1493 cmd32 = cmd; 1494 data32 = data; 1495 cmd = CIOCCRYPTAEAD; 1496 data = (void *)&thunk.aeadc; 1497 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc); 1498 break; 1499 case CIOCKEY32: 1500 case CIOCKEY232: 1501 cmd32 = cmd; 1502 data32 = data; 1503 if (cmd == CIOCKEY32) 1504 cmd = CIOCKEY; 1505 else 1506 cmd = CIOCKEY2; 1507 data = (void *)&thunk.kopc; 1508 crypt_kop_from_32((struct crypt_kop32 *)data32, &thunk.kopc); 1509 break; 1510 } 1511 #endif 1512 1513 devfs_get_cdevpriv((void **)&fcr); 1514 1515 switch (cmd) { 1516 #ifdef COMPAT_FREEBSD12 1517 case CRIOGET: 1518 /* 1519 * NB: This may fail in cases that the old 1520 * implementation did not if the current process has 1521 * restricted filesystem access (e.g. running in a 1522 * jail that does not expose /dev/crypto or in 1523 * capability mode). 1524 */ 1525 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE, 1526 O_RDWR, 0); 1527 if (error == 0) 1528 *(uint32_t *)data = td->td_retval[0]; 1529 break; 1530 #endif 1531 case CIOCGSESSION: 1532 case CIOCGSESSION2: 1533 if (cmd == CIOCGSESSION) { 1534 session2_op_from_op((void *)data, &thunk.sopc); 1535 sop = &thunk.sopc; 1536 } else 1537 sop = (struct session2_op *)data; 1538 1539 error = cse_create(fcr, sop); 1540 if (cmd == CIOCGSESSION && error == 0) 1541 session2_op_to_op(sop, (void *)data); 1542 break; 1543 case CIOCFSESSION: 1544 ses = *(uint32_t *)data; 1545 if (!cse_delete(fcr, ses)) { 1546 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1547 return (EINVAL); 1548 } 1549 break; 1550 case CIOCCRYPT: 1551 cop = (struct crypt_op *)data; 1552 cse = cse_find(fcr, cop->ses); 1553 if (cse == NULL) { 1554 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1555 return (EINVAL); 1556 } 1557 error = cryptodev_op(cse, cop); 1558 cse_free(cse); 1559 break; 1560 case CIOCKEY: 1561 case CIOCKEY2: 1562 if (ratecheck(&keywarn, &warninterval)) 1563 gone_in(14, 1564 "Asymmetric crypto operations via /dev/crypto"); 1565 1566 if (!crypto_userasymcrypto) { 1567 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1568 return (EPERM); /* XXX compat? */ 1569 } 1570 kop = (struct crypt_kop *)data; 1571 if (cmd == CIOCKEY) { 1572 /* NB: crypto core enforces s/w driver use */ 1573 kop->crk_crid = 1574 CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE; 1575 } 1576 mtx_lock(&Giant); 1577 error = cryptodev_key(kop); 1578 mtx_unlock(&Giant); 1579 break; 1580 case CIOCASYMFEAT: 1581 if (ratecheck(&featwarn, &warninterval)) 1582 gone_in(14, 1583 "Asymmetric crypto features via /dev/crypto"); 1584 1585 if (!crypto_userasymcrypto) { 1586 /* 1587 * NB: if user asym crypto operations are 1588 * not permitted return "no algorithms" 1589 * so well-behaved applications will just 1590 * fallback to doing them in software. 1591 */ 1592 *(int *)data = 0; 1593 } else { 1594 error = crypto_getfeat((int *)data); 1595 if (error) 1596 SDT_PROBE1(opencrypto, dev, ioctl, error, 1597 __LINE__); 1598 } 1599 break; 1600 case CIOCFINDDEV: 1601 error = cryptodev_find((struct crypt_find_op *)data); 1602 break; 1603 case CIOCCRYPTAEAD: 1604 caead = (struct crypt_aead *)data; 1605 cse = cse_find(fcr, caead->ses); 1606 if (cse == NULL) { 1607 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1608 return (EINVAL); 1609 } 1610 error = cryptodev_aead(cse, caead); 1611 cse_free(cse); 1612 break; 1613 default: 1614 error = EINVAL; 1615 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__); 1616 break; 1617 } 1618 1619 #ifdef COMPAT_FREEBSD32 1620 switch (cmd32) { 1621 case CIOCGSESSION32: 1622 if (error == 0) 1623 session_op_to_32((void *)data, data32); 1624 break; 1625 case CIOCGSESSION232: 1626 if (error == 0) 1627 session2_op_to_32((void *)data, data32); 1628 break; 1629 case CIOCCRYPT32: 1630 if (error == 0) 1631 crypt_op_to_32((void *)data, data32); 1632 break; 1633 case CIOCCRYPTAEAD32: 1634 if (error == 0) 1635 crypt_aead_to_32((void *)data, data32); 1636 break; 1637 case CIOCKEY32: 1638 case CIOCKEY232: 1639 crypt_kop_to_32((void *)data, data32); 1640 break; 1641 } 1642 #endif 1643 return (error); 1644 } 1645 1646 static struct cdevsw crypto_cdevsw = { 1647 .d_version = D_VERSION, 1648 .d_open = crypto_open, 1649 .d_ioctl = crypto_ioctl, 1650 .d_name = "crypto", 1651 }; 1652 static struct cdev *crypto_dev; 1653 1654 /* 1655 * Initialization code, both for static and dynamic loading. 1656 */ 1657 static int 1658 cryptodev_modevent(module_t mod, int type, void *unused) 1659 { 1660 switch (type) { 1661 case MOD_LOAD: 1662 if (bootverbose) 1663 printf("crypto: <crypto device>\n"); 1664 crypto_dev = make_dev(&crypto_cdevsw, 0, 1665 UID_ROOT, GID_WHEEL, 0666, 1666 "crypto"); 1667 return 0; 1668 case MOD_UNLOAD: 1669 /*XXX disallow if active sessions */ 1670 destroy_dev(crypto_dev); 1671 return 0; 1672 } 1673 return EINVAL; 1674 } 1675 1676 static moduledata_t cryptodev_mod = { 1677 "cryptodev", 1678 cryptodev_modevent, 1679 0 1680 }; 1681 MODULE_VERSION(cryptodev, 1); 1682 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 1683 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1); 1684 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1); 1685