1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/lock.h> 48 #include <sys/rwlock.h> 49 #include <sys/endian.h> 50 #include <sys/limits.h> 51 #include <sys/mutex.h> 52 53 #include <crypto/sha1.h> 54 #include <opencrypto/rmd160.h> 55 56 #include <opencrypto/cryptodev.h> 57 #include <opencrypto/xform.h> 58 59 #include <sys/kobj.h> 60 #include <sys/bus.h> 61 #include "cryptodev_if.h" 62 63 struct swcr_auth { 64 void *sw_ictx; 65 void *sw_octx; 66 const struct auth_hash *sw_axf; 67 uint16_t sw_mlen; 68 bool sw_hmac; 69 }; 70 71 struct swcr_encdec { 72 void *sw_ctx; 73 const struct enc_xform *sw_exf; 74 }; 75 76 struct swcr_compdec { 77 const struct comp_algo *sw_cxf; 78 }; 79 80 struct swcr_session { 81 struct mtx swcr_lock; 82 int (*swcr_process)(struct swcr_session *, struct cryptop *); 83 84 struct swcr_auth swcr_auth; 85 struct swcr_encdec swcr_encdec; 86 struct swcr_compdec swcr_compdec; 87 }; 88 89 static int32_t swcr_id; 90 91 static void swcr_freesession(device_t dev, crypto_session_t cses); 92 93 /* Used for CRYPTO_NULL_CBC. */ 94 static int 95 swcr_null(struct swcr_session *ses, struct cryptop *crp) 96 { 97 98 return (0); 99 } 100 101 /* 102 * Apply a symmetric encryption/decryption algorithm. 103 */ 104 static int 105 swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 106 { 107 unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 108 unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 109 const struct crypto_session_params *csp; 110 const struct enc_xform *exf; 111 const struct swcr_encdec *sw; 112 void *ctx; 113 size_t inlen, outlen; 114 int i, blks, resid; 115 struct crypto_buffer_cursor cc_in, cc_out; 116 const unsigned char *inblk; 117 unsigned char *outblk; 118 int error; 119 bool encrypting; 120 121 error = 0; 122 123 sw = &ses->swcr_encdec; 124 exf = sw->sw_exf; 125 csp = crypto_get_params(crp->crp_session); 126 127 if (exf->native_blocksize == 0) { 128 /* Check for non-padded data */ 129 if ((crp->crp_payload_length % exf->blocksize) != 0) 130 return (EINVAL); 131 132 blks = exf->blocksize; 133 } else 134 blks = exf->native_blocksize; 135 136 if (exf == &enc_xform_aes_icm && 137 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 138 return (EINVAL); 139 140 ctx = __builtin_alloca(exf->ctxsize); 141 if (crp->crp_cipher_key != NULL) { 142 error = exf->setkey(ctx, crp->crp_cipher_key, 143 csp->csp_cipher_klen); 144 if (error) 145 return (error); 146 } else 147 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 148 149 crypto_read_iv(crp, iv); 150 151 if (exf->reinit) { 152 /* 153 * xforms that provide a reinit method perform all IV 154 * handling themselves. 155 */ 156 exf->reinit(ctx, iv, csp->csp_ivlen); 157 } 158 159 ivp = iv; 160 161 crypto_cursor_init(&cc_in, &crp->crp_buf); 162 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 163 inblk = crypto_cursor_segment(&cc_in, &inlen); 164 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 165 crypto_cursor_init(&cc_out, &crp->crp_obuf); 166 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 167 } else 168 cc_out = cc_in; 169 outblk = crypto_cursor_segment(&cc_out, &outlen); 170 171 resid = crp->crp_payload_length; 172 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 173 174 /* 175 * Loop through encrypting blocks. 'inlen' is the remaining 176 * length of the current segment in the input buffer. 177 * 'outlen' is the remaining length of current segment in the 178 * output buffer. 179 */ 180 while (resid >= blks) { 181 /* 182 * If the current block is not contained within the 183 * current input/output segment, use 'blk' as a local 184 * buffer. 185 */ 186 if (inlen < blks) { 187 crypto_cursor_copydata(&cc_in, blks, blk); 188 inblk = blk; 189 } 190 if (outlen < blks) 191 outblk = blk; 192 193 /* 194 * Ciphers without a 'reinit' hook are assumed to be 195 * used in CBC mode where the chaining is done here. 196 */ 197 if (exf->reinit != NULL) { 198 if (encrypting) 199 exf->encrypt(ctx, inblk, outblk); 200 else 201 exf->decrypt(ctx, inblk, outblk); 202 } else if (encrypting) { 203 /* XOR with previous block */ 204 for (i = 0; i < blks; i++) 205 outblk[i] = inblk[i] ^ ivp[i]; 206 207 exf->encrypt(ctx, outblk, outblk); 208 209 /* 210 * Keep encrypted block for XOR'ing 211 * with next block 212 */ 213 memcpy(iv, outblk, blks); 214 ivp = iv; 215 } else { /* decrypt */ 216 /* 217 * Keep encrypted block for XOR'ing 218 * with next block 219 */ 220 nivp = (ivp == iv) ? iv2 : iv; 221 memcpy(nivp, inblk, blks); 222 223 exf->decrypt(ctx, inblk, outblk); 224 225 /* XOR with previous block */ 226 for (i = 0; i < blks; i++) 227 outblk[i] ^= ivp[i]; 228 229 ivp = nivp; 230 } 231 232 if (inlen < blks) { 233 inblk = crypto_cursor_segment(&cc_in, &inlen); 234 } else { 235 crypto_cursor_advance(&cc_in, blks); 236 inlen -= blks; 237 inblk += blks; 238 } 239 240 if (outlen < blks) { 241 crypto_cursor_copyback(&cc_out, blks, blk); 242 outblk = crypto_cursor_segment(&cc_out, &outlen); 243 } else { 244 crypto_cursor_advance(&cc_out, blks); 245 outlen -= blks; 246 outblk += blks; 247 } 248 249 resid -= blks; 250 } 251 252 /* Handle trailing partial block for stream ciphers. */ 253 if (resid > 0) { 254 KASSERT(exf->native_blocksize != 0, 255 ("%s: partial block of %d bytes for cipher %s", 256 __func__, i, exf->name)); 257 KASSERT(exf->reinit != NULL, 258 ("%s: partial block cipher %s without reinit hook", 259 __func__, exf->name)); 260 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 261 262 inblk = crypto_cursor_segment(&cc_in, &inlen); 263 outblk = crypto_cursor_segment(&cc_out, &outlen); 264 if (inlen < resid) { 265 crypto_cursor_copydata(&cc_in, resid, blk); 266 inblk = blk; 267 } 268 if (outlen < resid) 269 outblk = blk; 270 if (encrypting) 271 exf->encrypt_last(ctx, inblk, outblk, 272 resid); 273 else 274 exf->decrypt_last(ctx, inblk, outblk, 275 resid); 276 if (outlen < resid) 277 crypto_cursor_copyback(&cc_out, resid, blk); 278 } 279 280 explicit_bzero(ctx, exf->ctxsize); 281 explicit_bzero(blk, sizeof(blk)); 282 explicit_bzero(iv, sizeof(iv)); 283 explicit_bzero(iv2, sizeof(iv2)); 284 return (0); 285 } 286 287 /* 288 * Compute or verify hash. 289 */ 290 static int 291 swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 292 { 293 u_char aalg[HASH_MAX_LEN]; 294 const struct crypto_session_params *csp; 295 const struct swcr_auth *sw; 296 const struct auth_hash *axf; 297 union authctx ctx; 298 int err; 299 300 sw = &ses->swcr_auth; 301 302 axf = sw->sw_axf; 303 304 csp = crypto_get_params(crp->crp_session); 305 if (crp->crp_auth_key != NULL) { 306 if (sw->sw_hmac) { 307 hmac_init_ipad(axf, crp->crp_auth_key, 308 csp->csp_auth_klen, &ctx); 309 } else { 310 axf->Init(&ctx); 311 axf->Setkey(&ctx, crp->crp_auth_key, 312 csp->csp_auth_klen); 313 } 314 } else 315 memcpy(&ctx, sw->sw_ictx, axf->ctxsize); 316 317 if (crp->crp_aad != NULL) 318 err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 319 else 320 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 321 axf->Update, &ctx); 322 if (err) 323 goto out; 324 325 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 326 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 327 err = crypto_apply_buf(&crp->crp_obuf, 328 crp->crp_payload_output_start, crp->crp_payload_length, 329 axf->Update, &ctx); 330 else 331 err = crypto_apply(crp, crp->crp_payload_start, 332 crp->crp_payload_length, axf->Update, &ctx); 333 if (err) 334 goto out; 335 336 if (csp->csp_flags & CSP_F_ESN) 337 axf->Update(&ctx, crp->crp_esn, 4); 338 339 axf->Final(aalg, &ctx); 340 if (sw->sw_hmac) { 341 if (crp->crp_auth_key != NULL) 342 hmac_init_opad(axf, crp->crp_auth_key, 343 csp->csp_auth_klen, &ctx); 344 else 345 memcpy(&ctx, sw->sw_octx, axf->ctxsize); 346 axf->Update(&ctx, aalg, axf->hashsize); 347 axf->Final(aalg, &ctx); 348 } 349 350 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 351 u_char uaalg[HASH_MAX_LEN]; 352 353 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 354 if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 355 err = EBADMSG; 356 explicit_bzero(uaalg, sizeof(uaalg)); 357 } else { 358 /* Inject the authentication data */ 359 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 360 } 361 explicit_bzero(aalg, sizeof(aalg)); 362 out: 363 explicit_bzero(&ctx, sizeof(ctx)); 364 return (err); 365 } 366 367 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 368 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 369 370 static int 371 swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 372 { 373 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 374 u_char *blk = (u_char *)blkbuf; 375 u_char tag[GMAC_DIGEST_LEN]; 376 u_char iv[AES_BLOCK_LEN]; 377 struct crypto_buffer_cursor cc; 378 const u_char *inblk; 379 union authctx ctx; 380 const struct swcr_auth *swa; 381 const struct auth_hash *axf; 382 uint32_t *blkp; 383 size_t len; 384 int blksz, error, ivlen, resid; 385 386 swa = &ses->swcr_auth; 387 axf = swa->sw_axf; 388 blksz = GMAC_BLOCK_LEN; 389 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 390 __func__)); 391 392 if (crp->crp_auth_key != NULL) { 393 axf->Init(&ctx); 394 axf->Setkey(&ctx, crp->crp_auth_key, 395 crypto_get_params(crp->crp_session)->csp_auth_klen); 396 } else 397 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 398 399 /* Initialize the IV */ 400 ivlen = AES_GCM_IV_LEN; 401 crypto_read_iv(crp, iv); 402 403 axf->Reinit(&ctx, iv, ivlen); 404 crypto_cursor_init(&cc, &crp->crp_buf); 405 crypto_cursor_advance(&cc, crp->crp_payload_start); 406 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 407 inblk = crypto_cursor_segment(&cc, &len); 408 if (len >= blksz) { 409 len = rounddown(MIN(len, resid), blksz); 410 crypto_cursor_advance(&cc, len); 411 } else { 412 len = blksz; 413 crypto_cursor_copydata(&cc, len, blk); 414 inblk = blk; 415 } 416 axf->Update(&ctx, inblk, len); 417 } 418 if (resid > 0) { 419 memset(blk, 0, blksz); 420 crypto_cursor_copydata(&cc, resid, blk); 421 axf->Update(&ctx, blk, blksz); 422 } 423 424 /* length block */ 425 memset(blk, 0, blksz); 426 blkp = (uint32_t *)blk + 1; 427 *blkp = htobe32(crp->crp_payload_length * 8); 428 axf->Update(&ctx, blk, blksz); 429 430 /* Finalize MAC */ 431 axf->Final(tag, &ctx); 432 433 error = 0; 434 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 435 u_char tag2[GMAC_DIGEST_LEN]; 436 437 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 438 tag2); 439 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 440 error = EBADMSG; 441 explicit_bzero(tag2, sizeof(tag2)); 442 } else { 443 /* Inject the authentication data */ 444 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 445 } 446 explicit_bzero(blkbuf, sizeof(blkbuf)); 447 explicit_bzero(tag, sizeof(tag)); 448 explicit_bzero(iv, sizeof(iv)); 449 return (error); 450 } 451 452 static int 453 swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 454 { 455 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 456 u_char *blk = (u_char *)blkbuf; 457 u_char tag[GMAC_DIGEST_LEN]; 458 struct crypto_buffer_cursor cc_in, cc_out; 459 const u_char *inblk; 460 u_char *outblk; 461 const struct swcr_auth *swa; 462 const struct swcr_encdec *swe; 463 const struct enc_xform *exf; 464 void *ctx; 465 uint32_t *blkp; 466 size_t len; 467 int blksz, error, ivlen, r, resid; 468 469 swa = &ses->swcr_auth; 470 swe = &ses->swcr_encdec; 471 exf = swe->sw_exf; 472 blksz = GMAC_BLOCK_LEN; 473 KASSERT(blksz == exf->native_blocksize, 474 ("%s: blocksize mismatch", __func__)); 475 476 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 477 return (EINVAL); 478 479 ivlen = AES_GCM_IV_LEN; 480 481 ctx = __builtin_alloca(exf->ctxsize); 482 if (crp->crp_cipher_key != NULL) 483 exf->setkey(ctx, crp->crp_cipher_key, 484 crypto_get_params(crp->crp_session)->csp_cipher_klen); 485 else 486 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 487 exf->reinit(ctx, crp->crp_iv, ivlen); 488 489 /* Supply MAC with AAD */ 490 if (crp->crp_aad != NULL) { 491 len = rounddown(crp->crp_aad_length, blksz); 492 if (len != 0) 493 exf->update(ctx, crp->crp_aad, len); 494 if (crp->crp_aad_length != len) { 495 memset(blk, 0, blksz); 496 memcpy(blk, (char *)crp->crp_aad + len, 497 crp->crp_aad_length - len); 498 exf->update(ctx, blk, blksz); 499 } 500 } else { 501 crypto_cursor_init(&cc_in, &crp->crp_buf); 502 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 503 for (resid = crp->crp_aad_length; resid >= blksz; 504 resid -= len) { 505 inblk = crypto_cursor_segment(&cc_in, &len); 506 if (len >= blksz) { 507 len = rounddown(MIN(len, resid), blksz); 508 crypto_cursor_advance(&cc_in, len); 509 } else { 510 len = blksz; 511 crypto_cursor_copydata(&cc_in, len, blk); 512 inblk = blk; 513 } 514 exf->update(ctx, inblk, len); 515 } 516 if (resid > 0) { 517 memset(blk, 0, blksz); 518 crypto_cursor_copydata(&cc_in, resid, blk); 519 exf->update(ctx, blk, blksz); 520 } 521 } 522 523 /* Do encryption with MAC */ 524 crypto_cursor_init(&cc_in, &crp->crp_buf); 525 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 526 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 527 crypto_cursor_init(&cc_out, &crp->crp_obuf); 528 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 529 } else 530 cc_out = cc_in; 531 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 532 inblk = crypto_cursor_segment(&cc_in, &len); 533 if (len < blksz) { 534 crypto_cursor_copydata(&cc_in, blksz, blk); 535 inblk = blk; 536 } else { 537 crypto_cursor_advance(&cc_in, blksz); 538 } 539 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 540 outblk = crypto_cursor_segment(&cc_out, &len); 541 if (len < blksz) 542 outblk = blk; 543 exf->encrypt(ctx, inblk, outblk); 544 exf->update(ctx, outblk, blksz); 545 if (outblk == blk) 546 crypto_cursor_copyback(&cc_out, blksz, blk); 547 else 548 crypto_cursor_advance(&cc_out, blksz); 549 } else { 550 exf->update(ctx, inblk, blksz); 551 } 552 } 553 if (resid > 0) { 554 crypto_cursor_copydata(&cc_in, resid, blk); 555 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 556 exf->encrypt_last(ctx, blk, blk, resid); 557 crypto_cursor_copyback(&cc_out, resid, blk); 558 } 559 exf->update(ctx, blk, resid); 560 } 561 562 /* length block */ 563 memset(blk, 0, blksz); 564 blkp = (uint32_t *)blk + 1; 565 *blkp = htobe32(crp->crp_aad_length * 8); 566 blkp = (uint32_t *)blk + 3; 567 *blkp = htobe32(crp->crp_payload_length * 8); 568 exf->update(ctx, blk, blksz); 569 570 /* Finalize MAC */ 571 exf->final(tag, ctx); 572 573 /* Validate tag */ 574 error = 0; 575 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 576 u_char tag2[GMAC_DIGEST_LEN]; 577 578 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 579 580 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 581 explicit_bzero(tag2, sizeof(tag2)); 582 if (r != 0) { 583 error = EBADMSG; 584 goto out; 585 } 586 587 /* tag matches, decrypt data */ 588 crypto_cursor_init(&cc_in, &crp->crp_buf); 589 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 590 for (resid = crp->crp_payload_length; resid > blksz; 591 resid -= blksz) { 592 inblk = crypto_cursor_segment(&cc_in, &len); 593 if (len < blksz) { 594 crypto_cursor_copydata(&cc_in, blksz, blk); 595 inblk = blk; 596 } else 597 crypto_cursor_advance(&cc_in, blksz); 598 outblk = crypto_cursor_segment(&cc_out, &len); 599 if (len < blksz) 600 outblk = blk; 601 exf->decrypt(ctx, inblk, outblk); 602 if (outblk == blk) 603 crypto_cursor_copyback(&cc_out, blksz, blk); 604 else 605 crypto_cursor_advance(&cc_out, blksz); 606 } 607 if (resid > 0) { 608 crypto_cursor_copydata(&cc_in, resid, blk); 609 exf->decrypt_last(ctx, blk, blk, resid); 610 crypto_cursor_copyback(&cc_out, resid, blk); 611 } 612 } else { 613 /* Inject the authentication data */ 614 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 615 } 616 617 out: 618 explicit_bzero(ctx, exf->ctxsize); 619 explicit_bzero(blkbuf, sizeof(blkbuf)); 620 explicit_bzero(tag, sizeof(tag)); 621 622 return (error); 623 } 624 625 static void 626 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 627 u_int data_length, u_int tag_length, uint8_t *b0) 628 { 629 uint8_t *bp; 630 uint8_t flags, L; 631 632 KASSERT(nonce_length >= 7 && nonce_length <= 13, 633 ("nonce_length must be between 7 and 13 bytes")); 634 635 /* 636 * Need to determine the L field value. This is the number of 637 * bytes needed to specify the length of the message; the length 638 * is whatever is left in the 16 bytes after specifying flags and 639 * the nonce. 640 */ 641 L = 15 - nonce_length; 642 643 flags = ((aad_length > 0) << 6) + 644 (((tag_length - 2) / 2) << 3) + 645 L - 1; 646 647 /* 648 * Now we need to set up the first block, which has flags, nonce, 649 * and the message length. 650 */ 651 b0[0] = flags; 652 memcpy(b0 + 1, nonce, nonce_length); 653 bp = b0 + 1 + nonce_length; 654 655 /* Need to copy L' [aka L-1] bytes of data_length */ 656 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 657 *dst = data_length; 658 data_length >>= 8; 659 } 660 } 661 662 /* NB: OCF only supports AAD lengths < 2^32. */ 663 static int 664 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 665 { 666 if (aad_length < ((1 << 16) - (1 << 8))) { 667 be16enc(blk, aad_length); 668 return (sizeof(uint16_t)); 669 } else { 670 blk[0] = 0xff; 671 blk[1] = 0xfe; 672 be32enc(blk + 2, aad_length); 673 return (2 + sizeof(uint32_t)); 674 } 675 } 676 677 static int 678 swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 679 { 680 u_char iv[AES_BLOCK_LEN]; 681 u_char blk[CCM_CBC_BLOCK_LEN]; 682 u_char tag[AES_CBC_MAC_HASH_LEN]; 683 union authctx ctx; 684 const struct crypto_session_params *csp; 685 const struct swcr_auth *swa; 686 const struct auth_hash *axf; 687 int error, ivlen, len; 688 689 csp = crypto_get_params(crp->crp_session); 690 swa = &ses->swcr_auth; 691 axf = swa->sw_axf; 692 693 if (crp->crp_auth_key != NULL) { 694 axf->Init(&ctx); 695 axf->Setkey(&ctx, crp->crp_auth_key, csp->csp_auth_klen); 696 } else 697 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 698 699 /* Initialize the IV */ 700 ivlen = csp->csp_ivlen; 701 crypto_read_iv(crp, iv); 702 703 /* Supply MAC with IV */ 704 axf->Reinit(&ctx, crp->crp_iv, ivlen); 705 706 /* Supply MAC with b0. */ 707 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 708 swa->sw_mlen, blk); 709 axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 710 711 len = build_ccm_aad_length(crp->crp_payload_length, blk); 712 axf->Update(&ctx, blk, len); 713 714 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 715 axf->Update, &ctx); 716 717 /* Finalize MAC */ 718 axf->Final(tag, &ctx); 719 720 error = 0; 721 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 722 u_char tag2[AES_CBC_MAC_HASH_LEN]; 723 724 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 725 tag2); 726 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 727 error = EBADMSG; 728 explicit_bzero(tag2, sizeof(tag)); 729 } else { 730 /* Inject the authentication data */ 731 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 732 } 733 explicit_bzero(tag, sizeof(tag)); 734 explicit_bzero(blk, sizeof(blk)); 735 explicit_bzero(iv, sizeof(iv)); 736 return (error); 737 } 738 739 static int 740 swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 741 { 742 const struct crypto_session_params *csp; 743 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 744 u_char *blk = (u_char *)blkbuf; 745 u_char tag[AES_CBC_MAC_HASH_LEN]; 746 struct crypto_buffer_cursor cc_in, cc_out; 747 const u_char *inblk; 748 u_char *outblk; 749 const struct swcr_auth *swa; 750 const struct swcr_encdec *swe; 751 const struct enc_xform *exf; 752 void *ctx; 753 size_t len; 754 int blksz, error, ivlen, r, resid; 755 756 csp = crypto_get_params(crp->crp_session); 757 swa = &ses->swcr_auth; 758 swe = &ses->swcr_encdec; 759 exf = swe->sw_exf; 760 blksz = AES_BLOCK_LEN; 761 KASSERT(blksz == exf->native_blocksize, 762 ("%s: blocksize mismatch", __func__)); 763 764 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 765 return (EMSGSIZE); 766 767 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 768 return (EINVAL); 769 770 ivlen = csp->csp_ivlen; 771 772 ctx = __builtin_alloca(exf->ctxsize); 773 if (crp->crp_cipher_key != NULL) 774 exf->setkey(ctx, crp->crp_cipher_key, 775 crypto_get_params(crp->crp_session)->csp_cipher_klen); 776 else 777 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 778 exf->reinit(ctx, crp->crp_iv, ivlen); 779 780 /* Supply MAC with b0. */ 781 _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 782 "blkbuf too small for b0"); 783 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 784 crp->crp_payload_length, swa->sw_mlen, blk); 785 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 786 787 /* Supply MAC with AAD */ 788 if (crp->crp_aad_length != 0) { 789 len = build_ccm_aad_length(crp->crp_aad_length, blk); 790 exf->update(ctx, blk, len); 791 if (crp->crp_aad != NULL) 792 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 793 else 794 crypto_apply(crp, crp->crp_aad_start, 795 crp->crp_aad_length, exf->update, ctx); 796 797 /* Pad the AAD (including length field) to a full block. */ 798 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 799 if (len != 0) { 800 len = CCM_CBC_BLOCK_LEN - len; 801 memset(blk, 0, CCM_CBC_BLOCK_LEN); 802 exf->update(ctx, blk, len); 803 } 804 } 805 806 /* Do encryption/decryption with MAC */ 807 crypto_cursor_init(&cc_in, &crp->crp_buf); 808 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 809 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 810 crypto_cursor_init(&cc_out, &crp->crp_obuf); 811 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 812 } else 813 cc_out = cc_in; 814 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 815 inblk = crypto_cursor_segment(&cc_in, &len); 816 if (len < blksz) { 817 crypto_cursor_copydata(&cc_in, blksz, blk); 818 inblk = blk; 819 } else 820 crypto_cursor_advance(&cc_in, blksz); 821 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 822 outblk = crypto_cursor_segment(&cc_out, &len); 823 if (len < blksz) 824 outblk = blk; 825 exf->update(ctx, inblk, blksz); 826 exf->encrypt(ctx, inblk, outblk); 827 if (outblk == blk) 828 crypto_cursor_copyback(&cc_out, blksz, blk); 829 else 830 crypto_cursor_advance(&cc_out, blksz); 831 } else { 832 /* 833 * One of the problems with CCM+CBC is that 834 * the authentication is done on the 835 * unencrypted data. As a result, we have to 836 * decrypt the data twice: once to generate 837 * the tag and a second time after the tag is 838 * verified. 839 */ 840 exf->decrypt(ctx, inblk, blk); 841 exf->update(ctx, blk, blksz); 842 } 843 } 844 if (resid > 0) { 845 crypto_cursor_copydata(&cc_in, resid, blk); 846 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 847 exf->update(ctx, blk, resid); 848 exf->encrypt_last(ctx, blk, blk, resid); 849 crypto_cursor_copyback(&cc_out, resid, blk); 850 } else { 851 exf->decrypt_last(ctx, blk, blk, resid); 852 exf->update(ctx, blk, resid); 853 } 854 } 855 856 /* Finalize MAC */ 857 exf->final(tag, ctx); 858 859 /* Validate tag */ 860 error = 0; 861 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 862 u_char tag2[AES_CBC_MAC_HASH_LEN]; 863 864 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 865 tag2); 866 867 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 868 explicit_bzero(tag2, sizeof(tag2)); 869 if (r != 0) { 870 error = EBADMSG; 871 goto out; 872 } 873 874 /* tag matches, decrypt data */ 875 exf->reinit(ctx, crp->crp_iv, ivlen); 876 crypto_cursor_init(&cc_in, &crp->crp_buf); 877 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 878 for (resid = crp->crp_payload_length; resid > blksz; 879 resid -= blksz) { 880 inblk = crypto_cursor_segment(&cc_in, &len); 881 if (len < blksz) { 882 crypto_cursor_copydata(&cc_in, blksz, blk); 883 inblk = blk; 884 } else 885 crypto_cursor_advance(&cc_in, blksz); 886 outblk = crypto_cursor_segment(&cc_out, &len); 887 if (len < blksz) 888 outblk = blk; 889 exf->decrypt(ctx, inblk, outblk); 890 if (outblk == blk) 891 crypto_cursor_copyback(&cc_out, blksz, blk); 892 else 893 crypto_cursor_advance(&cc_out, blksz); 894 } 895 if (resid > 0) { 896 crypto_cursor_copydata(&cc_in, resid, blk); 897 exf->decrypt_last(ctx, blk, blk, resid); 898 crypto_cursor_copyback(&cc_out, resid, blk); 899 } 900 } else { 901 /* Inject the authentication data */ 902 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 903 } 904 905 out: 906 explicit_bzero(ctx, exf->ctxsize); 907 explicit_bzero(blkbuf, sizeof(blkbuf)); 908 explicit_bzero(tag, sizeof(tag)); 909 return (error); 910 } 911 912 static int 913 swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) 914 { 915 const struct crypto_session_params *csp; 916 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 917 u_char *blk = (u_char *)blkbuf; 918 u_char tag[POLY1305_HASH_LEN]; 919 struct crypto_buffer_cursor cc_in, cc_out; 920 const u_char *inblk; 921 u_char *outblk; 922 uint64_t *blkp; 923 const struct swcr_auth *swa; 924 const struct swcr_encdec *swe; 925 const struct enc_xform *exf; 926 void *ctx; 927 size_t len; 928 int blksz, error, r, resid; 929 930 swa = &ses->swcr_auth; 931 swe = &ses->swcr_encdec; 932 exf = swe->sw_exf; 933 blksz = exf->native_blocksize; 934 KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 935 936 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 937 return (EINVAL); 938 939 csp = crypto_get_params(crp->crp_session); 940 941 ctx = __builtin_alloca(exf->ctxsize); 942 if (crp->crp_cipher_key != NULL) 943 exf->setkey(ctx, crp->crp_cipher_key, 944 csp->csp_cipher_klen); 945 else 946 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 947 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 948 949 /* Supply MAC with AAD */ 950 if (crp->crp_aad != NULL) 951 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 952 else 953 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 954 exf->update, ctx); 955 if (crp->crp_aad_length % 16 != 0) { 956 /* padding1 */ 957 memset(blk, 0, 16); 958 exf->update(ctx, blk, 16 - crp->crp_aad_length % 16); 959 } 960 961 /* Do encryption with MAC */ 962 crypto_cursor_init(&cc_in, &crp->crp_buf); 963 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 964 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 965 crypto_cursor_init(&cc_out, &crp->crp_obuf); 966 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 967 } else 968 cc_out = cc_in; 969 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 970 inblk = crypto_cursor_segment(&cc_in, &len); 971 if (len < blksz) { 972 crypto_cursor_copydata(&cc_in, blksz, blk); 973 inblk = blk; 974 } else 975 crypto_cursor_advance(&cc_in, blksz); 976 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 977 outblk = crypto_cursor_segment(&cc_out, &len); 978 if (len < blksz) 979 outblk = blk; 980 exf->encrypt(ctx, inblk, outblk); 981 exf->update(ctx, outblk, blksz); 982 if (outblk == blk) 983 crypto_cursor_copyback(&cc_out, blksz, blk); 984 else 985 crypto_cursor_advance(&cc_out, blksz); 986 } else { 987 exf->update(ctx, inblk, blksz); 988 } 989 } 990 if (resid > 0) { 991 crypto_cursor_copydata(&cc_in, resid, blk); 992 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 993 exf->encrypt_last(ctx, blk, blk, resid); 994 crypto_cursor_copyback(&cc_out, resid, blk); 995 } 996 exf->update(ctx, blk, resid); 997 if (resid % 16 != 0) { 998 /* padding2 */ 999 memset(blk, 0, 16); 1000 exf->update(ctx, blk, 16 - resid % 16); 1001 } 1002 } 1003 1004 /* lengths */ 1005 blkp = (uint64_t *)blk; 1006 blkp[0] = htole64(crp->crp_aad_length); 1007 blkp[1] = htole64(crp->crp_payload_length); 1008 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1009 1010 /* Finalize MAC */ 1011 exf->final(tag, ctx); 1012 1013 /* Validate tag */ 1014 error = 0; 1015 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1016 u_char tag2[POLY1305_HASH_LEN]; 1017 1018 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 1019 1020 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 1021 explicit_bzero(tag2, sizeof(tag2)); 1022 if (r != 0) { 1023 error = EBADMSG; 1024 goto out; 1025 } 1026 1027 /* tag matches, decrypt data */ 1028 crypto_cursor_init(&cc_in, &crp->crp_buf); 1029 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1030 for (resid = crp->crp_payload_length; resid > blksz; 1031 resid -= blksz) { 1032 inblk = crypto_cursor_segment(&cc_in, &len); 1033 if (len < blksz) { 1034 crypto_cursor_copydata(&cc_in, blksz, blk); 1035 inblk = blk; 1036 } else 1037 crypto_cursor_advance(&cc_in, blksz); 1038 outblk = crypto_cursor_segment(&cc_out, &len); 1039 if (len < blksz) 1040 outblk = blk; 1041 exf->decrypt(ctx, inblk, outblk); 1042 if (outblk == blk) 1043 crypto_cursor_copyback(&cc_out, blksz, blk); 1044 else 1045 crypto_cursor_advance(&cc_out, blksz); 1046 } 1047 if (resid > 0) { 1048 crypto_cursor_copydata(&cc_in, resid, blk); 1049 exf->decrypt_last(ctx, blk, blk, resid); 1050 crypto_cursor_copyback(&cc_out, resid, blk); 1051 } 1052 } else { 1053 /* Inject the authentication data */ 1054 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 1055 } 1056 1057 out: 1058 explicit_bzero(ctx, exf->ctxsize); 1059 explicit_bzero(blkbuf, sizeof(blkbuf)); 1060 explicit_bzero(tag, sizeof(tag)); 1061 return (error); 1062 } 1063 1064 /* 1065 * Apply a cipher and a digest to perform EtA. 1066 */ 1067 static int 1068 swcr_eta(struct swcr_session *ses, struct cryptop *crp) 1069 { 1070 int error; 1071 1072 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1073 error = swcr_encdec(ses, crp); 1074 if (error == 0) 1075 error = swcr_authcompute(ses, crp); 1076 } else { 1077 error = swcr_authcompute(ses, crp); 1078 if (error == 0) 1079 error = swcr_encdec(ses, crp); 1080 } 1081 return (error); 1082 } 1083 1084 /* 1085 * Apply a compression/decompression algorithm 1086 */ 1087 static int 1088 swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 1089 { 1090 const struct comp_algo *cxf; 1091 uint8_t *data, *out; 1092 int adj; 1093 uint32_t result; 1094 1095 cxf = ses->swcr_compdec.sw_cxf; 1096 1097 /* We must handle the whole buffer of data in one time 1098 * then if there is not all the data in the mbuf, we must 1099 * copy in a buffer. 1100 */ 1101 1102 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1103 if (data == NULL) 1104 return (EINVAL); 1105 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1106 data); 1107 1108 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1109 result = cxf->compress(data, crp->crp_payload_length, &out); 1110 else 1111 result = cxf->decompress(data, crp->crp_payload_length, &out); 1112 1113 free(data, M_CRYPTO_DATA); 1114 if (result == 0) 1115 return (EINVAL); 1116 crp->crp_olen = result; 1117 1118 /* Check the compressed size when doing compression */ 1119 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1120 if (result >= crp->crp_payload_length) { 1121 /* Compression was useless, we lost time */ 1122 free(out, M_CRYPTO_DATA); 1123 return (0); 1124 } 1125 } 1126 1127 /* Copy back the (de)compressed data. m_copyback is 1128 * extending the mbuf as necessary. 1129 */ 1130 crypto_copyback(crp, crp->crp_payload_start, result, out); 1131 if (result < crp->crp_payload_length) { 1132 switch (crp->crp_buf.cb_type) { 1133 case CRYPTO_BUF_MBUF: 1134 case CRYPTO_BUF_SINGLE_MBUF: 1135 adj = result - crp->crp_payload_length; 1136 m_adj(crp->crp_buf.cb_mbuf, adj); 1137 break; 1138 case CRYPTO_BUF_UIO: { 1139 struct uio *uio = crp->crp_buf.cb_uio; 1140 int ind; 1141 1142 adj = crp->crp_payload_length - result; 1143 ind = uio->uio_iovcnt - 1; 1144 1145 while (adj > 0 && ind >= 0) { 1146 if (adj < uio->uio_iov[ind].iov_len) { 1147 uio->uio_iov[ind].iov_len -= adj; 1148 break; 1149 } 1150 1151 adj -= uio->uio_iov[ind].iov_len; 1152 uio->uio_iov[ind].iov_len = 0; 1153 ind--; 1154 uio->uio_iovcnt--; 1155 } 1156 } 1157 break; 1158 case CRYPTO_BUF_VMPAGE: 1159 adj = crp->crp_payload_length - result; 1160 crp->crp_buf.cb_vm_page_len -= adj; 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 free(out, M_CRYPTO_DATA); 1167 return 0; 1168 } 1169 1170 static int 1171 swcr_setup_cipher(struct swcr_session *ses, 1172 const struct crypto_session_params *csp) 1173 { 1174 struct swcr_encdec *swe; 1175 const struct enc_xform *txf; 1176 int error; 1177 1178 swe = &ses->swcr_encdec; 1179 txf = crypto_cipher(csp); 1180 if (csp->csp_cipher_key != NULL) { 1181 if (txf->ctxsize != 0) { 1182 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1183 M_NOWAIT); 1184 if (swe->sw_ctx == NULL) 1185 return (ENOMEM); 1186 } 1187 error = txf->setkey(swe->sw_ctx, 1188 csp->csp_cipher_key, csp->csp_cipher_klen); 1189 if (error) 1190 return (error); 1191 } 1192 swe->sw_exf = txf; 1193 return (0); 1194 } 1195 1196 static int 1197 swcr_setup_auth(struct swcr_session *ses, 1198 const struct crypto_session_params *csp) 1199 { 1200 struct swcr_auth *swa; 1201 const struct auth_hash *axf; 1202 1203 swa = &ses->swcr_auth; 1204 1205 axf = crypto_auth_hash(csp); 1206 swa->sw_axf = axf; 1207 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1208 return (EINVAL); 1209 if (csp->csp_auth_mlen == 0) 1210 swa->sw_mlen = axf->hashsize; 1211 else 1212 swa->sw_mlen = csp->csp_auth_mlen; 1213 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1214 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1215 M_NOWAIT); 1216 if (swa->sw_ictx == NULL) 1217 return (ENOBUFS); 1218 } 1219 1220 switch (csp->csp_auth_alg) { 1221 case CRYPTO_SHA1_HMAC: 1222 case CRYPTO_SHA2_224_HMAC: 1223 case CRYPTO_SHA2_256_HMAC: 1224 case CRYPTO_SHA2_384_HMAC: 1225 case CRYPTO_SHA2_512_HMAC: 1226 case CRYPTO_NULL_HMAC: 1227 case CRYPTO_RIPEMD160_HMAC: 1228 swa->sw_hmac = true; 1229 if (csp->csp_auth_key != NULL) { 1230 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1231 M_NOWAIT); 1232 if (swa->sw_octx == NULL) 1233 return (ENOBUFS); 1234 hmac_init_ipad(axf, csp->csp_auth_key, 1235 csp->csp_auth_klen, swa->sw_ictx); 1236 hmac_init_opad(axf, csp->csp_auth_key, 1237 csp->csp_auth_klen, swa->sw_octx); 1238 } 1239 break; 1240 case CRYPTO_SHA1: 1241 case CRYPTO_SHA2_224: 1242 case CRYPTO_SHA2_256: 1243 case CRYPTO_SHA2_384: 1244 case CRYPTO_SHA2_512: 1245 axf->Init(swa->sw_ictx); 1246 break; 1247 case CRYPTO_AES_NIST_GMAC: 1248 case CRYPTO_AES_CCM_CBC_MAC: 1249 case CRYPTO_POLY1305: 1250 if (csp->csp_auth_key != NULL) { 1251 axf->Init(swa->sw_ictx); 1252 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1253 csp->csp_auth_klen); 1254 } 1255 break; 1256 case CRYPTO_BLAKE2B: 1257 case CRYPTO_BLAKE2S: 1258 /* 1259 * Blake2b and Blake2s support an optional key but do 1260 * not require one. 1261 */ 1262 if (csp->csp_auth_klen == 0) 1263 axf->Init(swa->sw_ictx); 1264 else if (csp->csp_auth_key != NULL) 1265 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1266 csp->csp_auth_klen); 1267 break; 1268 } 1269 1270 if (csp->csp_mode == CSP_MODE_DIGEST) { 1271 switch (csp->csp_auth_alg) { 1272 case CRYPTO_AES_NIST_GMAC: 1273 ses->swcr_process = swcr_gmac; 1274 break; 1275 case CRYPTO_AES_CCM_CBC_MAC: 1276 ses->swcr_process = swcr_ccm_cbc_mac; 1277 break; 1278 default: 1279 ses->swcr_process = swcr_authcompute; 1280 } 1281 } 1282 1283 return (0); 1284 } 1285 1286 static int 1287 swcr_setup_aead(struct swcr_session *ses, 1288 const struct crypto_session_params *csp) 1289 { 1290 struct swcr_auth *swa; 1291 int error; 1292 1293 error = swcr_setup_cipher(ses, csp); 1294 if (error) 1295 return (error); 1296 1297 swa = &ses->swcr_auth; 1298 if (csp->csp_auth_mlen == 0) 1299 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1300 else 1301 swa->sw_mlen = csp->csp_auth_mlen; 1302 return (0); 1303 } 1304 1305 static bool 1306 swcr_auth_supported(const struct crypto_session_params *csp) 1307 { 1308 const struct auth_hash *axf; 1309 1310 axf = crypto_auth_hash(csp); 1311 if (axf == NULL) 1312 return (false); 1313 switch (csp->csp_auth_alg) { 1314 case CRYPTO_SHA1_HMAC: 1315 case CRYPTO_SHA2_224_HMAC: 1316 case CRYPTO_SHA2_256_HMAC: 1317 case CRYPTO_SHA2_384_HMAC: 1318 case CRYPTO_SHA2_512_HMAC: 1319 case CRYPTO_NULL_HMAC: 1320 case CRYPTO_RIPEMD160_HMAC: 1321 break; 1322 case CRYPTO_AES_NIST_GMAC: 1323 switch (csp->csp_auth_klen * 8) { 1324 case 128: 1325 case 192: 1326 case 256: 1327 break; 1328 default: 1329 return (false); 1330 } 1331 if (csp->csp_auth_key == NULL) 1332 return (false); 1333 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1334 return (false); 1335 break; 1336 case CRYPTO_POLY1305: 1337 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1338 return (false); 1339 break; 1340 case CRYPTO_AES_CCM_CBC_MAC: 1341 switch (csp->csp_auth_klen * 8) { 1342 case 128: 1343 case 192: 1344 case 256: 1345 break; 1346 default: 1347 return (false); 1348 } 1349 if (csp->csp_auth_key == NULL) 1350 return (false); 1351 break; 1352 } 1353 return (true); 1354 } 1355 1356 static bool 1357 swcr_cipher_supported(const struct crypto_session_params *csp) 1358 { 1359 const struct enc_xform *txf; 1360 1361 txf = crypto_cipher(csp); 1362 if (txf == NULL) 1363 return (false); 1364 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1365 txf->ivsize != csp->csp_ivlen) 1366 return (false); 1367 return (true); 1368 } 1369 1370 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1371 1372 static int 1373 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1374 { 1375 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1376 return (EINVAL); 1377 switch (csp->csp_mode) { 1378 case CSP_MODE_COMPRESS: 1379 switch (csp->csp_cipher_alg) { 1380 case CRYPTO_DEFLATE_COMP: 1381 break; 1382 default: 1383 return (EINVAL); 1384 } 1385 break; 1386 case CSP_MODE_CIPHER: 1387 switch (csp->csp_cipher_alg) { 1388 case CRYPTO_AES_NIST_GCM_16: 1389 case CRYPTO_AES_CCM_16: 1390 case CRYPTO_CHACHA20_POLY1305: 1391 return (EINVAL); 1392 default: 1393 if (!swcr_cipher_supported(csp)) 1394 return (EINVAL); 1395 break; 1396 } 1397 break; 1398 case CSP_MODE_DIGEST: 1399 if (!swcr_auth_supported(csp)) 1400 return (EINVAL); 1401 break; 1402 case CSP_MODE_AEAD: 1403 switch (csp->csp_cipher_alg) { 1404 case CRYPTO_AES_NIST_GCM_16: 1405 case CRYPTO_AES_CCM_16: 1406 switch (csp->csp_cipher_klen * 8) { 1407 case 128: 1408 case 192: 1409 case 256: 1410 break; 1411 default: 1412 return (EINVAL); 1413 } 1414 break; 1415 case CRYPTO_CHACHA20_POLY1305: 1416 break; 1417 default: 1418 return (EINVAL); 1419 } 1420 break; 1421 case CSP_MODE_ETA: 1422 /* AEAD algorithms cannot be used for EtA. */ 1423 switch (csp->csp_cipher_alg) { 1424 case CRYPTO_AES_NIST_GCM_16: 1425 case CRYPTO_AES_CCM_16: 1426 case CRYPTO_CHACHA20_POLY1305: 1427 return (EINVAL); 1428 } 1429 switch (csp->csp_auth_alg) { 1430 case CRYPTO_AES_NIST_GMAC: 1431 case CRYPTO_AES_CCM_CBC_MAC: 1432 return (EINVAL); 1433 } 1434 1435 if (!swcr_cipher_supported(csp) || 1436 !swcr_auth_supported(csp)) 1437 return (EINVAL); 1438 break; 1439 default: 1440 return (EINVAL); 1441 } 1442 1443 return (CRYPTODEV_PROBE_SOFTWARE); 1444 } 1445 1446 /* 1447 * Generate a new software session. 1448 */ 1449 static int 1450 swcr_newsession(device_t dev, crypto_session_t cses, 1451 const struct crypto_session_params *csp) 1452 { 1453 struct swcr_session *ses; 1454 const struct comp_algo *cxf; 1455 int error; 1456 1457 ses = crypto_get_driver_session(cses); 1458 mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1459 1460 error = 0; 1461 switch (csp->csp_mode) { 1462 case CSP_MODE_COMPRESS: 1463 switch (csp->csp_cipher_alg) { 1464 case CRYPTO_DEFLATE_COMP: 1465 cxf = &comp_algo_deflate; 1466 break; 1467 #ifdef INVARIANTS 1468 default: 1469 panic("bad compression algo"); 1470 #endif 1471 } 1472 ses->swcr_compdec.sw_cxf = cxf; 1473 ses->swcr_process = swcr_compdec; 1474 break; 1475 case CSP_MODE_CIPHER: 1476 switch (csp->csp_cipher_alg) { 1477 case CRYPTO_NULL_CBC: 1478 ses->swcr_process = swcr_null; 1479 break; 1480 #ifdef INVARIANTS 1481 case CRYPTO_AES_NIST_GCM_16: 1482 case CRYPTO_AES_CCM_16: 1483 case CRYPTO_CHACHA20_POLY1305: 1484 panic("bad cipher algo"); 1485 #endif 1486 default: 1487 error = swcr_setup_cipher(ses, csp); 1488 if (error == 0) 1489 ses->swcr_process = swcr_encdec; 1490 } 1491 break; 1492 case CSP_MODE_DIGEST: 1493 error = swcr_setup_auth(ses, csp); 1494 break; 1495 case CSP_MODE_AEAD: 1496 switch (csp->csp_cipher_alg) { 1497 case CRYPTO_AES_NIST_GCM_16: 1498 error = swcr_setup_aead(ses, csp); 1499 if (error == 0) 1500 ses->swcr_process = swcr_gcm; 1501 break; 1502 case CRYPTO_AES_CCM_16: 1503 error = swcr_setup_aead(ses, csp); 1504 if (error == 0) 1505 ses->swcr_process = swcr_ccm; 1506 break; 1507 case CRYPTO_CHACHA20_POLY1305: 1508 error = swcr_setup_aead(ses, csp); 1509 if (error == 0) 1510 ses->swcr_process = swcr_chacha20_poly1305; 1511 break; 1512 #ifdef INVARIANTS 1513 default: 1514 panic("bad aead algo"); 1515 #endif 1516 } 1517 break; 1518 case CSP_MODE_ETA: 1519 #ifdef INVARIANTS 1520 switch (csp->csp_cipher_alg) { 1521 case CRYPTO_AES_NIST_GCM_16: 1522 case CRYPTO_AES_CCM_16: 1523 case CRYPTO_CHACHA20_POLY1305: 1524 panic("bad eta cipher algo"); 1525 } 1526 switch (csp->csp_auth_alg) { 1527 case CRYPTO_AES_NIST_GMAC: 1528 case CRYPTO_AES_CCM_CBC_MAC: 1529 panic("bad eta auth algo"); 1530 } 1531 #endif 1532 1533 error = swcr_setup_auth(ses, csp); 1534 if (error) 1535 break; 1536 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1537 /* Effectively degrade to digest mode. */ 1538 ses->swcr_process = swcr_authcompute; 1539 break; 1540 } 1541 1542 error = swcr_setup_cipher(ses, csp); 1543 if (error == 0) 1544 ses->swcr_process = swcr_eta; 1545 break; 1546 default: 1547 error = EINVAL; 1548 } 1549 1550 if (error) 1551 swcr_freesession(dev, cses); 1552 return (error); 1553 } 1554 1555 static void 1556 swcr_freesession(device_t dev, crypto_session_t cses) 1557 { 1558 struct swcr_session *ses; 1559 1560 ses = crypto_get_driver_session(cses); 1561 1562 mtx_destroy(&ses->swcr_lock); 1563 1564 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1565 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1566 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1567 } 1568 1569 /* 1570 * Process a software request. 1571 */ 1572 static int 1573 swcr_process(device_t dev, struct cryptop *crp, int hint) 1574 { 1575 struct swcr_session *ses; 1576 1577 ses = crypto_get_driver_session(crp->crp_session); 1578 mtx_lock(&ses->swcr_lock); 1579 1580 crp->crp_etype = ses->swcr_process(ses, crp); 1581 1582 mtx_unlock(&ses->swcr_lock); 1583 crypto_done(crp); 1584 return (0); 1585 } 1586 1587 static void 1588 swcr_identify(driver_t *drv, device_t parent) 1589 { 1590 /* NB: order 10 is so we get attached after h/w devices */ 1591 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1592 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1593 panic("cryptosoft: could not attach"); 1594 } 1595 1596 static int 1597 swcr_probe(device_t dev) 1598 { 1599 device_set_desc(dev, "software crypto"); 1600 device_quiet(dev); 1601 return (BUS_PROBE_NOWILDCARD); 1602 } 1603 1604 static int 1605 swcr_attach(device_t dev) 1606 { 1607 1608 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1609 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1610 if (swcr_id < 0) { 1611 device_printf(dev, "cannot initialize!"); 1612 return (ENXIO); 1613 } 1614 1615 return (0); 1616 } 1617 1618 static int 1619 swcr_detach(device_t dev) 1620 { 1621 crypto_unregister_all(swcr_id); 1622 return 0; 1623 } 1624 1625 static device_method_t swcr_methods[] = { 1626 DEVMETHOD(device_identify, swcr_identify), 1627 DEVMETHOD(device_probe, swcr_probe), 1628 DEVMETHOD(device_attach, swcr_attach), 1629 DEVMETHOD(device_detach, swcr_detach), 1630 1631 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1632 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1633 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1634 DEVMETHOD(cryptodev_process, swcr_process), 1635 1636 {0, 0}, 1637 }; 1638 1639 static driver_t swcr_driver = { 1640 "cryptosoft", 1641 swcr_methods, 1642 0, /* NB: no softc */ 1643 }; 1644 static devclass_t swcr_devclass; 1645 1646 /* 1647 * NB: We explicitly reference the crypto module so we 1648 * get the necessary ordering when built as a loadable 1649 * module. This is required because we bundle the crypto 1650 * module code together with the cryptosoft driver (otherwise 1651 * normal module dependencies would handle things). 1652 */ 1653 extern int crypto_modevent(struct module *, int, void *); 1654 /* XXX where to attach */ 1655 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1656 MODULE_VERSION(cryptosoft, 1); 1657 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1658