1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen, todo; 109 int blksz, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blksz = exf->blocksize; 128 } else 129 blksz = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 158 159 /* 160 * Loop through encrypting blocks. 'inlen' is the remaining 161 * length of the current segment in the input buffer. 162 * 'outlen' is the remaining length of current segment in the 163 * output buffer. 164 */ 165 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 166 /* 167 * If the current block is not contained within the 168 * current input/output segment, use 'blk' as a local 169 * buffer. 170 */ 171 if (inlen < blksz) { 172 crypto_cursor_copydata(&cc_in, blksz, blk); 173 inblk = blk; 174 inlen = blksz; 175 } 176 if (outlen < blksz) { 177 outblk = blk; 178 outlen = blksz; 179 } 180 181 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 182 183 if (encrypting) 184 exf->encrypt_multi(ctx, inblk, outblk, todo); 185 else 186 exf->decrypt_multi(ctx, inblk, outblk, todo); 187 188 if (inblk == blk) { 189 inblk = crypto_cursor_segment(&cc_in, &inlen); 190 } else { 191 crypto_cursor_advance(&cc_in, todo); 192 inlen -= todo; 193 inblk += todo; 194 if (inlen == 0) 195 inblk = crypto_cursor_segment(&cc_in, &inlen); 196 } 197 198 if (outblk == blk) { 199 crypto_cursor_copyback(&cc_out, blksz, blk); 200 outblk = crypto_cursor_segment(&cc_out, &outlen); 201 } else { 202 crypto_cursor_advance(&cc_out, todo); 203 outlen -= todo; 204 outblk += todo; 205 if (outlen == 0) 206 outblk = crypto_cursor_segment(&cc_out, 207 &outlen); 208 } 209 } 210 211 /* Handle trailing partial block for stream ciphers. */ 212 if (resid > 0) { 213 KASSERT(exf->native_blocksize != 0, 214 ("%s: partial block of %d bytes for cipher %s", 215 __func__, resid, exf->name)); 216 KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 217 218 inblk = crypto_cursor_segment(&cc_in, &inlen); 219 outblk = crypto_cursor_segment(&cc_out, &outlen); 220 if (inlen < resid) { 221 crypto_cursor_copydata(&cc_in, resid, blk); 222 inblk = blk; 223 } 224 if (outlen < resid) 225 outblk = blk; 226 if (encrypting) 227 exf->encrypt_last(ctx, inblk, outblk, 228 resid); 229 else 230 exf->decrypt_last(ctx, inblk, outblk, 231 resid); 232 if (outlen < resid) 233 crypto_cursor_copyback(&cc_out, resid, blk); 234 } 235 236 explicit_bzero(ctx, exf->ctxsize); 237 explicit_bzero(blk, sizeof(blk)); 238 return (0); 239 } 240 241 /* 242 * Compute or verify hash. 243 */ 244 static int 245 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 246 { 247 struct { 248 union authctx ctx; 249 u_char aalg[HASH_MAX_LEN]; 250 u_char uaalg[HASH_MAX_LEN]; 251 } s; 252 const struct crypto_session_params *csp; 253 const struct swcr_auth *sw; 254 const struct auth_hash *axf; 255 int err; 256 257 sw = &ses->swcr_auth; 258 259 axf = sw->sw_axf; 260 261 csp = crypto_get_params(crp->crp_session); 262 if (crp->crp_auth_key != NULL) { 263 if (sw->sw_hmac) { 264 hmac_init_ipad(axf, crp->crp_auth_key, 265 csp->csp_auth_klen, &s.ctx); 266 } else { 267 axf->Init(&s.ctx); 268 axf->Setkey(&s.ctx, crp->crp_auth_key, 269 csp->csp_auth_klen); 270 } 271 } else 272 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 273 274 if (crp->crp_aad != NULL) 275 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 276 else 277 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 278 axf->Update, &s.ctx); 279 if (err) 280 goto out; 281 282 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 283 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 284 err = crypto_apply_buf(&crp->crp_obuf, 285 crp->crp_payload_output_start, crp->crp_payload_length, 286 axf->Update, &s.ctx); 287 else 288 err = crypto_apply(crp, crp->crp_payload_start, 289 crp->crp_payload_length, axf->Update, &s.ctx); 290 if (err) 291 goto out; 292 293 if (csp->csp_flags & CSP_F_ESN) 294 axf->Update(&s.ctx, crp->crp_esn, 4); 295 296 axf->Final(s.aalg, &s.ctx); 297 if (sw->sw_hmac) { 298 if (crp->crp_auth_key != NULL) 299 hmac_init_opad(axf, crp->crp_auth_key, 300 csp->csp_auth_klen, &s.ctx); 301 else 302 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 303 axf->Update(&s.ctx, s.aalg, axf->hashsize); 304 axf->Final(s.aalg, &s.ctx); 305 } 306 307 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 308 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 309 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 310 err = EBADMSG; 311 } else { 312 /* Inject the authentication data */ 313 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 314 } 315 out: 316 explicit_bzero(&s, sizeof(s)); 317 return (err); 318 } 319 320 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 321 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 322 323 static int 324 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 325 { 326 struct { 327 union authctx ctx; 328 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 329 u_char tag[GMAC_DIGEST_LEN]; 330 u_char tag2[GMAC_DIGEST_LEN]; 331 } s; 332 u_char *blk = (u_char *)s.blkbuf; 333 struct crypto_buffer_cursor cc; 334 const u_char *inblk; 335 const struct swcr_auth *swa; 336 const struct auth_hash *axf; 337 uint32_t *blkp; 338 size_t len; 339 int blksz, error, ivlen, resid; 340 341 swa = &ses->swcr_auth; 342 axf = swa->sw_axf; 343 blksz = GMAC_BLOCK_LEN; 344 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 345 __func__)); 346 347 if (crp->crp_auth_key != NULL) { 348 axf->Init(&s.ctx); 349 axf->Setkey(&s.ctx, crp->crp_auth_key, 350 crypto_get_params(crp->crp_session)->csp_auth_klen); 351 } else 352 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 353 354 /* Initialize the IV */ 355 ivlen = AES_GCM_IV_LEN; 356 crypto_read_iv(crp, blk); 357 358 axf->Reinit(&s.ctx, blk, ivlen); 359 crypto_cursor_init(&cc, &crp->crp_buf); 360 crypto_cursor_advance(&cc, crp->crp_payload_start); 361 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 362 inblk = crypto_cursor_segment(&cc, &len); 363 if (len >= blksz) { 364 len = rounddown(MIN(len, resid), blksz); 365 crypto_cursor_advance(&cc, len); 366 } else { 367 len = blksz; 368 crypto_cursor_copydata(&cc, len, blk); 369 inblk = blk; 370 } 371 axf->Update(&s.ctx, inblk, len); 372 } 373 if (resid > 0) { 374 memset(blk, 0, blksz); 375 crypto_cursor_copydata(&cc, resid, blk); 376 axf->Update(&s.ctx, blk, blksz); 377 } 378 379 /* length block */ 380 memset(blk, 0, blksz); 381 blkp = (uint32_t *)blk + 1; 382 *blkp = htobe32(crp->crp_payload_length * 8); 383 axf->Update(&s.ctx, blk, blksz); 384 385 /* Finalize MAC */ 386 axf->Final(s.tag, &s.ctx); 387 388 error = 0; 389 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 390 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 391 s.tag2); 392 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 393 error = EBADMSG; 394 } else { 395 /* Inject the authentication data */ 396 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 397 } 398 explicit_bzero(&s, sizeof(s)); 399 return (error); 400 } 401 402 static int 403 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 404 { 405 struct { 406 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 407 u_char tag[GMAC_DIGEST_LEN]; 408 u_char tag2[GMAC_DIGEST_LEN]; 409 } s; 410 u_char *blk = (u_char *)s.blkbuf; 411 struct crypto_buffer_cursor cc_in, cc_out; 412 const u_char *inblk; 413 u_char *outblk; 414 size_t inlen, outlen, todo; 415 const struct swcr_auth *swa; 416 const struct swcr_encdec *swe; 417 const struct enc_xform *exf; 418 void *ctx; 419 uint32_t *blkp; 420 int blksz, error, ivlen, r, resid; 421 422 swa = &ses->swcr_auth; 423 swe = &ses->swcr_encdec; 424 exf = swe->sw_exf; 425 blksz = GMAC_BLOCK_LEN; 426 KASSERT(blksz == exf->native_blocksize, 427 ("%s: blocksize mismatch", __func__)); 428 429 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 430 return (EINVAL); 431 432 ivlen = AES_GCM_IV_LEN; 433 434 ctx = __builtin_alloca(exf->ctxsize); 435 if (crp->crp_cipher_key != NULL) 436 exf->setkey(ctx, crp->crp_cipher_key, 437 crypto_get_params(crp->crp_session)->csp_cipher_klen); 438 else 439 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 440 exf->reinit(ctx, crp->crp_iv, ivlen); 441 442 /* Supply MAC with AAD */ 443 if (crp->crp_aad != NULL) { 444 inlen = rounddown2(crp->crp_aad_length, blksz); 445 if (inlen != 0) 446 exf->update(ctx, crp->crp_aad, inlen); 447 if (crp->crp_aad_length != inlen) { 448 memset(blk, 0, blksz); 449 memcpy(blk, (char *)crp->crp_aad + inlen, 450 crp->crp_aad_length - inlen); 451 exf->update(ctx, blk, blksz); 452 } 453 } else { 454 crypto_cursor_init(&cc_in, &crp->crp_buf); 455 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 456 for (resid = crp->crp_aad_length; resid >= blksz; 457 resid -= inlen) { 458 inblk = crypto_cursor_segment(&cc_in, &inlen); 459 if (inlen >= blksz) { 460 inlen = rounddown2(MIN(inlen, resid), blksz); 461 crypto_cursor_advance(&cc_in, inlen); 462 } else { 463 inlen = blksz; 464 crypto_cursor_copydata(&cc_in, inlen, blk); 465 inblk = blk; 466 } 467 exf->update(ctx, inblk, inlen); 468 } 469 if (resid > 0) { 470 memset(blk, 0, blksz); 471 crypto_cursor_copydata(&cc_in, resid, blk); 472 exf->update(ctx, blk, blksz); 473 } 474 } 475 476 /* Do encryption with MAC */ 477 crypto_cursor_init(&cc_in, &crp->crp_buf); 478 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 479 inblk = crypto_cursor_segment(&cc_in, &inlen); 480 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 481 crypto_cursor_init(&cc_out, &crp->crp_obuf); 482 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 483 } else 484 cc_out = cc_in; 485 outblk = crypto_cursor_segment(&cc_out, &outlen); 486 487 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 488 if (inlen < blksz) { 489 crypto_cursor_copydata(&cc_in, blksz, blk); 490 inblk = blk; 491 inlen = blksz; 492 } 493 494 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 495 if (outlen < blksz) { 496 outblk = blk; 497 outlen = blksz; 498 } 499 500 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 501 blksz); 502 503 exf->encrypt_multi(ctx, inblk, outblk, todo); 504 exf->update(ctx, outblk, todo); 505 506 if (outblk == blk) { 507 crypto_cursor_copyback(&cc_out, blksz, blk); 508 outblk = crypto_cursor_segment(&cc_out, &outlen); 509 } else { 510 crypto_cursor_advance(&cc_out, todo); 511 outlen -= todo; 512 outblk += todo; 513 if (outlen == 0) 514 outblk = crypto_cursor_segment(&cc_out, 515 &outlen); 516 } 517 } else { 518 todo = rounddown2(MIN(resid, inlen), blksz); 519 exf->update(ctx, inblk, todo); 520 } 521 522 if (inblk == blk) { 523 inblk = crypto_cursor_segment(&cc_in, &inlen); 524 } else { 525 crypto_cursor_advance(&cc_in, todo); 526 inlen -= todo; 527 inblk += todo; 528 if (inlen == 0) 529 inblk = crypto_cursor_segment(&cc_in, &inlen); 530 } 531 } 532 if (resid > 0) { 533 crypto_cursor_copydata(&cc_in, resid, blk); 534 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 535 exf->encrypt_last(ctx, blk, blk, resid); 536 crypto_cursor_copyback(&cc_out, resid, blk); 537 } 538 exf->update(ctx, blk, resid); 539 } 540 541 /* length block */ 542 memset(blk, 0, blksz); 543 blkp = (uint32_t *)blk + 1; 544 *blkp = htobe32(crp->crp_aad_length * 8); 545 blkp = (uint32_t *)blk + 3; 546 *blkp = htobe32(crp->crp_payload_length * 8); 547 exf->update(ctx, blk, blksz); 548 549 /* Finalize MAC */ 550 exf->final(s.tag, ctx); 551 552 /* Validate tag */ 553 error = 0; 554 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 555 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 556 s.tag2); 557 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 558 if (r != 0) { 559 error = EBADMSG; 560 goto out; 561 } 562 563 /* tag matches, decrypt data */ 564 crypto_cursor_init(&cc_in, &crp->crp_buf); 565 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 566 inblk = crypto_cursor_segment(&cc_in, &inlen); 567 568 for (resid = crp->crp_payload_length; resid > blksz; 569 resid -= todo) { 570 if (inlen < blksz) { 571 crypto_cursor_copydata(&cc_in, blksz, blk); 572 inblk = blk; 573 inlen = blksz; 574 } 575 if (outlen < blksz) { 576 outblk = blk; 577 outlen = blksz; 578 } 579 580 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 581 blksz); 582 583 exf->decrypt_multi(ctx, inblk, outblk, todo); 584 585 if (inblk == blk) { 586 inblk = crypto_cursor_segment(&cc_in, &inlen); 587 } else { 588 crypto_cursor_advance(&cc_in, todo); 589 inlen -= todo; 590 inblk += todo; 591 if (inlen == 0) 592 inblk = crypto_cursor_segment(&cc_in, 593 &inlen); 594 } 595 596 if (outblk == blk) { 597 crypto_cursor_copyback(&cc_out, blksz, blk); 598 outblk = crypto_cursor_segment(&cc_out, 599 &outlen); 600 } else { 601 crypto_cursor_advance(&cc_out, todo); 602 outlen -= todo; 603 outblk += todo; 604 if (outlen == 0) 605 outblk = crypto_cursor_segment(&cc_out, 606 &outlen); 607 } 608 } 609 if (resid > 0) { 610 crypto_cursor_copydata(&cc_in, resid, blk); 611 exf->decrypt_last(ctx, blk, blk, resid); 612 crypto_cursor_copyback(&cc_out, resid, blk); 613 } 614 } else { 615 /* Inject the authentication data */ 616 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 617 s.tag); 618 } 619 620 out: 621 explicit_bzero(ctx, exf->ctxsize); 622 explicit_bzero(&s, sizeof(s)); 623 624 return (error); 625 } 626 627 static void 628 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 629 u_int data_length, u_int tag_length, uint8_t *b0) 630 { 631 uint8_t *bp; 632 uint8_t flags, L; 633 634 KASSERT(nonce_length >= 7 && nonce_length <= 13, 635 ("nonce_length must be between 7 and 13 bytes")); 636 637 /* 638 * Need to determine the L field value. This is the number of 639 * bytes needed to specify the length of the message; the length 640 * is whatever is left in the 16 bytes after specifying flags and 641 * the nonce. 642 */ 643 L = 15 - nonce_length; 644 645 flags = ((aad_length > 0) << 6) + 646 (((tag_length - 2) / 2) << 3) + 647 L - 1; 648 649 /* 650 * Now we need to set up the first block, which has flags, nonce, 651 * and the message length. 652 */ 653 b0[0] = flags; 654 memcpy(b0 + 1, nonce, nonce_length); 655 bp = b0 + 1 + nonce_length; 656 657 /* Need to copy L' [aka L-1] bytes of data_length */ 658 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 659 *dst = data_length; 660 data_length >>= 8; 661 } 662 } 663 664 /* NB: OCF only supports AAD lengths < 2^32. */ 665 static int 666 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 667 { 668 if (aad_length < ((1 << 16) - (1 << 8))) { 669 be16enc(blk, aad_length); 670 return (sizeof(uint16_t)); 671 } else { 672 blk[0] = 0xff; 673 blk[1] = 0xfe; 674 be32enc(blk + 2, aad_length); 675 return (2 + sizeof(uint32_t)); 676 } 677 } 678 679 static int 680 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 681 { 682 struct { 683 union authctx ctx; 684 u_char blk[CCM_CBC_BLOCK_LEN]; 685 u_char tag[AES_CBC_MAC_HASH_LEN]; 686 u_char tag2[AES_CBC_MAC_HASH_LEN]; 687 } s; 688 const struct crypto_session_params *csp; 689 const struct swcr_auth *swa; 690 const struct auth_hash *axf; 691 int error, ivlen, len; 692 693 csp = crypto_get_params(crp->crp_session); 694 swa = &ses->swcr_auth; 695 axf = swa->sw_axf; 696 697 if (crp->crp_auth_key != NULL) { 698 axf->Init(&s.ctx); 699 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 700 } else 701 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 702 703 /* Initialize the IV */ 704 ivlen = csp->csp_ivlen; 705 706 /* Supply MAC with IV */ 707 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 708 709 /* Supply MAC with b0. */ 710 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 711 swa->sw_mlen, s.blk); 712 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 713 714 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 715 axf->Update(&s.ctx, s.blk, len); 716 717 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 718 axf->Update, &s.ctx); 719 720 /* Finalize MAC */ 721 axf->Final(s.tag, &s.ctx); 722 723 error = 0; 724 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 725 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 726 s.tag2); 727 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 728 error = EBADMSG; 729 } else { 730 /* Inject the authentication data */ 731 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 732 s.tag); 733 } 734 explicit_bzero(&s, sizeof(s)); 735 return (error); 736 } 737 738 static int 739 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 740 { 741 const struct crypto_session_params *csp; 742 struct { 743 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 744 u_char tag[AES_CBC_MAC_HASH_LEN]; 745 u_char tag2[AES_CBC_MAC_HASH_LEN]; 746 } s; 747 u_char *blk = (u_char *)s.blkbuf; 748 struct crypto_buffer_cursor cc_in, cc_out; 749 const u_char *inblk; 750 u_char *outblk; 751 const struct swcr_auth *swa; 752 const struct swcr_encdec *swe; 753 const struct enc_xform *exf; 754 void *ctx; 755 size_t len; 756 int blksz, error, ivlen, r, resid; 757 758 csp = crypto_get_params(crp->crp_session); 759 swa = &ses->swcr_auth; 760 swe = &ses->swcr_encdec; 761 exf = swe->sw_exf; 762 blksz = AES_BLOCK_LEN; 763 KASSERT(blksz == exf->native_blocksize, 764 ("%s: blocksize mismatch", __func__)); 765 766 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 767 return (EMSGSIZE); 768 769 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 770 return (EINVAL); 771 772 ivlen = csp->csp_ivlen; 773 774 ctx = __builtin_alloca(exf->ctxsize); 775 if (crp->crp_cipher_key != NULL) 776 exf->setkey(ctx, crp->crp_cipher_key, 777 crypto_get_params(crp->crp_session)->csp_cipher_klen); 778 else 779 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 780 exf->reinit(ctx, crp->crp_iv, ivlen); 781 782 /* Supply MAC with b0. */ 783 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 784 "blkbuf too small for b0"); 785 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 786 crp->crp_payload_length, swa->sw_mlen, blk); 787 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 788 789 /* Supply MAC with AAD */ 790 if (crp->crp_aad_length != 0) { 791 len = build_ccm_aad_length(crp->crp_aad_length, blk); 792 exf->update(ctx, blk, len); 793 if (crp->crp_aad != NULL) 794 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 795 else 796 crypto_apply(crp, crp->crp_aad_start, 797 crp->crp_aad_length, exf->update, ctx); 798 799 /* Pad the AAD (including length field) to a full block. */ 800 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 801 if (len != 0) { 802 len = CCM_CBC_BLOCK_LEN - len; 803 memset(blk, 0, CCM_CBC_BLOCK_LEN); 804 exf->update(ctx, blk, len); 805 } 806 } 807 808 /* Do encryption/decryption with MAC */ 809 crypto_cursor_init(&cc_in, &crp->crp_buf); 810 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 811 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 812 crypto_cursor_init(&cc_out, &crp->crp_obuf); 813 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 814 } else 815 cc_out = cc_in; 816 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 817 inblk = crypto_cursor_segment(&cc_in, &len); 818 if (len < blksz) { 819 crypto_cursor_copydata(&cc_in, blksz, blk); 820 inblk = blk; 821 } else 822 crypto_cursor_advance(&cc_in, blksz); 823 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 824 outblk = crypto_cursor_segment(&cc_out, &len); 825 if (len < blksz) 826 outblk = blk; 827 exf->update(ctx, inblk, blksz); 828 exf->encrypt(ctx, inblk, outblk); 829 if (outblk == blk) 830 crypto_cursor_copyback(&cc_out, blksz, blk); 831 else 832 crypto_cursor_advance(&cc_out, blksz); 833 } else { 834 /* 835 * One of the problems with CCM+CBC is that 836 * the authentication is done on the 837 * unencrypted data. As a result, we have to 838 * decrypt the data twice: once to generate 839 * the tag and a second time after the tag is 840 * verified. 841 */ 842 exf->decrypt(ctx, inblk, blk); 843 exf->update(ctx, blk, blksz); 844 } 845 } 846 if (resid > 0) { 847 crypto_cursor_copydata(&cc_in, resid, blk); 848 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 849 exf->update(ctx, blk, resid); 850 exf->encrypt_last(ctx, blk, blk, resid); 851 crypto_cursor_copyback(&cc_out, resid, blk); 852 } else { 853 exf->decrypt_last(ctx, blk, blk, resid); 854 exf->update(ctx, blk, resid); 855 } 856 } 857 858 /* Finalize MAC */ 859 exf->final(s.tag, ctx); 860 861 /* Validate tag */ 862 error = 0; 863 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 864 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 865 s.tag2); 866 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 867 if (r != 0) { 868 error = EBADMSG; 869 goto out; 870 } 871 872 /* tag matches, decrypt data */ 873 exf->reinit(ctx, crp->crp_iv, ivlen); 874 crypto_cursor_init(&cc_in, &crp->crp_buf); 875 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 876 for (resid = crp->crp_payload_length; resid > blksz; 877 resid -= blksz) { 878 inblk = crypto_cursor_segment(&cc_in, &len); 879 if (len < blksz) { 880 crypto_cursor_copydata(&cc_in, blksz, blk); 881 inblk = blk; 882 } else 883 crypto_cursor_advance(&cc_in, blksz); 884 outblk = crypto_cursor_segment(&cc_out, &len); 885 if (len < blksz) 886 outblk = blk; 887 exf->decrypt(ctx, inblk, outblk); 888 if (outblk == blk) 889 crypto_cursor_copyback(&cc_out, blksz, blk); 890 else 891 crypto_cursor_advance(&cc_out, blksz); 892 } 893 if (resid > 0) { 894 crypto_cursor_copydata(&cc_in, resid, blk); 895 exf->decrypt_last(ctx, blk, blk, resid); 896 crypto_cursor_copyback(&cc_out, resid, blk); 897 } 898 } else { 899 /* Inject the authentication data */ 900 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 901 s.tag); 902 } 903 904 out: 905 explicit_bzero(ctx, exf->ctxsize); 906 explicit_bzero(&s, sizeof(s)); 907 return (error); 908 } 909 910 static int 911 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 912 { 913 const struct crypto_session_params *csp; 914 struct { 915 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 916 u_char tag[POLY1305_HASH_LEN]; 917 u_char tag2[POLY1305_HASH_LEN]; 918 } s; 919 u_char *blk = (u_char *)s.blkbuf; 920 struct crypto_buffer_cursor cc_in, cc_out; 921 const u_char *inblk; 922 u_char *outblk; 923 uint64_t *blkp; 924 const struct swcr_auth *swa; 925 const struct swcr_encdec *swe; 926 const struct enc_xform *exf; 927 void *ctx; 928 size_t len; 929 int blksz, error, r, resid; 930 931 swa = &ses->swcr_auth; 932 swe = &ses->swcr_encdec; 933 exf = swe->sw_exf; 934 blksz = exf->native_blocksize; 935 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 936 937 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 938 return (EINVAL); 939 940 csp = crypto_get_params(crp->crp_session); 941 942 ctx = __builtin_alloca(exf->ctxsize); 943 if (crp->crp_cipher_key != NULL) 944 exf->setkey(ctx, crp->crp_cipher_key, 945 csp->csp_cipher_klen); 946 else 947 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 948 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 949 950 /* Supply MAC with AAD */ 951 if (crp->crp_aad != NULL) 952 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 953 else 954 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 955 exf->update, ctx); 956 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 957 /* padding1 */ 958 memset(blk, 0, POLY1305_BLOCK_LEN); 959 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 960 crp->crp_aad_length % POLY1305_BLOCK_LEN); 961 } 962 963 /* Do encryption with MAC */ 964 crypto_cursor_init(&cc_in, &crp->crp_buf); 965 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 966 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 967 crypto_cursor_init(&cc_out, &crp->crp_obuf); 968 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 969 } else 970 cc_out = cc_in; 971 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 972 inblk = crypto_cursor_segment(&cc_in, &len); 973 if (len < blksz) { 974 crypto_cursor_copydata(&cc_in, blksz, blk); 975 inblk = blk; 976 } else 977 crypto_cursor_advance(&cc_in, blksz); 978 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 979 outblk = crypto_cursor_segment(&cc_out, &len); 980 if (len < blksz) 981 outblk = blk; 982 exf->encrypt(ctx, inblk, outblk); 983 exf->update(ctx, outblk, blksz); 984 if (outblk == blk) 985 crypto_cursor_copyback(&cc_out, blksz, blk); 986 else 987 crypto_cursor_advance(&cc_out, blksz); 988 } else { 989 exf->update(ctx, inblk, blksz); 990 } 991 } 992 if (resid > 0) { 993 crypto_cursor_copydata(&cc_in, resid, blk); 994 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 995 exf->encrypt_last(ctx, blk, blk, resid); 996 crypto_cursor_copyback(&cc_out, resid, blk); 997 } 998 exf->update(ctx, blk, resid); 999 if (resid % POLY1305_BLOCK_LEN != 0) { 1000 /* padding2 */ 1001 memset(blk, 0, POLY1305_BLOCK_LEN); 1002 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1003 resid % POLY1305_BLOCK_LEN); 1004 } 1005 } 1006 1007 /* lengths */ 1008 blkp = (uint64_t *)blk; 1009 blkp[0] = htole64(crp->crp_aad_length); 1010 blkp[1] = htole64(crp->crp_payload_length); 1011 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1012 1013 /* Finalize MAC */ 1014 exf->final(s.tag, ctx); 1015 1016 /* Validate tag */ 1017 error = 0; 1018 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1019 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 1020 s.tag2); 1021 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1022 if (r != 0) { 1023 error = EBADMSG; 1024 goto out; 1025 } 1026 1027 /* tag matches, decrypt data */ 1028 crypto_cursor_init(&cc_in, &crp->crp_buf); 1029 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1030 for (resid = crp->crp_payload_length; resid > blksz; 1031 resid -= blksz) { 1032 inblk = crypto_cursor_segment(&cc_in, &len); 1033 if (len < blksz) { 1034 crypto_cursor_copydata(&cc_in, blksz, blk); 1035 inblk = blk; 1036 } else 1037 crypto_cursor_advance(&cc_in, blksz); 1038 outblk = crypto_cursor_segment(&cc_out, &len); 1039 if (len < blksz) 1040 outblk = blk; 1041 exf->decrypt(ctx, inblk, outblk); 1042 if (outblk == blk) 1043 crypto_cursor_copyback(&cc_out, blksz, blk); 1044 else 1045 crypto_cursor_advance(&cc_out, blksz); 1046 } 1047 if (resid > 0) { 1048 crypto_cursor_copydata(&cc_in, resid, blk); 1049 exf->decrypt_last(ctx, blk, blk, resid); 1050 crypto_cursor_copyback(&cc_out, resid, blk); 1051 } 1052 } else { 1053 /* Inject the authentication data */ 1054 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1055 s.tag); 1056 } 1057 1058 out: 1059 explicit_bzero(ctx, exf->ctxsize); 1060 explicit_bzero(&s, sizeof(s)); 1061 return (error); 1062 } 1063 1064 /* 1065 * Apply a cipher and a digest to perform EtA. 1066 */ 1067 static int 1068 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1069 { 1070 int error; 1071 1072 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1073 error = swcr_encdec(ses, crp); 1074 if (error == 0) 1075 error = swcr_authcompute(ses, crp); 1076 } else { 1077 error = swcr_authcompute(ses, crp); 1078 if (error == 0) 1079 error = swcr_encdec(ses, crp); 1080 } 1081 return (error); 1082 } 1083 1084 /* 1085 * Apply a compression/decompression algorithm 1086 */ 1087 static int 1088 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1089 { 1090 const struct comp_algo *cxf; 1091 uint8_t *data, *out; 1092 int adj; 1093 uint32_t result; 1094 1095 cxf = ses->swcr_compdec.sw_cxf; 1096 1097 /* We must handle the whole buffer of data in one time 1098 * then if there is not all the data in the mbuf, we must 1099 * copy in a buffer. 1100 */ 1101 1102 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1103 if (data == NULL) 1104 return (EINVAL); 1105 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1106 data); 1107 1108 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1109 result = cxf->compress(data, crp->crp_payload_length, &out); 1110 else 1111 result = cxf->decompress(data, crp->crp_payload_length, &out); 1112 1113 free(data, M_CRYPTO_DATA); 1114 if (result == 0) 1115 return (EINVAL); 1116 crp->crp_olen = result; 1117 1118 /* Check the compressed size when doing compression */ 1119 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1120 if (result >= crp->crp_payload_length) { 1121 /* Compression was useless, we lost time */ 1122 free(out, M_CRYPTO_DATA); 1123 return (0); 1124 } 1125 } 1126 1127 /* Copy back the (de)compressed data. m_copyback is 1128 * extending the mbuf as necessary. 1129 */ 1130 crypto_copyback(crp, crp->crp_payload_start, result, out); 1131 if (result < crp->crp_payload_length) { 1132 switch (crp->crp_buf.cb_type) { 1133 case CRYPTO_BUF_MBUF: 1134 case CRYPTO_BUF_SINGLE_MBUF: 1135 adj = result - crp->crp_payload_length; 1136 m_adj(crp->crp_buf.cb_mbuf, adj); 1137 break; 1138 case CRYPTO_BUF_UIO: { 1139 struct uio *uio = crp->crp_buf.cb_uio; 1140 int ind; 1141 1142 adj = crp->crp_payload_length - result; 1143 ind = uio->uio_iovcnt - 1; 1144 1145 while (adj > 0 && ind >= 0) { 1146 if (adj < uio->uio_iov[ind].iov_len) { 1147 uio->uio_iov[ind].iov_len -= adj; 1148 break; 1149 } 1150 1151 adj -= uio->uio_iov[ind].iov_len; 1152 uio->uio_iov[ind].iov_len = 0; 1153 ind--; 1154 uio->uio_iovcnt--; 1155 } 1156 } 1157 break; 1158 case CRYPTO_BUF_VMPAGE: 1159 adj = crp->crp_payload_length - result; 1160 crp->crp_buf.cb_vm_page_len -= adj; 1161 break; 1162 default: 1163 break; 1164 } 1165 } 1166 free(out, M_CRYPTO_DATA); 1167 return 0; 1168 } 1169 1170 static int 1171 swcr_setup_cipher(struct swcr_session *ses, 1172 const struct crypto_session_params *csp) 1173 { 1174 struct swcr_encdec *swe; 1175 const struct enc_xform *txf; 1176 int error; 1177 1178 swe = &ses->swcr_encdec; 1179 txf = crypto_cipher(csp); 1180 if (csp->csp_cipher_key != NULL) { 1181 if (txf->ctxsize != 0) { 1182 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1183 M_NOWAIT); 1184 if (swe->sw_ctx == NULL) 1185 return (ENOMEM); 1186 } 1187 error = txf->setkey(swe->sw_ctx, 1188 csp->csp_cipher_key, csp->csp_cipher_klen); 1189 if (error) 1190 return (error); 1191 } 1192 swe->sw_exf = txf; 1193 return (0); 1194 } 1195 1196 static int 1197 swcr_setup_auth(struct swcr_session *ses, 1198 const struct crypto_session_params *csp) 1199 { 1200 struct swcr_auth *swa; 1201 const struct auth_hash *axf; 1202 1203 swa = &ses->swcr_auth; 1204 1205 axf = crypto_auth_hash(csp); 1206 swa->sw_axf = axf; 1207 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1208 return (EINVAL); 1209 if (csp->csp_auth_mlen == 0) 1210 swa->sw_mlen = axf->hashsize; 1211 else 1212 swa->sw_mlen = csp->csp_auth_mlen; 1213 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1214 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1215 M_NOWAIT); 1216 if (swa->sw_ictx == NULL) 1217 return (ENOBUFS); 1218 } 1219 1220 switch (csp->csp_auth_alg) { 1221 case CRYPTO_SHA1_HMAC: 1222 case CRYPTO_SHA2_224_HMAC: 1223 case CRYPTO_SHA2_256_HMAC: 1224 case CRYPTO_SHA2_384_HMAC: 1225 case CRYPTO_SHA2_512_HMAC: 1226 case CRYPTO_RIPEMD160_HMAC: 1227 swa->sw_hmac = true; 1228 if (csp->csp_auth_key != NULL) { 1229 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1230 M_NOWAIT); 1231 if (swa->sw_octx == NULL) 1232 return (ENOBUFS); 1233 hmac_init_ipad(axf, csp->csp_auth_key, 1234 csp->csp_auth_klen, swa->sw_ictx); 1235 hmac_init_opad(axf, csp->csp_auth_key, 1236 csp->csp_auth_klen, swa->sw_octx); 1237 } 1238 break; 1239 case CRYPTO_RIPEMD160: 1240 case CRYPTO_SHA1: 1241 case CRYPTO_SHA2_224: 1242 case CRYPTO_SHA2_256: 1243 case CRYPTO_SHA2_384: 1244 case CRYPTO_SHA2_512: 1245 case CRYPTO_NULL_HMAC: 1246 axf->Init(swa->sw_ictx); 1247 break; 1248 case CRYPTO_AES_NIST_GMAC: 1249 case CRYPTO_AES_CCM_CBC_MAC: 1250 case CRYPTO_POLY1305: 1251 if (csp->csp_auth_key != NULL) { 1252 axf->Init(swa->sw_ictx); 1253 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1254 csp->csp_auth_klen); 1255 } 1256 break; 1257 case CRYPTO_BLAKE2B: 1258 case CRYPTO_BLAKE2S: 1259 /* 1260 * Blake2b and Blake2s support an optional key but do 1261 * not require one. 1262 */ 1263 if (csp->csp_auth_klen == 0) 1264 axf->Init(swa->sw_ictx); 1265 else if (csp->csp_auth_key != NULL) 1266 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1267 csp->csp_auth_klen); 1268 break; 1269 } 1270 1271 if (csp->csp_mode == CSP_MODE_DIGEST) { 1272 switch (csp->csp_auth_alg) { 1273 case CRYPTO_AES_NIST_GMAC: 1274 ses->swcr_process = swcr_gmac; 1275 break; 1276 case CRYPTO_AES_CCM_CBC_MAC: 1277 ses->swcr_process = swcr_ccm_cbc_mac; 1278 break; 1279 default: 1280 ses->swcr_process = swcr_authcompute; 1281 } 1282 } 1283 1284 return (0); 1285 } 1286 1287 static int 1288 swcr_setup_aead(struct swcr_session *ses, 1289 const struct crypto_session_params *csp) 1290 { 1291 struct swcr_auth *swa; 1292 int error; 1293 1294 error = swcr_setup_cipher(ses, csp); 1295 if (error) 1296 return (error); 1297 1298 swa = &ses->swcr_auth; 1299 if (csp->csp_auth_mlen == 0) 1300 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1301 else 1302 swa->sw_mlen = csp->csp_auth_mlen; 1303 return (0); 1304 } 1305 1306 static bool 1307 swcr_auth_supported(const struct crypto_session_params *csp) 1308 { 1309 const struct auth_hash *axf; 1310 1311 axf = crypto_auth_hash(csp); 1312 if (axf == NULL) 1313 return (false); 1314 switch (csp->csp_auth_alg) { 1315 case CRYPTO_SHA1_HMAC: 1316 case CRYPTO_SHA2_224_HMAC: 1317 case CRYPTO_SHA2_256_HMAC: 1318 case CRYPTO_SHA2_384_HMAC: 1319 case CRYPTO_SHA2_512_HMAC: 1320 case CRYPTO_NULL_HMAC: 1321 case CRYPTO_RIPEMD160_HMAC: 1322 break; 1323 case CRYPTO_AES_NIST_GMAC: 1324 switch (csp->csp_auth_klen * 8) { 1325 case 128: 1326 case 192: 1327 case 256: 1328 break; 1329 default: 1330 return (false); 1331 } 1332 if (csp->csp_auth_key == NULL) 1333 return (false); 1334 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1335 return (false); 1336 break; 1337 case CRYPTO_POLY1305: 1338 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1339 return (false); 1340 break; 1341 case CRYPTO_AES_CCM_CBC_MAC: 1342 switch (csp->csp_auth_klen * 8) { 1343 case 128: 1344 case 192: 1345 case 256: 1346 break; 1347 default: 1348 return (false); 1349 } 1350 if (csp->csp_auth_key == NULL) 1351 return (false); 1352 break; 1353 } 1354 return (true); 1355 } 1356 1357 static bool 1358 swcr_cipher_supported(const struct crypto_session_params *csp) 1359 { 1360 const struct enc_xform *txf; 1361 1362 txf = crypto_cipher(csp); 1363 if (txf == NULL) 1364 return (false); 1365 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1366 txf->ivsize != csp->csp_ivlen) 1367 return (false); 1368 return (true); 1369 } 1370 1371 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1372 1373 static int 1374 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1375 { 1376 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1377 return (EINVAL); 1378 switch (csp->csp_mode) { 1379 case CSP_MODE_COMPRESS: 1380 switch (csp->csp_cipher_alg) { 1381 case CRYPTO_DEFLATE_COMP: 1382 break; 1383 default: 1384 return (EINVAL); 1385 } 1386 break; 1387 case CSP_MODE_CIPHER: 1388 switch (csp->csp_cipher_alg) { 1389 case CRYPTO_AES_NIST_GCM_16: 1390 case CRYPTO_AES_CCM_16: 1391 case CRYPTO_CHACHA20_POLY1305: 1392 case CRYPTO_XCHACHA20_POLY1305: 1393 return (EINVAL); 1394 default: 1395 if (!swcr_cipher_supported(csp)) 1396 return (EINVAL); 1397 break; 1398 } 1399 break; 1400 case CSP_MODE_DIGEST: 1401 if (!swcr_auth_supported(csp)) 1402 return (EINVAL); 1403 break; 1404 case CSP_MODE_AEAD: 1405 switch (csp->csp_cipher_alg) { 1406 case CRYPTO_AES_NIST_GCM_16: 1407 case CRYPTO_AES_CCM_16: 1408 switch (csp->csp_cipher_klen * 8) { 1409 case 128: 1410 case 192: 1411 case 256: 1412 break; 1413 default: 1414 return (EINVAL); 1415 } 1416 break; 1417 case CRYPTO_CHACHA20_POLY1305: 1418 case CRYPTO_XCHACHA20_POLY1305: 1419 break; 1420 default: 1421 return (EINVAL); 1422 } 1423 break; 1424 case CSP_MODE_ETA: 1425 /* AEAD algorithms cannot be used for EtA. */ 1426 switch (csp->csp_cipher_alg) { 1427 case CRYPTO_AES_NIST_GCM_16: 1428 case CRYPTO_AES_CCM_16: 1429 case CRYPTO_CHACHA20_POLY1305: 1430 case CRYPTO_XCHACHA20_POLY1305: 1431 return (EINVAL); 1432 } 1433 switch (csp->csp_auth_alg) { 1434 case CRYPTO_AES_NIST_GMAC: 1435 case CRYPTO_AES_CCM_CBC_MAC: 1436 return (EINVAL); 1437 } 1438 1439 if (!swcr_cipher_supported(csp) || 1440 !swcr_auth_supported(csp)) 1441 return (EINVAL); 1442 break; 1443 default: 1444 return (EINVAL); 1445 } 1446 1447 return (CRYPTODEV_PROBE_SOFTWARE); 1448 } 1449 1450 /* 1451 * Generate a new software session. 1452 */ 1453 static int 1454 swcr_newsession(device_t dev, crypto_session_t cses, 1455 const struct crypto_session_params *csp) 1456 { 1457 struct swcr_session *ses; 1458 const struct comp_algo *cxf; 1459 int error; 1460 1461 ses = crypto_get_driver_session(cses); 1462 1463 error = 0; 1464 switch (csp->csp_mode) { 1465 case CSP_MODE_COMPRESS: 1466 switch (csp->csp_cipher_alg) { 1467 case CRYPTO_DEFLATE_COMP: 1468 cxf = &comp_algo_deflate; 1469 break; 1470 #ifdef INVARIANTS 1471 default: 1472 panic("bad compression algo"); 1473 #endif 1474 } 1475 ses->swcr_compdec.sw_cxf = cxf; 1476 ses->swcr_process = swcr_compdec; 1477 break; 1478 case CSP_MODE_CIPHER: 1479 switch (csp->csp_cipher_alg) { 1480 case CRYPTO_NULL_CBC: 1481 ses->swcr_process = swcr_null; 1482 break; 1483 #ifdef INVARIANTS 1484 case CRYPTO_AES_NIST_GCM_16: 1485 case CRYPTO_AES_CCM_16: 1486 case CRYPTO_CHACHA20_POLY1305: 1487 case CRYPTO_XCHACHA20_POLY1305: 1488 panic("bad cipher algo"); 1489 #endif 1490 default: 1491 error = swcr_setup_cipher(ses, csp); 1492 if (error == 0) 1493 ses->swcr_process = swcr_encdec; 1494 } 1495 break; 1496 case CSP_MODE_DIGEST: 1497 error = swcr_setup_auth(ses, csp); 1498 break; 1499 case CSP_MODE_AEAD: 1500 switch (csp->csp_cipher_alg) { 1501 case CRYPTO_AES_NIST_GCM_16: 1502 error = swcr_setup_aead(ses, csp); 1503 if (error == 0) 1504 ses->swcr_process = swcr_gcm; 1505 break; 1506 case CRYPTO_AES_CCM_16: 1507 error = swcr_setup_aead(ses, csp); 1508 if (error == 0) 1509 ses->swcr_process = swcr_ccm; 1510 break; 1511 case CRYPTO_CHACHA20_POLY1305: 1512 case CRYPTO_XCHACHA20_POLY1305: 1513 error = swcr_setup_aead(ses, csp); 1514 if (error == 0) 1515 ses->swcr_process = swcr_chacha20_poly1305; 1516 break; 1517 #ifdef INVARIANTS 1518 default: 1519 panic("bad aead algo"); 1520 #endif 1521 } 1522 break; 1523 case CSP_MODE_ETA: 1524 #ifdef INVARIANTS 1525 switch (csp->csp_cipher_alg) { 1526 case CRYPTO_AES_NIST_GCM_16: 1527 case CRYPTO_AES_CCM_16: 1528 case CRYPTO_CHACHA20_POLY1305: 1529 case CRYPTO_XCHACHA20_POLY1305: 1530 panic("bad eta cipher algo"); 1531 } 1532 switch (csp->csp_auth_alg) { 1533 case CRYPTO_AES_NIST_GMAC: 1534 case CRYPTO_AES_CCM_CBC_MAC: 1535 panic("bad eta auth algo"); 1536 } 1537 #endif 1538 1539 error = swcr_setup_auth(ses, csp); 1540 if (error) 1541 break; 1542 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1543 /* Effectively degrade to digest mode. */ 1544 ses->swcr_process = swcr_authcompute; 1545 break; 1546 } 1547 1548 error = swcr_setup_cipher(ses, csp); 1549 if (error == 0) 1550 ses->swcr_process = swcr_eta; 1551 break; 1552 default: 1553 error = EINVAL; 1554 } 1555 1556 if (error) 1557 swcr_freesession(dev, cses); 1558 return (error); 1559 } 1560 1561 static void 1562 swcr_freesession(device_t dev, crypto_session_t cses) 1563 { 1564 struct swcr_session *ses; 1565 1566 ses = crypto_get_driver_session(cses); 1567 1568 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1569 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1570 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1571 } 1572 1573 /* 1574 * Process a software request. 1575 */ 1576 static int 1577 swcr_process(device_t dev, struct cryptop *crp, int hint) 1578 { 1579 struct swcr_session *ses; 1580 1581 ses = crypto_get_driver_session(crp->crp_session); 1582 1583 crp->crp_etype = ses->swcr_process(ses, crp); 1584 1585 crypto_done(crp); 1586 return (0); 1587 } 1588 1589 static void 1590 swcr_identify(driver_t *drv, device_t parent) 1591 { 1592 /* NB: order 10 is so we get attached after h/w devices */ 1593 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1594 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1595 panic("cryptosoft: could not attach"); 1596 } 1597 1598 static int 1599 swcr_probe(device_t dev) 1600 { 1601 device_set_desc(dev, "software crypto"); 1602 device_quiet(dev); 1603 return (BUS_PROBE_NOWILDCARD); 1604 } 1605 1606 static int 1607 swcr_attach(device_t dev) 1608 { 1609 1610 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1611 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1612 if (swcr_id < 0) { 1613 device_printf(dev, "cannot initialize!"); 1614 return (ENXIO); 1615 } 1616 1617 return (0); 1618 } 1619 1620 static int 1621 swcr_detach(device_t dev) 1622 { 1623 crypto_unregister_all(swcr_id); 1624 return 0; 1625 } 1626 1627 static device_method_t swcr_methods[] = { 1628 DEVMETHOD(device_identify, swcr_identify), 1629 DEVMETHOD(device_probe, swcr_probe), 1630 DEVMETHOD(device_attach, swcr_attach), 1631 DEVMETHOD(device_detach, swcr_detach), 1632 1633 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1634 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1635 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1636 DEVMETHOD(cryptodev_process, swcr_process), 1637 1638 {0, 0}, 1639 }; 1640 1641 static driver_t swcr_driver = { 1642 "cryptosoft", 1643 swcr_methods, 1644 0, /* NB: no softc */ 1645 }; 1646 static devclass_t swcr_devclass; 1647 1648 /* 1649 * NB: We explicitly reference the crypto module so we 1650 * get the necessary ordering when built as a loadable 1651 * module. This is required because we bundle the crypto 1652 * module code together with the cryptosoft driver (otherwise 1653 * normal module dependencies would handle things). 1654 */ 1655 extern int crypto_modevent(struct module *, int, void *); 1656 /* XXX where to attach */ 1657 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1658 MODULE_VERSION(cryptosoft, 1); 1659 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1660