1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen, todo; 109 int blksz, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blksz = exf->blocksize; 128 } else 129 blksz = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 158 159 /* 160 * Loop through encrypting blocks. 'inlen' is the remaining 161 * length of the current segment in the input buffer. 162 * 'outlen' is the remaining length of current segment in the 163 * output buffer. 164 */ 165 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 166 /* 167 * If the current block is not contained within the 168 * current input/output segment, use 'blk' as a local 169 * buffer. 170 */ 171 if (inlen < blksz) { 172 crypto_cursor_copydata(&cc_in, blksz, blk); 173 inblk = blk; 174 inlen = blksz; 175 } 176 if (outlen < blksz) { 177 outblk = blk; 178 outlen = blksz; 179 } 180 181 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 182 183 if (encrypting) 184 exf->encrypt_multi(ctx, inblk, outblk, todo); 185 else 186 exf->decrypt_multi(ctx, inblk, outblk, todo); 187 188 if (inblk == blk) { 189 inblk = crypto_cursor_segment(&cc_in, &inlen); 190 } else { 191 crypto_cursor_advance(&cc_in, todo); 192 inlen -= todo; 193 inblk += todo; 194 if (inlen == 0) 195 inblk = crypto_cursor_segment(&cc_in, &inlen); 196 } 197 198 if (outblk == blk) { 199 crypto_cursor_copyback(&cc_out, blksz, blk); 200 outblk = crypto_cursor_segment(&cc_out, &outlen); 201 } else { 202 crypto_cursor_advance(&cc_out, todo); 203 outlen -= todo; 204 outblk += todo; 205 if (outlen == 0) 206 outblk = crypto_cursor_segment(&cc_out, 207 &outlen); 208 } 209 } 210 211 /* Handle trailing partial block for stream ciphers. */ 212 if (resid > 0) { 213 KASSERT(exf->native_blocksize != 0, 214 ("%s: partial block of %d bytes for cipher %s", 215 __func__, resid, exf->name)); 216 KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 217 218 inblk = crypto_cursor_segment(&cc_in, &inlen); 219 outblk = crypto_cursor_segment(&cc_out, &outlen); 220 if (inlen < resid) { 221 crypto_cursor_copydata(&cc_in, resid, blk); 222 inblk = blk; 223 } 224 if (outlen < resid) 225 outblk = blk; 226 if (encrypting) 227 exf->encrypt_last(ctx, inblk, outblk, 228 resid); 229 else 230 exf->decrypt_last(ctx, inblk, outblk, 231 resid); 232 if (outlen < resid) 233 crypto_cursor_copyback(&cc_out, resid, blk); 234 } 235 236 explicit_bzero(ctx, exf->ctxsize); 237 explicit_bzero(blk, sizeof(blk)); 238 return (0); 239 } 240 241 /* 242 * Compute or verify hash. 243 */ 244 static int 245 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 246 { 247 struct { 248 union authctx ctx; 249 u_char aalg[HASH_MAX_LEN]; 250 u_char uaalg[HASH_MAX_LEN]; 251 } s; 252 const struct crypto_session_params *csp; 253 const struct swcr_auth *sw; 254 const struct auth_hash *axf; 255 int err; 256 257 sw = &ses->swcr_auth; 258 259 axf = sw->sw_axf; 260 261 csp = crypto_get_params(crp->crp_session); 262 if (crp->crp_auth_key != NULL) { 263 if (sw->sw_hmac) { 264 hmac_init_ipad(axf, crp->crp_auth_key, 265 csp->csp_auth_klen, &s.ctx); 266 } else { 267 axf->Init(&s.ctx); 268 axf->Setkey(&s.ctx, crp->crp_auth_key, 269 csp->csp_auth_klen); 270 } 271 } else 272 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 273 274 if (crp->crp_aad != NULL) 275 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 276 else 277 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 278 axf->Update, &s.ctx); 279 if (err) 280 goto out; 281 282 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 283 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 284 err = crypto_apply_buf(&crp->crp_obuf, 285 crp->crp_payload_output_start, crp->crp_payload_length, 286 axf->Update, &s.ctx); 287 else 288 err = crypto_apply(crp, crp->crp_payload_start, 289 crp->crp_payload_length, axf->Update, &s.ctx); 290 if (err) 291 goto out; 292 293 if (csp->csp_flags & CSP_F_ESN) 294 axf->Update(&s.ctx, crp->crp_esn, 4); 295 296 axf->Final(s.aalg, &s.ctx); 297 if (sw->sw_hmac) { 298 if (crp->crp_auth_key != NULL) 299 hmac_init_opad(axf, crp->crp_auth_key, 300 csp->csp_auth_klen, &s.ctx); 301 else 302 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 303 axf->Update(&s.ctx, s.aalg, axf->hashsize); 304 axf->Final(s.aalg, &s.ctx); 305 } 306 307 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 308 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 309 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 310 err = EBADMSG; 311 } else { 312 /* Inject the authentication data */ 313 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 314 } 315 out: 316 explicit_bzero(&s, sizeof(s)); 317 return (err); 318 } 319 320 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 321 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 322 323 static int 324 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 325 { 326 struct { 327 union authctx ctx; 328 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 329 u_char tag[GMAC_DIGEST_LEN]; 330 u_char tag2[GMAC_DIGEST_LEN]; 331 } s; 332 u_char *blk = (u_char *)s.blkbuf; 333 struct crypto_buffer_cursor cc; 334 const u_char *inblk; 335 const struct swcr_auth *swa; 336 const struct auth_hash *axf; 337 uint32_t *blkp; 338 size_t len; 339 int blksz, error, ivlen, resid; 340 341 swa = &ses->swcr_auth; 342 axf = swa->sw_axf; 343 blksz = GMAC_BLOCK_LEN; 344 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 345 __func__)); 346 347 if (crp->crp_auth_key != NULL) { 348 axf->Init(&s.ctx); 349 axf->Setkey(&s.ctx, crp->crp_auth_key, 350 crypto_get_params(crp->crp_session)->csp_auth_klen); 351 } else 352 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 353 354 /* Initialize the IV */ 355 ivlen = AES_GCM_IV_LEN; 356 crypto_read_iv(crp, blk); 357 358 axf->Reinit(&s.ctx, blk, ivlen); 359 crypto_cursor_init(&cc, &crp->crp_buf); 360 crypto_cursor_advance(&cc, crp->crp_payload_start); 361 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 362 inblk = crypto_cursor_segment(&cc, &len); 363 if (len >= blksz) { 364 len = rounddown(MIN(len, resid), blksz); 365 crypto_cursor_advance(&cc, len); 366 } else { 367 len = blksz; 368 crypto_cursor_copydata(&cc, len, blk); 369 inblk = blk; 370 } 371 axf->Update(&s.ctx, inblk, len); 372 } 373 if (resid > 0) { 374 memset(blk, 0, blksz); 375 crypto_cursor_copydata(&cc, resid, blk); 376 axf->Update(&s.ctx, blk, blksz); 377 } 378 379 /* length block */ 380 memset(blk, 0, blksz); 381 blkp = (uint32_t *)blk + 1; 382 *blkp = htobe32(crp->crp_payload_length * 8); 383 axf->Update(&s.ctx, blk, blksz); 384 385 /* Finalize MAC */ 386 axf->Final(s.tag, &s.ctx); 387 388 error = 0; 389 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 390 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 391 s.tag2); 392 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 393 error = EBADMSG; 394 } else { 395 /* Inject the authentication data */ 396 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 397 } 398 explicit_bzero(&s, sizeof(s)); 399 return (error); 400 } 401 402 static int 403 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 404 { 405 struct { 406 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 407 u_char tag[GMAC_DIGEST_LEN]; 408 u_char tag2[GMAC_DIGEST_LEN]; 409 } s; 410 u_char *blk = (u_char *)s.blkbuf; 411 struct crypto_buffer_cursor cc_in, cc_out; 412 const u_char *inblk; 413 u_char *outblk; 414 size_t inlen, outlen, todo; 415 const struct swcr_auth *swa; 416 const struct swcr_encdec *swe; 417 const struct enc_xform *exf; 418 void *ctx; 419 uint32_t *blkp; 420 int blksz, error, ivlen, r, resid; 421 422 swa = &ses->swcr_auth; 423 swe = &ses->swcr_encdec; 424 exf = swe->sw_exf; 425 blksz = GMAC_BLOCK_LEN; 426 KASSERT(blksz == exf->native_blocksize, 427 ("%s: blocksize mismatch", __func__)); 428 429 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 430 return (EINVAL); 431 432 ivlen = AES_GCM_IV_LEN; 433 434 ctx = __builtin_alloca(exf->ctxsize); 435 if (crp->crp_cipher_key != NULL) 436 exf->setkey(ctx, crp->crp_cipher_key, 437 crypto_get_params(crp->crp_session)->csp_cipher_klen); 438 else 439 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 440 exf->reinit(ctx, crp->crp_iv, ivlen); 441 442 /* Supply MAC with AAD */ 443 if (crp->crp_aad != NULL) { 444 inlen = rounddown2(crp->crp_aad_length, blksz); 445 if (inlen != 0) 446 exf->update(ctx, crp->crp_aad, inlen); 447 if (crp->crp_aad_length != inlen) { 448 memset(blk, 0, blksz); 449 memcpy(blk, (char *)crp->crp_aad + inlen, 450 crp->crp_aad_length - inlen); 451 exf->update(ctx, blk, blksz); 452 } 453 } else { 454 crypto_cursor_init(&cc_in, &crp->crp_buf); 455 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 456 for (resid = crp->crp_aad_length; resid >= blksz; 457 resid -= inlen) { 458 inblk = crypto_cursor_segment(&cc_in, &inlen); 459 if (inlen >= blksz) { 460 inlen = rounddown2(MIN(inlen, resid), blksz); 461 crypto_cursor_advance(&cc_in, inlen); 462 } else { 463 inlen = blksz; 464 crypto_cursor_copydata(&cc_in, inlen, blk); 465 inblk = blk; 466 } 467 exf->update(ctx, inblk, inlen); 468 } 469 if (resid > 0) { 470 memset(blk, 0, blksz); 471 crypto_cursor_copydata(&cc_in, resid, blk); 472 exf->update(ctx, blk, blksz); 473 } 474 } 475 476 /* Do encryption with MAC */ 477 crypto_cursor_init(&cc_in, &crp->crp_buf); 478 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 479 inblk = crypto_cursor_segment(&cc_in, &inlen); 480 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 481 crypto_cursor_init(&cc_out, &crp->crp_obuf); 482 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 483 } else 484 cc_out = cc_in; 485 outblk = crypto_cursor_segment(&cc_out, &outlen); 486 487 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 488 if (inlen < blksz) { 489 crypto_cursor_copydata(&cc_in, blksz, blk); 490 inblk = blk; 491 inlen = blksz; 492 } 493 494 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 495 if (outlen < blksz) { 496 outblk = blk; 497 outlen = blksz; 498 } 499 500 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 501 blksz); 502 503 exf->encrypt_multi(ctx, inblk, outblk, todo); 504 exf->update(ctx, outblk, todo); 505 506 if (outblk == blk) { 507 crypto_cursor_copyback(&cc_out, blksz, blk); 508 outblk = crypto_cursor_segment(&cc_out, &outlen); 509 } else { 510 crypto_cursor_advance(&cc_out, todo); 511 outlen -= todo; 512 outblk += todo; 513 if (outlen == 0) 514 outblk = crypto_cursor_segment(&cc_out, 515 &outlen); 516 } 517 } else { 518 todo = rounddown2(MIN(resid, inlen), blksz); 519 exf->update(ctx, inblk, todo); 520 } 521 522 if (inblk == blk) { 523 inblk = crypto_cursor_segment(&cc_in, &inlen); 524 } else { 525 crypto_cursor_advance(&cc_in, todo); 526 inlen -= todo; 527 inblk += todo; 528 if (inlen == 0) 529 inblk = crypto_cursor_segment(&cc_in, &inlen); 530 } 531 } 532 if (resid > 0) { 533 crypto_cursor_copydata(&cc_in, resid, blk); 534 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 535 exf->encrypt_last(ctx, blk, blk, resid); 536 crypto_cursor_copyback(&cc_out, resid, blk); 537 } 538 exf->update(ctx, blk, resid); 539 } 540 541 /* length block */ 542 memset(blk, 0, blksz); 543 blkp = (uint32_t *)blk + 1; 544 *blkp = htobe32(crp->crp_aad_length * 8); 545 blkp = (uint32_t *)blk + 3; 546 *blkp = htobe32(crp->crp_payload_length * 8); 547 exf->update(ctx, blk, blksz); 548 549 /* Finalize MAC */ 550 exf->final(s.tag, ctx); 551 552 /* Validate tag */ 553 error = 0; 554 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 555 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 556 s.tag2); 557 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 558 if (r != 0) { 559 error = EBADMSG; 560 goto out; 561 } 562 563 /* tag matches, decrypt data */ 564 crypto_cursor_init(&cc_in, &crp->crp_buf); 565 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 566 inblk = crypto_cursor_segment(&cc_in, &inlen); 567 568 for (resid = crp->crp_payload_length; resid > blksz; 569 resid -= todo) { 570 if (inlen < blksz) { 571 crypto_cursor_copydata(&cc_in, blksz, blk); 572 inblk = blk; 573 inlen = blksz; 574 } 575 if (outlen < blksz) { 576 outblk = blk; 577 outlen = blksz; 578 } 579 580 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 581 blksz); 582 583 exf->decrypt_multi(ctx, inblk, outblk, todo); 584 585 if (inblk == blk) { 586 inblk = crypto_cursor_segment(&cc_in, &inlen); 587 } else { 588 crypto_cursor_advance(&cc_in, todo); 589 inlen -= todo; 590 inblk += todo; 591 if (inlen == 0) 592 inblk = crypto_cursor_segment(&cc_in, 593 &inlen); 594 } 595 596 if (outblk == blk) { 597 crypto_cursor_copyback(&cc_out, blksz, blk); 598 outblk = crypto_cursor_segment(&cc_out, 599 &outlen); 600 } else { 601 crypto_cursor_advance(&cc_out, todo); 602 outlen -= todo; 603 outblk += todo; 604 if (outlen == 0) 605 outblk = crypto_cursor_segment(&cc_out, 606 &outlen); 607 } 608 } 609 if (resid > 0) { 610 crypto_cursor_copydata(&cc_in, resid, blk); 611 exf->decrypt_last(ctx, blk, blk, resid); 612 crypto_cursor_copyback(&cc_out, resid, blk); 613 } 614 } else { 615 /* Inject the authentication data */ 616 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 617 s.tag); 618 } 619 620 out: 621 explicit_bzero(ctx, exf->ctxsize); 622 explicit_bzero(&s, sizeof(s)); 623 624 return (error); 625 } 626 627 static void 628 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 629 u_int data_length, u_int tag_length, uint8_t *b0) 630 { 631 uint8_t *bp; 632 uint8_t flags, L; 633 634 KASSERT(nonce_length >= 7 && nonce_length <= 13, 635 ("nonce_length must be between 7 and 13 bytes")); 636 637 /* 638 * Need to determine the L field value. This is the number of 639 * bytes needed to specify the length of the message; the length 640 * is whatever is left in the 16 bytes after specifying flags and 641 * the nonce. 642 */ 643 L = 15 - nonce_length; 644 645 flags = ((aad_length > 0) << 6) + 646 (((tag_length - 2) / 2) << 3) + 647 L - 1; 648 649 /* 650 * Now we need to set up the first block, which has flags, nonce, 651 * and the message length. 652 */ 653 b0[0] = flags; 654 memcpy(b0 + 1, nonce, nonce_length); 655 bp = b0 + 1 + nonce_length; 656 657 /* Need to copy L' [aka L-1] bytes of data_length */ 658 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 659 *dst = data_length; 660 data_length >>= 8; 661 } 662 } 663 664 /* NB: OCF only supports AAD lengths < 2^32. */ 665 static int 666 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 667 { 668 if (aad_length < ((1 << 16) - (1 << 8))) { 669 be16enc(blk, aad_length); 670 return (sizeof(uint16_t)); 671 } else { 672 blk[0] = 0xff; 673 blk[1] = 0xfe; 674 be32enc(blk + 2, aad_length); 675 return (2 + sizeof(uint32_t)); 676 } 677 } 678 679 static int 680 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 681 { 682 struct { 683 union authctx ctx; 684 u_char blk[CCM_CBC_BLOCK_LEN]; 685 u_char tag[AES_CBC_MAC_HASH_LEN]; 686 u_char tag2[AES_CBC_MAC_HASH_LEN]; 687 } s; 688 const struct crypto_session_params *csp; 689 const struct swcr_auth *swa; 690 const struct auth_hash *axf; 691 int error, ivlen, len; 692 693 csp = crypto_get_params(crp->crp_session); 694 swa = &ses->swcr_auth; 695 axf = swa->sw_axf; 696 697 if (crp->crp_auth_key != NULL) { 698 axf->Init(&s.ctx); 699 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 700 } else 701 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 702 703 /* Initialize the IV */ 704 ivlen = csp->csp_ivlen; 705 706 /* Supply MAC with IV */ 707 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 708 709 /* Supply MAC with b0. */ 710 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 711 swa->sw_mlen, s.blk); 712 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 713 714 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 715 axf->Update(&s.ctx, s.blk, len); 716 717 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 718 axf->Update, &s.ctx); 719 720 /* Finalize MAC */ 721 axf->Final(s.tag, &s.ctx); 722 723 error = 0; 724 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 725 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 726 s.tag2); 727 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 728 error = EBADMSG; 729 } else { 730 /* Inject the authentication data */ 731 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 732 s.tag); 733 } 734 explicit_bzero(&s, sizeof(s)); 735 return (error); 736 } 737 738 static int 739 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 740 { 741 const struct crypto_session_params *csp; 742 struct { 743 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 744 u_char tag[AES_CBC_MAC_HASH_LEN]; 745 u_char tag2[AES_CBC_MAC_HASH_LEN]; 746 } s; 747 u_char *blk = (u_char *)s.blkbuf; 748 struct crypto_buffer_cursor cc_in, cc_out; 749 const u_char *inblk; 750 u_char *outblk; 751 size_t inlen, outlen, todo; 752 const struct swcr_auth *swa; 753 const struct swcr_encdec *swe; 754 const struct enc_xform *exf; 755 void *ctx; 756 size_t len; 757 int blksz, error, ivlen, r, resid; 758 759 csp = crypto_get_params(crp->crp_session); 760 swa = &ses->swcr_auth; 761 swe = &ses->swcr_encdec; 762 exf = swe->sw_exf; 763 blksz = AES_BLOCK_LEN; 764 KASSERT(blksz == exf->native_blocksize, 765 ("%s: blocksize mismatch", __func__)); 766 767 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 768 return (EMSGSIZE); 769 770 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 771 return (EINVAL); 772 773 ivlen = csp->csp_ivlen; 774 775 ctx = __builtin_alloca(exf->ctxsize); 776 if (crp->crp_cipher_key != NULL) 777 exf->setkey(ctx, crp->crp_cipher_key, 778 crypto_get_params(crp->crp_session)->csp_cipher_klen); 779 else 780 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 781 exf->reinit(ctx, crp->crp_iv, ivlen); 782 783 /* Supply MAC with b0. */ 784 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 785 "blkbuf too small for b0"); 786 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 787 crp->crp_payload_length, swa->sw_mlen, blk); 788 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 789 790 /* Supply MAC with AAD */ 791 if (crp->crp_aad_length != 0) { 792 len = build_ccm_aad_length(crp->crp_aad_length, blk); 793 exf->update(ctx, blk, len); 794 if (crp->crp_aad != NULL) 795 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 796 else 797 crypto_apply(crp, crp->crp_aad_start, 798 crp->crp_aad_length, exf->update, ctx); 799 800 /* Pad the AAD (including length field) to a full block. */ 801 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 802 if (len != 0) { 803 len = CCM_CBC_BLOCK_LEN - len; 804 memset(blk, 0, CCM_CBC_BLOCK_LEN); 805 exf->update(ctx, blk, len); 806 } 807 } 808 809 /* Do encryption/decryption with MAC */ 810 crypto_cursor_init(&cc_in, &crp->crp_buf); 811 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 812 inblk = crypto_cursor_segment(&cc_in, &inlen); 813 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 814 crypto_cursor_init(&cc_out, &crp->crp_obuf); 815 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 816 } else 817 cc_out = cc_in; 818 outblk = crypto_cursor_segment(&cc_out, &outlen); 819 820 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 821 if (inlen < blksz) { 822 crypto_cursor_copydata(&cc_in, blksz, blk); 823 inblk = blk; 824 inlen = blksz; 825 } 826 827 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 828 if (outlen < blksz) { 829 outblk = blk; 830 outlen = blksz; 831 } 832 833 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 834 blksz); 835 836 exf->update(ctx, inblk, todo); 837 exf->encrypt_multi(ctx, inblk, outblk, todo); 838 839 if (outblk == blk) { 840 crypto_cursor_copyback(&cc_out, blksz, blk); 841 outblk = crypto_cursor_segment(&cc_out, &outlen); 842 } else { 843 crypto_cursor_advance(&cc_out, todo); 844 outlen -= todo; 845 outblk += todo; 846 if (outlen == 0) 847 outblk = crypto_cursor_segment(&cc_out, 848 &outlen); 849 } 850 } else { 851 /* 852 * One of the problems with CCM+CBC is that 853 * the authentication is done on the 854 * unencrypted data. As a result, we have to 855 * decrypt the data twice: once to generate 856 * the tag and a second time after the tag is 857 * verified. 858 */ 859 todo = blksz; 860 exf->decrypt(ctx, inblk, blk); 861 exf->update(ctx, blk, todo); 862 } 863 864 if (inblk == blk) { 865 inblk = crypto_cursor_segment(&cc_in, &inlen); 866 } else { 867 crypto_cursor_advance(&cc_in, todo); 868 inlen -= todo; 869 inblk += todo; 870 if (inlen == 0) 871 inblk = crypto_cursor_segment(&cc_in, &inlen); 872 } 873 } 874 if (resid > 0) { 875 crypto_cursor_copydata(&cc_in, resid, blk); 876 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 877 exf->update(ctx, blk, resid); 878 exf->encrypt_last(ctx, blk, blk, resid); 879 crypto_cursor_copyback(&cc_out, resid, blk); 880 } else { 881 exf->decrypt_last(ctx, blk, blk, resid); 882 exf->update(ctx, blk, resid); 883 } 884 } 885 886 /* Finalize MAC */ 887 exf->final(s.tag, ctx); 888 889 /* Validate tag */ 890 error = 0; 891 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 892 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 893 s.tag2); 894 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 895 if (r != 0) { 896 error = EBADMSG; 897 goto out; 898 } 899 900 /* tag matches, decrypt data */ 901 exf->reinit(ctx, crp->crp_iv, ivlen); 902 crypto_cursor_init(&cc_in, &crp->crp_buf); 903 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 904 inblk = crypto_cursor_segment(&cc_in, &inlen); 905 906 for (resid = crp->crp_payload_length; resid >= blksz; 907 resid -= todo) { 908 if (inlen < blksz) { 909 crypto_cursor_copydata(&cc_in, blksz, blk); 910 inblk = blk; 911 inlen = blksz; 912 } 913 if (outlen < blksz) { 914 outblk = blk; 915 outlen = blksz; 916 } 917 918 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 919 blksz); 920 921 exf->decrypt_multi(ctx, inblk, outblk, todo); 922 923 if (inblk == blk) { 924 inblk = crypto_cursor_segment(&cc_in, &inlen); 925 } else { 926 crypto_cursor_advance(&cc_in, todo); 927 inlen -= todo; 928 inblk += todo; 929 if (inlen == 0) 930 inblk = crypto_cursor_segment(&cc_in, 931 &inlen); 932 } 933 934 if (outblk == blk) { 935 crypto_cursor_copyback(&cc_out, blksz, blk); 936 outblk = crypto_cursor_segment(&cc_out, 937 &outlen); 938 } else { 939 crypto_cursor_advance(&cc_out, todo); 940 outlen -= todo; 941 outblk += todo; 942 if (outlen == 0) 943 outblk = crypto_cursor_segment(&cc_out, 944 &outlen); 945 } 946 } 947 if (resid > 0) { 948 crypto_cursor_copydata(&cc_in, resid, blk); 949 exf->decrypt_last(ctx, blk, blk, resid); 950 crypto_cursor_copyback(&cc_out, resid, blk); 951 } 952 } else { 953 /* Inject the authentication data */ 954 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 955 s.tag); 956 } 957 958 out: 959 explicit_bzero(ctx, exf->ctxsize); 960 explicit_bzero(&s, sizeof(s)); 961 return (error); 962 } 963 964 static int 965 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 966 { 967 const struct crypto_session_params *csp; 968 struct { 969 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 970 u_char tag[POLY1305_HASH_LEN]; 971 u_char tag2[POLY1305_HASH_LEN]; 972 } s; 973 u_char *blk = (u_char *)s.blkbuf; 974 struct crypto_buffer_cursor cc_in, cc_out; 975 const u_char *inblk; 976 u_char *outblk; 977 uint64_t *blkp; 978 const struct swcr_auth *swa; 979 const struct swcr_encdec *swe; 980 const struct enc_xform *exf; 981 void *ctx; 982 size_t len; 983 int blksz, error, r, resid; 984 985 swa = &ses->swcr_auth; 986 swe = &ses->swcr_encdec; 987 exf = swe->sw_exf; 988 blksz = exf->native_blocksize; 989 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 990 991 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 992 return (EINVAL); 993 994 csp = crypto_get_params(crp->crp_session); 995 996 ctx = __builtin_alloca(exf->ctxsize); 997 if (crp->crp_cipher_key != NULL) 998 exf->setkey(ctx, crp->crp_cipher_key, 999 csp->csp_cipher_klen); 1000 else 1001 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 1002 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 1003 1004 /* Supply MAC with AAD */ 1005 if (crp->crp_aad != NULL) 1006 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 1007 else 1008 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 1009 exf->update, ctx); 1010 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 1011 /* padding1 */ 1012 memset(blk, 0, POLY1305_BLOCK_LEN); 1013 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1014 crp->crp_aad_length % POLY1305_BLOCK_LEN); 1015 } 1016 1017 /* Do encryption with MAC */ 1018 crypto_cursor_init(&cc_in, &crp->crp_buf); 1019 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1020 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1021 crypto_cursor_init(&cc_out, &crp->crp_obuf); 1022 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1023 } else 1024 cc_out = cc_in; 1025 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 1026 inblk = crypto_cursor_segment(&cc_in, &len); 1027 if (len < blksz) { 1028 crypto_cursor_copydata(&cc_in, blksz, blk); 1029 inblk = blk; 1030 } else 1031 crypto_cursor_advance(&cc_in, blksz); 1032 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1033 outblk = crypto_cursor_segment(&cc_out, &len); 1034 if (len < blksz) 1035 outblk = blk; 1036 exf->encrypt(ctx, inblk, outblk); 1037 exf->update(ctx, outblk, blksz); 1038 if (outblk == blk) 1039 crypto_cursor_copyback(&cc_out, blksz, blk); 1040 else 1041 crypto_cursor_advance(&cc_out, blksz); 1042 } else { 1043 exf->update(ctx, inblk, blksz); 1044 } 1045 } 1046 if (resid > 0) { 1047 crypto_cursor_copydata(&cc_in, resid, blk); 1048 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1049 exf->encrypt_last(ctx, blk, blk, resid); 1050 crypto_cursor_copyback(&cc_out, resid, blk); 1051 } 1052 exf->update(ctx, blk, resid); 1053 if (resid % POLY1305_BLOCK_LEN != 0) { 1054 /* padding2 */ 1055 memset(blk, 0, POLY1305_BLOCK_LEN); 1056 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1057 resid % POLY1305_BLOCK_LEN); 1058 } 1059 } 1060 1061 /* lengths */ 1062 blkp = (uint64_t *)blk; 1063 blkp[0] = htole64(crp->crp_aad_length); 1064 blkp[1] = htole64(crp->crp_payload_length); 1065 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1066 1067 /* Finalize MAC */ 1068 exf->final(s.tag, ctx); 1069 1070 /* Validate tag */ 1071 error = 0; 1072 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1073 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 1074 s.tag2); 1075 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1076 if (r != 0) { 1077 error = EBADMSG; 1078 goto out; 1079 } 1080 1081 /* tag matches, decrypt data */ 1082 crypto_cursor_init(&cc_in, &crp->crp_buf); 1083 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1084 for (resid = crp->crp_payload_length; resid > blksz; 1085 resid -= blksz) { 1086 inblk = crypto_cursor_segment(&cc_in, &len); 1087 if (len < blksz) { 1088 crypto_cursor_copydata(&cc_in, blksz, blk); 1089 inblk = blk; 1090 } else 1091 crypto_cursor_advance(&cc_in, blksz); 1092 outblk = crypto_cursor_segment(&cc_out, &len); 1093 if (len < blksz) 1094 outblk = blk; 1095 exf->decrypt(ctx, inblk, outblk); 1096 if (outblk == blk) 1097 crypto_cursor_copyback(&cc_out, blksz, blk); 1098 else 1099 crypto_cursor_advance(&cc_out, blksz); 1100 } 1101 if (resid > 0) { 1102 crypto_cursor_copydata(&cc_in, resid, blk); 1103 exf->decrypt_last(ctx, blk, blk, resid); 1104 crypto_cursor_copyback(&cc_out, resid, blk); 1105 } 1106 } else { 1107 /* Inject the authentication data */ 1108 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1109 s.tag); 1110 } 1111 1112 out: 1113 explicit_bzero(ctx, exf->ctxsize); 1114 explicit_bzero(&s, sizeof(s)); 1115 return (error); 1116 } 1117 1118 /* 1119 * Apply a cipher and a digest to perform EtA. 1120 */ 1121 static int 1122 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1123 { 1124 int error; 1125 1126 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1127 error = swcr_encdec(ses, crp); 1128 if (error == 0) 1129 error = swcr_authcompute(ses, crp); 1130 } else { 1131 error = swcr_authcompute(ses, crp); 1132 if (error == 0) 1133 error = swcr_encdec(ses, crp); 1134 } 1135 return (error); 1136 } 1137 1138 /* 1139 * Apply a compression/decompression algorithm 1140 */ 1141 static int 1142 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1143 { 1144 const struct comp_algo *cxf; 1145 uint8_t *data, *out; 1146 int adj; 1147 uint32_t result; 1148 1149 cxf = ses->swcr_compdec.sw_cxf; 1150 1151 /* We must handle the whole buffer of data in one time 1152 * then if there is not all the data in the mbuf, we must 1153 * copy in a buffer. 1154 */ 1155 1156 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1157 if (data == NULL) 1158 return (EINVAL); 1159 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1160 data); 1161 1162 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1163 result = cxf->compress(data, crp->crp_payload_length, &out); 1164 else 1165 result = cxf->decompress(data, crp->crp_payload_length, &out); 1166 1167 free(data, M_CRYPTO_DATA); 1168 if (result == 0) 1169 return (EINVAL); 1170 crp->crp_olen = result; 1171 1172 /* Check the compressed size when doing compression */ 1173 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1174 if (result >= crp->crp_payload_length) { 1175 /* Compression was useless, we lost time */ 1176 free(out, M_CRYPTO_DATA); 1177 return (0); 1178 } 1179 } 1180 1181 /* Copy back the (de)compressed data. m_copyback is 1182 * extending the mbuf as necessary. 1183 */ 1184 crypto_copyback(crp, crp->crp_payload_start, result, out); 1185 if (result < crp->crp_payload_length) { 1186 switch (crp->crp_buf.cb_type) { 1187 case CRYPTO_BUF_MBUF: 1188 case CRYPTO_BUF_SINGLE_MBUF: 1189 adj = result - crp->crp_payload_length; 1190 m_adj(crp->crp_buf.cb_mbuf, adj); 1191 break; 1192 case CRYPTO_BUF_UIO: { 1193 struct uio *uio = crp->crp_buf.cb_uio; 1194 int ind; 1195 1196 adj = crp->crp_payload_length - result; 1197 ind = uio->uio_iovcnt - 1; 1198 1199 while (adj > 0 && ind >= 0) { 1200 if (adj < uio->uio_iov[ind].iov_len) { 1201 uio->uio_iov[ind].iov_len -= adj; 1202 break; 1203 } 1204 1205 adj -= uio->uio_iov[ind].iov_len; 1206 uio->uio_iov[ind].iov_len = 0; 1207 ind--; 1208 uio->uio_iovcnt--; 1209 } 1210 } 1211 break; 1212 case CRYPTO_BUF_VMPAGE: 1213 adj = crp->crp_payload_length - result; 1214 crp->crp_buf.cb_vm_page_len -= adj; 1215 break; 1216 default: 1217 break; 1218 } 1219 } 1220 free(out, M_CRYPTO_DATA); 1221 return 0; 1222 } 1223 1224 static int 1225 swcr_setup_cipher(struct swcr_session *ses, 1226 const struct crypto_session_params *csp) 1227 { 1228 struct swcr_encdec *swe; 1229 const struct enc_xform *txf; 1230 int error; 1231 1232 swe = &ses->swcr_encdec; 1233 txf = crypto_cipher(csp); 1234 if (csp->csp_cipher_key != NULL) { 1235 if (txf->ctxsize != 0) { 1236 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1237 M_NOWAIT); 1238 if (swe->sw_ctx == NULL) 1239 return (ENOMEM); 1240 } 1241 error = txf->setkey(swe->sw_ctx, 1242 csp->csp_cipher_key, csp->csp_cipher_klen); 1243 if (error) 1244 return (error); 1245 } 1246 swe->sw_exf = txf; 1247 return (0); 1248 } 1249 1250 static int 1251 swcr_setup_auth(struct swcr_session *ses, 1252 const struct crypto_session_params *csp) 1253 { 1254 struct swcr_auth *swa; 1255 const struct auth_hash *axf; 1256 1257 swa = &ses->swcr_auth; 1258 1259 axf = crypto_auth_hash(csp); 1260 swa->sw_axf = axf; 1261 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1262 return (EINVAL); 1263 if (csp->csp_auth_mlen == 0) 1264 swa->sw_mlen = axf->hashsize; 1265 else 1266 swa->sw_mlen = csp->csp_auth_mlen; 1267 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1268 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1269 M_NOWAIT); 1270 if (swa->sw_ictx == NULL) 1271 return (ENOBUFS); 1272 } 1273 1274 switch (csp->csp_auth_alg) { 1275 case CRYPTO_SHA1_HMAC: 1276 case CRYPTO_SHA2_224_HMAC: 1277 case CRYPTO_SHA2_256_HMAC: 1278 case CRYPTO_SHA2_384_HMAC: 1279 case CRYPTO_SHA2_512_HMAC: 1280 case CRYPTO_RIPEMD160_HMAC: 1281 swa->sw_hmac = true; 1282 if (csp->csp_auth_key != NULL) { 1283 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1284 M_NOWAIT); 1285 if (swa->sw_octx == NULL) 1286 return (ENOBUFS); 1287 hmac_init_ipad(axf, csp->csp_auth_key, 1288 csp->csp_auth_klen, swa->sw_ictx); 1289 hmac_init_opad(axf, csp->csp_auth_key, 1290 csp->csp_auth_klen, swa->sw_octx); 1291 } 1292 break; 1293 case CRYPTO_RIPEMD160: 1294 case CRYPTO_SHA1: 1295 case CRYPTO_SHA2_224: 1296 case CRYPTO_SHA2_256: 1297 case CRYPTO_SHA2_384: 1298 case CRYPTO_SHA2_512: 1299 case CRYPTO_NULL_HMAC: 1300 axf->Init(swa->sw_ictx); 1301 break; 1302 case CRYPTO_AES_NIST_GMAC: 1303 case CRYPTO_AES_CCM_CBC_MAC: 1304 case CRYPTO_POLY1305: 1305 if (csp->csp_auth_key != NULL) { 1306 axf->Init(swa->sw_ictx); 1307 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1308 csp->csp_auth_klen); 1309 } 1310 break; 1311 case CRYPTO_BLAKE2B: 1312 case CRYPTO_BLAKE2S: 1313 /* 1314 * Blake2b and Blake2s support an optional key but do 1315 * not require one. 1316 */ 1317 if (csp->csp_auth_klen == 0) 1318 axf->Init(swa->sw_ictx); 1319 else if (csp->csp_auth_key != NULL) 1320 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1321 csp->csp_auth_klen); 1322 break; 1323 } 1324 1325 if (csp->csp_mode == CSP_MODE_DIGEST) { 1326 switch (csp->csp_auth_alg) { 1327 case CRYPTO_AES_NIST_GMAC: 1328 ses->swcr_process = swcr_gmac; 1329 break; 1330 case CRYPTO_AES_CCM_CBC_MAC: 1331 ses->swcr_process = swcr_ccm_cbc_mac; 1332 break; 1333 default: 1334 ses->swcr_process = swcr_authcompute; 1335 } 1336 } 1337 1338 return (0); 1339 } 1340 1341 static int 1342 swcr_setup_aead(struct swcr_session *ses, 1343 const struct crypto_session_params *csp) 1344 { 1345 struct swcr_auth *swa; 1346 int error; 1347 1348 error = swcr_setup_cipher(ses, csp); 1349 if (error) 1350 return (error); 1351 1352 swa = &ses->swcr_auth; 1353 if (csp->csp_auth_mlen == 0) 1354 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1355 else 1356 swa->sw_mlen = csp->csp_auth_mlen; 1357 return (0); 1358 } 1359 1360 static bool 1361 swcr_auth_supported(const struct crypto_session_params *csp) 1362 { 1363 const struct auth_hash *axf; 1364 1365 axf = crypto_auth_hash(csp); 1366 if (axf == NULL) 1367 return (false); 1368 switch (csp->csp_auth_alg) { 1369 case CRYPTO_SHA1_HMAC: 1370 case CRYPTO_SHA2_224_HMAC: 1371 case CRYPTO_SHA2_256_HMAC: 1372 case CRYPTO_SHA2_384_HMAC: 1373 case CRYPTO_SHA2_512_HMAC: 1374 case CRYPTO_NULL_HMAC: 1375 case CRYPTO_RIPEMD160_HMAC: 1376 break; 1377 case CRYPTO_AES_NIST_GMAC: 1378 switch (csp->csp_auth_klen * 8) { 1379 case 128: 1380 case 192: 1381 case 256: 1382 break; 1383 default: 1384 return (false); 1385 } 1386 if (csp->csp_auth_key == NULL) 1387 return (false); 1388 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1389 return (false); 1390 break; 1391 case CRYPTO_POLY1305: 1392 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1393 return (false); 1394 break; 1395 case CRYPTO_AES_CCM_CBC_MAC: 1396 switch (csp->csp_auth_klen * 8) { 1397 case 128: 1398 case 192: 1399 case 256: 1400 break; 1401 default: 1402 return (false); 1403 } 1404 if (csp->csp_auth_key == NULL) 1405 return (false); 1406 break; 1407 } 1408 return (true); 1409 } 1410 1411 static bool 1412 swcr_cipher_supported(const struct crypto_session_params *csp) 1413 { 1414 const struct enc_xform *txf; 1415 1416 txf = crypto_cipher(csp); 1417 if (txf == NULL) 1418 return (false); 1419 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1420 txf->ivsize != csp->csp_ivlen) 1421 return (false); 1422 return (true); 1423 } 1424 1425 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1426 1427 static int 1428 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1429 { 1430 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1431 return (EINVAL); 1432 switch (csp->csp_mode) { 1433 case CSP_MODE_COMPRESS: 1434 switch (csp->csp_cipher_alg) { 1435 case CRYPTO_DEFLATE_COMP: 1436 break; 1437 default: 1438 return (EINVAL); 1439 } 1440 break; 1441 case CSP_MODE_CIPHER: 1442 switch (csp->csp_cipher_alg) { 1443 case CRYPTO_AES_NIST_GCM_16: 1444 case CRYPTO_AES_CCM_16: 1445 case CRYPTO_CHACHA20_POLY1305: 1446 case CRYPTO_XCHACHA20_POLY1305: 1447 return (EINVAL); 1448 default: 1449 if (!swcr_cipher_supported(csp)) 1450 return (EINVAL); 1451 break; 1452 } 1453 break; 1454 case CSP_MODE_DIGEST: 1455 if (!swcr_auth_supported(csp)) 1456 return (EINVAL); 1457 break; 1458 case CSP_MODE_AEAD: 1459 switch (csp->csp_cipher_alg) { 1460 case CRYPTO_AES_NIST_GCM_16: 1461 case CRYPTO_AES_CCM_16: 1462 switch (csp->csp_cipher_klen * 8) { 1463 case 128: 1464 case 192: 1465 case 256: 1466 break; 1467 default: 1468 return (EINVAL); 1469 } 1470 break; 1471 case CRYPTO_CHACHA20_POLY1305: 1472 case CRYPTO_XCHACHA20_POLY1305: 1473 break; 1474 default: 1475 return (EINVAL); 1476 } 1477 break; 1478 case CSP_MODE_ETA: 1479 /* AEAD algorithms cannot be used for EtA. */ 1480 switch (csp->csp_cipher_alg) { 1481 case CRYPTO_AES_NIST_GCM_16: 1482 case CRYPTO_AES_CCM_16: 1483 case CRYPTO_CHACHA20_POLY1305: 1484 case CRYPTO_XCHACHA20_POLY1305: 1485 return (EINVAL); 1486 } 1487 switch (csp->csp_auth_alg) { 1488 case CRYPTO_AES_NIST_GMAC: 1489 case CRYPTO_AES_CCM_CBC_MAC: 1490 return (EINVAL); 1491 } 1492 1493 if (!swcr_cipher_supported(csp) || 1494 !swcr_auth_supported(csp)) 1495 return (EINVAL); 1496 break; 1497 default: 1498 return (EINVAL); 1499 } 1500 1501 return (CRYPTODEV_PROBE_SOFTWARE); 1502 } 1503 1504 /* 1505 * Generate a new software session. 1506 */ 1507 static int 1508 swcr_newsession(device_t dev, crypto_session_t cses, 1509 const struct crypto_session_params *csp) 1510 { 1511 struct swcr_session *ses; 1512 const struct comp_algo *cxf; 1513 int error; 1514 1515 ses = crypto_get_driver_session(cses); 1516 1517 error = 0; 1518 switch (csp->csp_mode) { 1519 case CSP_MODE_COMPRESS: 1520 switch (csp->csp_cipher_alg) { 1521 case CRYPTO_DEFLATE_COMP: 1522 cxf = &comp_algo_deflate; 1523 break; 1524 #ifdef INVARIANTS 1525 default: 1526 panic("bad compression algo"); 1527 #endif 1528 } 1529 ses->swcr_compdec.sw_cxf = cxf; 1530 ses->swcr_process = swcr_compdec; 1531 break; 1532 case CSP_MODE_CIPHER: 1533 switch (csp->csp_cipher_alg) { 1534 case CRYPTO_NULL_CBC: 1535 ses->swcr_process = swcr_null; 1536 break; 1537 #ifdef INVARIANTS 1538 case CRYPTO_AES_NIST_GCM_16: 1539 case CRYPTO_AES_CCM_16: 1540 case CRYPTO_CHACHA20_POLY1305: 1541 case CRYPTO_XCHACHA20_POLY1305: 1542 panic("bad cipher algo"); 1543 #endif 1544 default: 1545 error = swcr_setup_cipher(ses, csp); 1546 if (error == 0) 1547 ses->swcr_process = swcr_encdec; 1548 } 1549 break; 1550 case CSP_MODE_DIGEST: 1551 error = swcr_setup_auth(ses, csp); 1552 break; 1553 case CSP_MODE_AEAD: 1554 switch (csp->csp_cipher_alg) { 1555 case CRYPTO_AES_NIST_GCM_16: 1556 error = swcr_setup_aead(ses, csp); 1557 if (error == 0) 1558 ses->swcr_process = swcr_gcm; 1559 break; 1560 case CRYPTO_AES_CCM_16: 1561 error = swcr_setup_aead(ses, csp); 1562 if (error == 0) 1563 ses->swcr_process = swcr_ccm; 1564 break; 1565 case CRYPTO_CHACHA20_POLY1305: 1566 case CRYPTO_XCHACHA20_POLY1305: 1567 error = swcr_setup_aead(ses, csp); 1568 if (error == 0) 1569 ses->swcr_process = swcr_chacha20_poly1305; 1570 break; 1571 #ifdef INVARIANTS 1572 default: 1573 panic("bad aead algo"); 1574 #endif 1575 } 1576 break; 1577 case CSP_MODE_ETA: 1578 #ifdef INVARIANTS 1579 switch (csp->csp_cipher_alg) { 1580 case CRYPTO_AES_NIST_GCM_16: 1581 case CRYPTO_AES_CCM_16: 1582 case CRYPTO_CHACHA20_POLY1305: 1583 case CRYPTO_XCHACHA20_POLY1305: 1584 panic("bad eta cipher algo"); 1585 } 1586 switch (csp->csp_auth_alg) { 1587 case CRYPTO_AES_NIST_GMAC: 1588 case CRYPTO_AES_CCM_CBC_MAC: 1589 panic("bad eta auth algo"); 1590 } 1591 #endif 1592 1593 error = swcr_setup_auth(ses, csp); 1594 if (error) 1595 break; 1596 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1597 /* Effectively degrade to digest mode. */ 1598 ses->swcr_process = swcr_authcompute; 1599 break; 1600 } 1601 1602 error = swcr_setup_cipher(ses, csp); 1603 if (error == 0) 1604 ses->swcr_process = swcr_eta; 1605 break; 1606 default: 1607 error = EINVAL; 1608 } 1609 1610 if (error) 1611 swcr_freesession(dev, cses); 1612 return (error); 1613 } 1614 1615 static void 1616 swcr_freesession(device_t dev, crypto_session_t cses) 1617 { 1618 struct swcr_session *ses; 1619 1620 ses = crypto_get_driver_session(cses); 1621 1622 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1623 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1624 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1625 } 1626 1627 /* 1628 * Process a software request. 1629 */ 1630 static int 1631 swcr_process(device_t dev, struct cryptop *crp, int hint) 1632 { 1633 struct swcr_session *ses; 1634 1635 ses = crypto_get_driver_session(crp->crp_session); 1636 1637 crp->crp_etype = ses->swcr_process(ses, crp); 1638 1639 crypto_done(crp); 1640 return (0); 1641 } 1642 1643 static void 1644 swcr_identify(driver_t *drv, device_t parent) 1645 { 1646 /* NB: order 10 is so we get attached after h/w devices */ 1647 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1648 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1649 panic("cryptosoft: could not attach"); 1650 } 1651 1652 static int 1653 swcr_probe(device_t dev) 1654 { 1655 device_set_desc(dev, "software crypto"); 1656 device_quiet(dev); 1657 return (BUS_PROBE_NOWILDCARD); 1658 } 1659 1660 static int 1661 swcr_attach(device_t dev) 1662 { 1663 1664 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1665 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1666 if (swcr_id < 0) { 1667 device_printf(dev, "cannot initialize!"); 1668 return (ENXIO); 1669 } 1670 1671 return (0); 1672 } 1673 1674 static int 1675 swcr_detach(device_t dev) 1676 { 1677 crypto_unregister_all(swcr_id); 1678 return 0; 1679 } 1680 1681 static device_method_t swcr_methods[] = { 1682 DEVMETHOD(device_identify, swcr_identify), 1683 DEVMETHOD(device_probe, swcr_probe), 1684 DEVMETHOD(device_attach, swcr_attach), 1685 DEVMETHOD(device_detach, swcr_detach), 1686 1687 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1688 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1689 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1690 DEVMETHOD(cryptodev_process, swcr_process), 1691 1692 {0, 0}, 1693 }; 1694 1695 static driver_t swcr_driver = { 1696 "cryptosoft", 1697 swcr_methods, 1698 0, /* NB: no softc */ 1699 }; 1700 static devclass_t swcr_devclass; 1701 1702 /* 1703 * NB: We explicitly reference the crypto module so we 1704 * get the necessary ordering when built as a loadable 1705 * module. This is required because we bundle the crypto 1706 * module code together with the cryptosoft driver (otherwise 1707 * normal module dependencies would handle things). 1708 */ 1709 extern int crypto_modevent(struct module *, int, void *); 1710 /* XXX where to attach */ 1711 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1712 MODULE_VERSION(cryptosoft, 1); 1713 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1714