1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen, todo; 109 int blksz, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blksz = exf->blocksize; 128 } else 129 blksz = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 158 159 /* 160 * Loop through encrypting blocks. 'inlen' is the remaining 161 * length of the current segment in the input buffer. 162 * 'outlen' is the remaining length of current segment in the 163 * output buffer. 164 */ 165 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 166 /* 167 * If the current block is not contained within the 168 * current input/output segment, use 'blk' as a local 169 * buffer. 170 */ 171 if (inlen < blksz) { 172 crypto_cursor_copydata(&cc_in, blksz, blk); 173 inblk = blk; 174 inlen = blksz; 175 } 176 if (outlen < blksz) { 177 outblk = blk; 178 outlen = blksz; 179 } 180 181 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blksz); 182 183 if (encrypting) 184 exf->encrypt_multi(ctx, inblk, outblk, todo); 185 else 186 exf->decrypt_multi(ctx, inblk, outblk, todo); 187 188 if (inblk == blk) { 189 inblk = crypto_cursor_segment(&cc_in, &inlen); 190 } else { 191 crypto_cursor_advance(&cc_in, todo); 192 inlen -= todo; 193 inblk += todo; 194 if (inlen == 0) 195 inblk = crypto_cursor_segment(&cc_in, &inlen); 196 } 197 198 if (outblk == blk) { 199 crypto_cursor_copyback(&cc_out, blksz, blk); 200 outblk = crypto_cursor_segment(&cc_out, &outlen); 201 } else { 202 crypto_cursor_advance(&cc_out, todo); 203 outlen -= todo; 204 outblk += todo; 205 if (outlen == 0) 206 outblk = crypto_cursor_segment(&cc_out, 207 &outlen); 208 } 209 } 210 211 /* Handle trailing partial block for stream ciphers. */ 212 if (resid > 0) { 213 KASSERT(exf->native_blocksize != 0, 214 ("%s: partial block of %d bytes for cipher %s", 215 __func__, resid, exf->name)); 216 KASSERT(resid < blksz, ("%s: partial block too big", __func__)); 217 218 inblk = crypto_cursor_segment(&cc_in, &inlen); 219 outblk = crypto_cursor_segment(&cc_out, &outlen); 220 if (inlen < resid) { 221 crypto_cursor_copydata(&cc_in, resid, blk); 222 inblk = blk; 223 } 224 if (outlen < resid) 225 outblk = blk; 226 if (encrypting) 227 exf->encrypt_last(ctx, inblk, outblk, 228 resid); 229 else 230 exf->decrypt_last(ctx, inblk, outblk, 231 resid); 232 if (outlen < resid) 233 crypto_cursor_copyback(&cc_out, resid, blk); 234 } 235 236 explicit_bzero(ctx, exf->ctxsize); 237 explicit_bzero(blk, sizeof(blk)); 238 return (0); 239 } 240 241 /* 242 * Compute or verify hash. 243 */ 244 static int 245 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 246 { 247 struct { 248 union authctx ctx; 249 u_char aalg[HASH_MAX_LEN]; 250 u_char uaalg[HASH_MAX_LEN]; 251 } s; 252 const struct crypto_session_params *csp; 253 const struct swcr_auth *sw; 254 const struct auth_hash *axf; 255 int err; 256 257 sw = &ses->swcr_auth; 258 259 axf = sw->sw_axf; 260 261 csp = crypto_get_params(crp->crp_session); 262 if (crp->crp_auth_key != NULL) { 263 if (sw->sw_hmac) { 264 hmac_init_ipad(axf, crp->crp_auth_key, 265 csp->csp_auth_klen, &s.ctx); 266 } else { 267 axf->Init(&s.ctx); 268 axf->Setkey(&s.ctx, crp->crp_auth_key, 269 csp->csp_auth_klen); 270 } 271 } else 272 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 273 274 if (crp->crp_aad != NULL) 275 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 276 else 277 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 278 axf->Update, &s.ctx); 279 if (err) 280 goto out; 281 282 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 283 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 284 err = crypto_apply_buf(&crp->crp_obuf, 285 crp->crp_payload_output_start, crp->crp_payload_length, 286 axf->Update, &s.ctx); 287 else 288 err = crypto_apply(crp, crp->crp_payload_start, 289 crp->crp_payload_length, axf->Update, &s.ctx); 290 if (err) 291 goto out; 292 293 if (csp->csp_flags & CSP_F_ESN) 294 axf->Update(&s.ctx, crp->crp_esn, 4); 295 296 axf->Final(s.aalg, &s.ctx); 297 if (sw->sw_hmac) { 298 if (crp->crp_auth_key != NULL) 299 hmac_init_opad(axf, crp->crp_auth_key, 300 csp->csp_auth_klen, &s.ctx); 301 else 302 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 303 axf->Update(&s.ctx, s.aalg, axf->hashsize); 304 axf->Final(s.aalg, &s.ctx); 305 } 306 307 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 308 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 309 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 310 err = EBADMSG; 311 } else { 312 /* Inject the authentication data */ 313 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 314 } 315 out: 316 explicit_bzero(&s, sizeof(s)); 317 return (err); 318 } 319 320 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 321 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 322 323 static int 324 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 325 { 326 struct { 327 union authctx ctx; 328 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 329 u_char tag[GMAC_DIGEST_LEN]; 330 u_char tag2[GMAC_DIGEST_LEN]; 331 } s; 332 u_char *blk = (u_char *)s.blkbuf; 333 struct crypto_buffer_cursor cc; 334 const u_char *inblk; 335 const struct swcr_auth *swa; 336 const struct auth_hash *axf; 337 uint32_t *blkp; 338 size_t len; 339 int blksz, error, ivlen, resid; 340 341 swa = &ses->swcr_auth; 342 axf = swa->sw_axf; 343 blksz = GMAC_BLOCK_LEN; 344 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 345 __func__)); 346 347 if (crp->crp_auth_key != NULL) { 348 axf->Init(&s.ctx); 349 axf->Setkey(&s.ctx, crp->crp_auth_key, 350 crypto_get_params(crp->crp_session)->csp_auth_klen); 351 } else 352 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 353 354 /* Initialize the IV */ 355 ivlen = AES_GCM_IV_LEN; 356 crypto_read_iv(crp, blk); 357 358 axf->Reinit(&s.ctx, blk, ivlen); 359 crypto_cursor_init(&cc, &crp->crp_buf); 360 crypto_cursor_advance(&cc, crp->crp_payload_start); 361 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 362 inblk = crypto_cursor_segment(&cc, &len); 363 if (len >= blksz) { 364 len = rounddown(MIN(len, resid), blksz); 365 crypto_cursor_advance(&cc, len); 366 } else { 367 len = blksz; 368 crypto_cursor_copydata(&cc, len, blk); 369 inblk = blk; 370 } 371 axf->Update(&s.ctx, inblk, len); 372 } 373 if (resid > 0) { 374 memset(blk, 0, blksz); 375 crypto_cursor_copydata(&cc, resid, blk); 376 axf->Update(&s.ctx, blk, blksz); 377 } 378 379 /* length block */ 380 memset(blk, 0, blksz); 381 blkp = (uint32_t *)blk + 1; 382 *blkp = htobe32(crp->crp_payload_length * 8); 383 axf->Update(&s.ctx, blk, blksz); 384 385 /* Finalize MAC */ 386 axf->Final(s.tag, &s.ctx); 387 388 error = 0; 389 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 390 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 391 s.tag2); 392 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 393 error = EBADMSG; 394 } else { 395 /* Inject the authentication data */ 396 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 397 } 398 explicit_bzero(&s, sizeof(s)); 399 return (error); 400 } 401 402 static int 403 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 404 { 405 struct { 406 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 407 u_char tag[GMAC_DIGEST_LEN]; 408 u_char tag2[GMAC_DIGEST_LEN]; 409 } s; 410 u_char *blk = (u_char *)s.blkbuf; 411 struct crypto_buffer_cursor cc_in, cc_out; 412 const u_char *inblk; 413 u_char *outblk; 414 size_t inlen, outlen, todo; 415 const struct swcr_auth *swa; 416 const struct swcr_encdec *swe; 417 const struct enc_xform *exf; 418 void *ctx; 419 uint32_t *blkp; 420 int blksz, error, ivlen, r, resid; 421 422 swa = &ses->swcr_auth; 423 swe = &ses->swcr_encdec; 424 exf = swe->sw_exf; 425 blksz = GMAC_BLOCK_LEN; 426 KASSERT(blksz == exf->native_blocksize, 427 ("%s: blocksize mismatch", __func__)); 428 429 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 430 return (EINVAL); 431 432 ivlen = AES_GCM_IV_LEN; 433 434 ctx = __builtin_alloca(exf->ctxsize); 435 if (crp->crp_cipher_key != NULL) 436 exf->setkey(ctx, crp->crp_cipher_key, 437 crypto_get_params(crp->crp_session)->csp_cipher_klen); 438 else 439 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 440 exf->reinit(ctx, crp->crp_iv, ivlen); 441 442 /* Supply MAC with AAD */ 443 if (crp->crp_aad != NULL) { 444 inlen = rounddown2(crp->crp_aad_length, blksz); 445 if (inlen != 0) 446 exf->update(ctx, crp->crp_aad, inlen); 447 if (crp->crp_aad_length != inlen) { 448 memset(blk, 0, blksz); 449 memcpy(blk, (char *)crp->crp_aad + inlen, 450 crp->crp_aad_length - inlen); 451 exf->update(ctx, blk, blksz); 452 } 453 } else { 454 crypto_cursor_init(&cc_in, &crp->crp_buf); 455 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 456 for (resid = crp->crp_aad_length; resid >= blksz; 457 resid -= inlen) { 458 inblk = crypto_cursor_segment(&cc_in, &inlen); 459 if (inlen >= blksz) { 460 inlen = rounddown2(MIN(inlen, resid), blksz); 461 crypto_cursor_advance(&cc_in, inlen); 462 } else { 463 inlen = blksz; 464 crypto_cursor_copydata(&cc_in, inlen, blk); 465 inblk = blk; 466 } 467 exf->update(ctx, inblk, inlen); 468 } 469 if (resid > 0) { 470 memset(blk, 0, blksz); 471 crypto_cursor_copydata(&cc_in, resid, blk); 472 exf->update(ctx, blk, blksz); 473 } 474 } 475 476 /* Do encryption with MAC */ 477 crypto_cursor_init(&cc_in, &crp->crp_buf); 478 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 479 inblk = crypto_cursor_segment(&cc_in, &inlen); 480 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 481 crypto_cursor_init(&cc_out, &crp->crp_obuf); 482 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 483 } else 484 cc_out = cc_in; 485 outblk = crypto_cursor_segment(&cc_out, &outlen); 486 487 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 488 if (inlen < blksz) { 489 crypto_cursor_copydata(&cc_in, blksz, blk); 490 inblk = blk; 491 inlen = blksz; 492 } 493 494 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 495 if (outlen < blksz) { 496 outblk = blk; 497 outlen = blksz; 498 } 499 500 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 501 blksz); 502 503 exf->encrypt_multi(ctx, inblk, outblk, todo); 504 exf->update(ctx, outblk, todo); 505 506 if (outblk == blk) { 507 crypto_cursor_copyback(&cc_out, blksz, blk); 508 outblk = crypto_cursor_segment(&cc_out, &outlen); 509 } else { 510 crypto_cursor_advance(&cc_out, todo); 511 outlen -= todo; 512 outblk += todo; 513 if (outlen == 0) 514 outblk = crypto_cursor_segment(&cc_out, 515 &outlen); 516 } 517 } else { 518 todo = rounddown2(MIN(resid, inlen), blksz); 519 exf->update(ctx, inblk, todo); 520 } 521 522 if (inblk == blk) { 523 inblk = crypto_cursor_segment(&cc_in, &inlen); 524 } else { 525 crypto_cursor_advance(&cc_in, todo); 526 inlen -= todo; 527 inblk += todo; 528 if (inlen == 0) 529 inblk = crypto_cursor_segment(&cc_in, &inlen); 530 } 531 } 532 if (resid > 0) { 533 crypto_cursor_copydata(&cc_in, resid, blk); 534 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 535 exf->encrypt_last(ctx, blk, blk, resid); 536 crypto_cursor_copyback(&cc_out, resid, blk); 537 } 538 exf->update(ctx, blk, resid); 539 } 540 541 /* length block */ 542 memset(blk, 0, blksz); 543 blkp = (uint32_t *)blk + 1; 544 *blkp = htobe32(crp->crp_aad_length * 8); 545 blkp = (uint32_t *)blk + 3; 546 *blkp = htobe32(crp->crp_payload_length * 8); 547 exf->update(ctx, blk, blksz); 548 549 /* Finalize MAC */ 550 exf->final(s.tag, ctx); 551 552 /* Validate tag */ 553 error = 0; 554 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 555 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 556 s.tag2); 557 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 558 if (r != 0) { 559 error = EBADMSG; 560 goto out; 561 } 562 563 /* tag matches, decrypt data */ 564 crypto_cursor_init(&cc_in, &crp->crp_buf); 565 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 566 inblk = crypto_cursor_segment(&cc_in, &inlen); 567 568 for (resid = crp->crp_payload_length; resid > blksz; 569 resid -= todo) { 570 if (inlen < blksz) { 571 crypto_cursor_copydata(&cc_in, blksz, blk); 572 inblk = blk; 573 inlen = blksz; 574 } 575 if (outlen < blksz) { 576 outblk = blk; 577 outlen = blksz; 578 } 579 580 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 581 blksz); 582 583 exf->decrypt_multi(ctx, inblk, outblk, todo); 584 585 if (inblk == blk) { 586 inblk = crypto_cursor_segment(&cc_in, &inlen); 587 } else { 588 crypto_cursor_advance(&cc_in, todo); 589 inlen -= todo; 590 inblk += todo; 591 if (inlen == 0) 592 inblk = crypto_cursor_segment(&cc_in, 593 &inlen); 594 } 595 596 if (outblk == blk) { 597 crypto_cursor_copyback(&cc_out, blksz, blk); 598 outblk = crypto_cursor_segment(&cc_out, 599 &outlen); 600 } else { 601 crypto_cursor_advance(&cc_out, todo); 602 outlen -= todo; 603 outblk += todo; 604 if (outlen == 0) 605 outblk = crypto_cursor_segment(&cc_out, 606 &outlen); 607 } 608 } 609 if (resid > 0) { 610 crypto_cursor_copydata(&cc_in, resid, blk); 611 exf->decrypt_last(ctx, blk, blk, resid); 612 crypto_cursor_copyback(&cc_out, resid, blk); 613 } 614 } else { 615 /* Inject the authentication data */ 616 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 617 s.tag); 618 } 619 620 out: 621 explicit_bzero(ctx, exf->ctxsize); 622 explicit_bzero(&s, sizeof(s)); 623 624 return (error); 625 } 626 627 static void 628 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 629 u_int data_length, u_int tag_length, uint8_t *b0) 630 { 631 uint8_t *bp; 632 uint8_t flags, L; 633 634 KASSERT(nonce_length >= 7 && nonce_length <= 13, 635 ("nonce_length must be between 7 and 13 bytes")); 636 637 /* 638 * Need to determine the L field value. This is the number of 639 * bytes needed to specify the length of the message; the length 640 * is whatever is left in the 16 bytes after specifying flags and 641 * the nonce. 642 */ 643 L = 15 - nonce_length; 644 645 flags = ((aad_length > 0) << 6) + 646 (((tag_length - 2) / 2) << 3) + 647 L - 1; 648 649 /* 650 * Now we need to set up the first block, which has flags, nonce, 651 * and the message length. 652 */ 653 b0[0] = flags; 654 memcpy(b0 + 1, nonce, nonce_length); 655 bp = b0 + 1 + nonce_length; 656 657 /* Need to copy L' [aka L-1] bytes of data_length */ 658 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 659 *dst = data_length; 660 data_length >>= 8; 661 } 662 } 663 664 /* NB: OCF only supports AAD lengths < 2^32. */ 665 static int 666 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 667 { 668 if (aad_length < ((1 << 16) - (1 << 8))) { 669 be16enc(blk, aad_length); 670 return (sizeof(uint16_t)); 671 } else { 672 blk[0] = 0xff; 673 blk[1] = 0xfe; 674 be32enc(blk + 2, aad_length); 675 return (2 + sizeof(uint32_t)); 676 } 677 } 678 679 static int 680 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 681 { 682 struct { 683 union authctx ctx; 684 u_char blk[CCM_CBC_BLOCK_LEN]; 685 u_char tag[AES_CBC_MAC_HASH_LEN]; 686 u_char tag2[AES_CBC_MAC_HASH_LEN]; 687 } s; 688 const struct crypto_session_params *csp; 689 const struct swcr_auth *swa; 690 const struct auth_hash *axf; 691 int error, ivlen, len; 692 693 csp = crypto_get_params(crp->crp_session); 694 swa = &ses->swcr_auth; 695 axf = swa->sw_axf; 696 697 if (crp->crp_auth_key != NULL) { 698 axf->Init(&s.ctx); 699 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 700 } else 701 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 702 703 /* Initialize the IV */ 704 ivlen = csp->csp_ivlen; 705 706 /* Supply MAC with IV */ 707 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 708 709 /* Supply MAC with b0. */ 710 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 711 swa->sw_mlen, s.blk); 712 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 713 714 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 715 axf->Update(&s.ctx, s.blk, len); 716 717 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 718 axf->Update, &s.ctx); 719 720 /* Finalize MAC */ 721 axf->Final(s.tag, &s.ctx); 722 723 error = 0; 724 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 725 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 726 s.tag2); 727 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 728 error = EBADMSG; 729 } else { 730 /* Inject the authentication data */ 731 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 732 s.tag); 733 } 734 explicit_bzero(&s, sizeof(s)); 735 return (error); 736 } 737 738 static int 739 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 740 { 741 const struct crypto_session_params *csp; 742 struct { 743 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 744 u_char tag[AES_CBC_MAC_HASH_LEN]; 745 u_char tag2[AES_CBC_MAC_HASH_LEN]; 746 } s; 747 u_char *blk = (u_char *)s.blkbuf; 748 struct crypto_buffer_cursor cc_in, cc_out; 749 const u_char *inblk; 750 u_char *outblk; 751 size_t inlen, outlen, todo; 752 const struct swcr_auth *swa; 753 const struct swcr_encdec *swe; 754 const struct enc_xform *exf; 755 void *ctx; 756 size_t len; 757 int blksz, error, ivlen, r, resid; 758 759 csp = crypto_get_params(crp->crp_session); 760 swa = &ses->swcr_auth; 761 swe = &ses->swcr_encdec; 762 exf = swe->sw_exf; 763 blksz = AES_BLOCK_LEN; 764 KASSERT(blksz == exf->native_blocksize, 765 ("%s: blocksize mismatch", __func__)); 766 767 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 768 return (EMSGSIZE); 769 770 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 771 return (EINVAL); 772 773 ivlen = csp->csp_ivlen; 774 775 ctx = __builtin_alloca(exf->ctxsize); 776 if (crp->crp_cipher_key != NULL) 777 exf->setkey(ctx, crp->crp_cipher_key, 778 crypto_get_params(crp->crp_session)->csp_cipher_klen); 779 else 780 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 781 exf->reinit(ctx, crp->crp_iv, ivlen); 782 783 /* Supply MAC with b0. */ 784 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 785 "blkbuf too small for b0"); 786 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 787 crp->crp_payload_length, swa->sw_mlen, blk); 788 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 789 790 /* Supply MAC with AAD */ 791 if (crp->crp_aad_length != 0) { 792 len = build_ccm_aad_length(crp->crp_aad_length, blk); 793 exf->update(ctx, blk, len); 794 if (crp->crp_aad != NULL) 795 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 796 else 797 crypto_apply(crp, crp->crp_aad_start, 798 crp->crp_aad_length, exf->update, ctx); 799 800 /* Pad the AAD (including length field) to a full block. */ 801 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 802 if (len != 0) { 803 len = CCM_CBC_BLOCK_LEN - len; 804 memset(blk, 0, CCM_CBC_BLOCK_LEN); 805 exf->update(ctx, blk, len); 806 } 807 } 808 809 /* Do encryption/decryption with MAC */ 810 crypto_cursor_init(&cc_in, &crp->crp_buf); 811 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 812 inblk = crypto_cursor_segment(&cc_in, &inlen); 813 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 814 crypto_cursor_init(&cc_out, &crp->crp_obuf); 815 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 816 } else 817 cc_out = cc_in; 818 outblk = crypto_cursor_segment(&cc_out, &outlen); 819 820 for (resid = crp->crp_payload_length; resid >= blksz; resid -= todo) { 821 if (inlen < blksz) { 822 crypto_cursor_copydata(&cc_in, blksz, blk); 823 inblk = blk; 824 inlen = blksz; 825 } 826 827 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 828 if (outlen < blksz) { 829 outblk = blk; 830 outlen = blksz; 831 } 832 833 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 834 blksz); 835 836 exf->update(ctx, inblk, todo); 837 exf->encrypt_multi(ctx, inblk, outblk, todo); 838 839 if (outblk == blk) { 840 crypto_cursor_copyback(&cc_out, blksz, blk); 841 outblk = crypto_cursor_segment(&cc_out, &outlen); 842 } else { 843 crypto_cursor_advance(&cc_out, todo); 844 outlen -= todo; 845 outblk += todo; 846 if (outlen == 0) 847 outblk = crypto_cursor_segment(&cc_out, 848 &outlen); 849 } 850 } else { 851 /* 852 * One of the problems with CCM+CBC is that 853 * the authentication is done on the 854 * unencrypted data. As a result, we have to 855 * decrypt the data twice: once to generate 856 * the tag and a second time after the tag is 857 * verified. 858 */ 859 todo = blksz; 860 exf->decrypt(ctx, inblk, blk); 861 exf->update(ctx, blk, todo); 862 } 863 864 if (inblk == blk) { 865 inblk = crypto_cursor_segment(&cc_in, &inlen); 866 } else { 867 crypto_cursor_advance(&cc_in, todo); 868 inlen -= todo; 869 inblk += todo; 870 if (inlen == 0) 871 inblk = crypto_cursor_segment(&cc_in, &inlen); 872 } 873 } 874 if (resid > 0) { 875 crypto_cursor_copydata(&cc_in, resid, blk); 876 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 877 exf->update(ctx, blk, resid); 878 exf->encrypt_last(ctx, blk, blk, resid); 879 crypto_cursor_copyback(&cc_out, resid, blk); 880 } else { 881 exf->decrypt_last(ctx, blk, blk, resid); 882 exf->update(ctx, blk, resid); 883 } 884 } 885 886 /* Finalize MAC */ 887 exf->final(s.tag, ctx); 888 889 /* Validate tag */ 890 error = 0; 891 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 892 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 893 s.tag2); 894 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 895 if (r != 0) { 896 error = EBADMSG; 897 goto out; 898 } 899 900 /* tag matches, decrypt data */ 901 exf->reinit(ctx, crp->crp_iv, ivlen); 902 crypto_cursor_init(&cc_in, &crp->crp_buf); 903 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 904 inblk = crypto_cursor_segment(&cc_in, &inlen); 905 906 for (resid = crp->crp_payload_length; resid >= blksz; 907 resid -= todo) { 908 if (inlen < blksz) { 909 crypto_cursor_copydata(&cc_in, blksz, blk); 910 inblk = blk; 911 inlen = blksz; 912 } 913 if (outlen < blksz) { 914 outblk = blk; 915 outlen = blksz; 916 } 917 918 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 919 blksz); 920 921 exf->decrypt_multi(ctx, inblk, outblk, todo); 922 923 if (inblk == blk) { 924 inblk = crypto_cursor_segment(&cc_in, &inlen); 925 } else { 926 crypto_cursor_advance(&cc_in, todo); 927 inlen -= todo; 928 inblk += todo; 929 if (inlen == 0) 930 inblk = crypto_cursor_segment(&cc_in, 931 &inlen); 932 } 933 934 if (outblk == blk) { 935 crypto_cursor_copyback(&cc_out, blksz, blk); 936 outblk = crypto_cursor_segment(&cc_out, 937 &outlen); 938 } else { 939 crypto_cursor_advance(&cc_out, todo); 940 outlen -= todo; 941 outblk += todo; 942 if (outlen == 0) 943 outblk = crypto_cursor_segment(&cc_out, 944 &outlen); 945 } 946 } 947 if (resid > 0) { 948 crypto_cursor_copydata(&cc_in, resid, blk); 949 exf->decrypt_last(ctx, blk, blk, resid); 950 crypto_cursor_copyback(&cc_out, resid, blk); 951 } 952 } else { 953 /* Inject the authentication data */ 954 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 955 s.tag); 956 } 957 958 out: 959 explicit_bzero(ctx, exf->ctxsize); 960 explicit_bzero(&s, sizeof(s)); 961 return (error); 962 } 963 964 static int 965 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 966 { 967 const struct crypto_session_params *csp; 968 struct { 969 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 970 u_char tag[POLY1305_HASH_LEN]; 971 u_char tag2[POLY1305_HASH_LEN]; 972 } s; 973 u_char *blk = (u_char *)s.blkbuf; 974 struct crypto_buffer_cursor cc_in, cc_out; 975 const u_char *inblk; 976 u_char *outblk; 977 size_t inlen, outlen, todo; 978 uint64_t *blkp; 979 const struct swcr_auth *swa; 980 const struct swcr_encdec *swe; 981 const struct enc_xform *exf; 982 void *ctx; 983 int blksz, error, r, resid; 984 985 swa = &ses->swcr_auth; 986 swe = &ses->swcr_encdec; 987 exf = swe->sw_exf; 988 blksz = exf->native_blocksize; 989 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 990 991 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 992 return (EINVAL); 993 994 csp = crypto_get_params(crp->crp_session); 995 996 ctx = __builtin_alloca(exf->ctxsize); 997 if (crp->crp_cipher_key != NULL) 998 exf->setkey(ctx, crp->crp_cipher_key, 999 csp->csp_cipher_klen); 1000 else 1001 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 1002 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 1003 1004 /* Supply MAC with AAD */ 1005 if (crp->crp_aad != NULL) 1006 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 1007 else 1008 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 1009 exf->update, ctx); 1010 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 1011 /* padding1 */ 1012 memset(blk, 0, POLY1305_BLOCK_LEN); 1013 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1014 crp->crp_aad_length % POLY1305_BLOCK_LEN); 1015 } 1016 1017 /* Do encryption with MAC */ 1018 crypto_cursor_init(&cc_in, &crp->crp_buf); 1019 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1020 inblk = crypto_cursor_segment(&cc_in, &inlen); 1021 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 1022 crypto_cursor_init(&cc_out, &crp->crp_obuf); 1023 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 1024 } else 1025 cc_out = cc_in; 1026 outblk = crypto_cursor_segment(&cc_out, &outlen); 1027 1028 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1029 for (resid = crp->crp_payload_length; resid >= blksz; 1030 resid -= todo) { 1031 if (inlen < blksz) { 1032 crypto_cursor_copydata(&cc_in, blksz, blk); 1033 inblk = blk; 1034 inlen = blksz; 1035 } 1036 1037 if (outlen < blksz) { 1038 outblk = blk; 1039 outlen = blksz; 1040 } 1041 1042 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1043 blksz); 1044 1045 exf->encrypt_multi(ctx, inblk, outblk, todo); 1046 exf->update(ctx, outblk, todo); 1047 1048 if (inblk == blk) { 1049 inblk = crypto_cursor_segment(&cc_in, &inlen); 1050 } else { 1051 crypto_cursor_advance(&cc_in, todo); 1052 inlen -= todo; 1053 inblk += todo; 1054 if (inlen == 0) 1055 inblk = crypto_cursor_segment(&cc_in, 1056 &inlen); 1057 } 1058 1059 if (outblk == blk) { 1060 crypto_cursor_copyback(&cc_out, blksz, blk); 1061 outblk = crypto_cursor_segment(&cc_out, &outlen); 1062 } else { 1063 crypto_cursor_advance(&cc_out, todo); 1064 outlen -= todo; 1065 outblk += todo; 1066 if (outlen == 0) 1067 outblk = crypto_cursor_segment(&cc_out, 1068 &outlen); 1069 } 1070 } 1071 if (resid > 0) { 1072 crypto_cursor_copydata(&cc_in, resid, blk); 1073 exf->encrypt_last(ctx, blk, blk, resid); 1074 crypto_cursor_copyback(&cc_out, resid, blk); 1075 exf->update(ctx, blk, resid); 1076 } 1077 } else 1078 crypto_apply(crp, crp->crp_payload_start, 1079 crp->crp_payload_length, exf->update, ctx); 1080 if (crp->crp_payload_length % POLY1305_BLOCK_LEN != 0) { 1081 /* padding2 */ 1082 memset(blk, 0, POLY1305_BLOCK_LEN); 1083 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 1084 crp->crp_payload_length % POLY1305_BLOCK_LEN); 1085 } 1086 1087 /* lengths */ 1088 blkp = (uint64_t *)blk; 1089 blkp[0] = htole64(crp->crp_aad_length); 1090 blkp[1] = htole64(crp->crp_payload_length); 1091 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1092 1093 /* Finalize MAC */ 1094 exf->final(s.tag, ctx); 1095 1096 /* Validate tag */ 1097 error = 0; 1098 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1099 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 1100 s.tag2); 1101 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 1102 if (r != 0) { 1103 error = EBADMSG; 1104 goto out; 1105 } 1106 1107 /* tag matches, decrypt data */ 1108 crypto_cursor_init(&cc_in, &crp->crp_buf); 1109 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1110 inblk = crypto_cursor_segment(&cc_in, &inlen); 1111 1112 for (resid = crp->crp_payload_length; resid > blksz; 1113 resid -= todo) { 1114 if (inlen < blksz) { 1115 crypto_cursor_copydata(&cc_in, blksz, blk); 1116 inblk = blk; 1117 inlen = blksz; 1118 } 1119 if (outlen < blksz) { 1120 outblk = blk; 1121 outlen = blksz; 1122 } 1123 1124 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), 1125 blksz); 1126 1127 exf->decrypt_multi(ctx, inblk, outblk, todo); 1128 1129 if (inblk == blk) { 1130 inblk = crypto_cursor_segment(&cc_in, &inlen); 1131 } else { 1132 crypto_cursor_advance(&cc_in, todo); 1133 inlen -= todo; 1134 inblk += todo; 1135 if (inlen == 0) 1136 inblk = crypto_cursor_segment(&cc_in, 1137 &inlen); 1138 } 1139 1140 if (outblk == blk) { 1141 crypto_cursor_copyback(&cc_out, blksz, blk); 1142 outblk = crypto_cursor_segment(&cc_out, 1143 &outlen); 1144 } else { 1145 crypto_cursor_advance(&cc_out, todo); 1146 outlen -= todo; 1147 outblk += todo; 1148 if (outlen == 0) 1149 outblk = crypto_cursor_segment(&cc_out, 1150 &outlen); 1151 } 1152 } 1153 if (resid > 0) { 1154 crypto_cursor_copydata(&cc_in, resid, blk); 1155 exf->decrypt_last(ctx, blk, blk, resid); 1156 crypto_cursor_copyback(&cc_out, resid, blk); 1157 } 1158 } else { 1159 /* Inject the authentication data */ 1160 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1161 s.tag); 1162 } 1163 1164 out: 1165 explicit_bzero(ctx, exf->ctxsize); 1166 explicit_bzero(&s, sizeof(s)); 1167 return (error); 1168 } 1169 1170 /* 1171 * Apply a cipher and a digest to perform EtA. 1172 */ 1173 static int 1174 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1175 { 1176 int error; 1177 1178 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1179 error = swcr_encdec(ses, crp); 1180 if (error == 0) 1181 error = swcr_authcompute(ses, crp); 1182 } else { 1183 error = swcr_authcompute(ses, crp); 1184 if (error == 0) 1185 error = swcr_encdec(ses, crp); 1186 } 1187 return (error); 1188 } 1189 1190 /* 1191 * Apply a compression/decompression algorithm 1192 */ 1193 static int 1194 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1195 { 1196 const struct comp_algo *cxf; 1197 uint8_t *data, *out; 1198 int adj; 1199 uint32_t result; 1200 1201 cxf = ses->swcr_compdec.sw_cxf; 1202 1203 /* We must handle the whole buffer of data in one time 1204 * then if there is not all the data in the mbuf, we must 1205 * copy in a buffer. 1206 */ 1207 1208 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1209 if (data == NULL) 1210 return (EINVAL); 1211 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1212 data); 1213 1214 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1215 result = cxf->compress(data, crp->crp_payload_length, &out); 1216 else 1217 result = cxf->decompress(data, crp->crp_payload_length, &out); 1218 1219 free(data, M_CRYPTO_DATA); 1220 if (result == 0) 1221 return (EINVAL); 1222 crp->crp_olen = result; 1223 1224 /* Check the compressed size when doing compression */ 1225 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1226 if (result >= crp->crp_payload_length) { 1227 /* Compression was useless, we lost time */ 1228 free(out, M_CRYPTO_DATA); 1229 return (0); 1230 } 1231 } 1232 1233 /* Copy back the (de)compressed data. m_copyback is 1234 * extending the mbuf as necessary. 1235 */ 1236 crypto_copyback(crp, crp->crp_payload_start, result, out); 1237 if (result < crp->crp_payload_length) { 1238 switch (crp->crp_buf.cb_type) { 1239 case CRYPTO_BUF_MBUF: 1240 case CRYPTO_BUF_SINGLE_MBUF: 1241 adj = result - crp->crp_payload_length; 1242 m_adj(crp->crp_buf.cb_mbuf, adj); 1243 break; 1244 case CRYPTO_BUF_UIO: { 1245 struct uio *uio = crp->crp_buf.cb_uio; 1246 int ind; 1247 1248 adj = crp->crp_payload_length - result; 1249 ind = uio->uio_iovcnt - 1; 1250 1251 while (adj > 0 && ind >= 0) { 1252 if (adj < uio->uio_iov[ind].iov_len) { 1253 uio->uio_iov[ind].iov_len -= adj; 1254 break; 1255 } 1256 1257 adj -= uio->uio_iov[ind].iov_len; 1258 uio->uio_iov[ind].iov_len = 0; 1259 ind--; 1260 uio->uio_iovcnt--; 1261 } 1262 } 1263 break; 1264 case CRYPTO_BUF_VMPAGE: 1265 adj = crp->crp_payload_length - result; 1266 crp->crp_buf.cb_vm_page_len -= adj; 1267 break; 1268 default: 1269 break; 1270 } 1271 } 1272 free(out, M_CRYPTO_DATA); 1273 return 0; 1274 } 1275 1276 static int 1277 swcr_setup_cipher(struct swcr_session *ses, 1278 const struct crypto_session_params *csp) 1279 { 1280 struct swcr_encdec *swe; 1281 const struct enc_xform *txf; 1282 int error; 1283 1284 swe = &ses->swcr_encdec; 1285 txf = crypto_cipher(csp); 1286 if (csp->csp_cipher_key != NULL) { 1287 if (txf->ctxsize != 0) { 1288 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1289 M_NOWAIT); 1290 if (swe->sw_ctx == NULL) 1291 return (ENOMEM); 1292 } 1293 error = txf->setkey(swe->sw_ctx, 1294 csp->csp_cipher_key, csp->csp_cipher_klen); 1295 if (error) 1296 return (error); 1297 } 1298 swe->sw_exf = txf; 1299 return (0); 1300 } 1301 1302 static int 1303 swcr_setup_auth(struct swcr_session *ses, 1304 const struct crypto_session_params *csp) 1305 { 1306 struct swcr_auth *swa; 1307 const struct auth_hash *axf; 1308 1309 swa = &ses->swcr_auth; 1310 1311 axf = crypto_auth_hash(csp); 1312 swa->sw_axf = axf; 1313 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1314 return (EINVAL); 1315 if (csp->csp_auth_mlen == 0) 1316 swa->sw_mlen = axf->hashsize; 1317 else 1318 swa->sw_mlen = csp->csp_auth_mlen; 1319 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1320 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1321 M_NOWAIT); 1322 if (swa->sw_ictx == NULL) 1323 return (ENOBUFS); 1324 } 1325 1326 switch (csp->csp_auth_alg) { 1327 case CRYPTO_SHA1_HMAC: 1328 case CRYPTO_SHA2_224_HMAC: 1329 case CRYPTO_SHA2_256_HMAC: 1330 case CRYPTO_SHA2_384_HMAC: 1331 case CRYPTO_SHA2_512_HMAC: 1332 case CRYPTO_RIPEMD160_HMAC: 1333 swa->sw_hmac = true; 1334 if (csp->csp_auth_key != NULL) { 1335 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1336 M_NOWAIT); 1337 if (swa->sw_octx == NULL) 1338 return (ENOBUFS); 1339 hmac_init_ipad(axf, csp->csp_auth_key, 1340 csp->csp_auth_klen, swa->sw_ictx); 1341 hmac_init_opad(axf, csp->csp_auth_key, 1342 csp->csp_auth_klen, swa->sw_octx); 1343 } 1344 break; 1345 case CRYPTO_RIPEMD160: 1346 case CRYPTO_SHA1: 1347 case CRYPTO_SHA2_224: 1348 case CRYPTO_SHA2_256: 1349 case CRYPTO_SHA2_384: 1350 case CRYPTO_SHA2_512: 1351 case CRYPTO_NULL_HMAC: 1352 axf->Init(swa->sw_ictx); 1353 break; 1354 case CRYPTO_AES_NIST_GMAC: 1355 case CRYPTO_AES_CCM_CBC_MAC: 1356 case CRYPTO_POLY1305: 1357 if (csp->csp_auth_key != NULL) { 1358 axf->Init(swa->sw_ictx); 1359 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1360 csp->csp_auth_klen); 1361 } 1362 break; 1363 case CRYPTO_BLAKE2B: 1364 case CRYPTO_BLAKE2S: 1365 /* 1366 * Blake2b and Blake2s support an optional key but do 1367 * not require one. 1368 */ 1369 if (csp->csp_auth_klen == 0) 1370 axf->Init(swa->sw_ictx); 1371 else if (csp->csp_auth_key != NULL) 1372 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1373 csp->csp_auth_klen); 1374 break; 1375 } 1376 1377 if (csp->csp_mode == CSP_MODE_DIGEST) { 1378 switch (csp->csp_auth_alg) { 1379 case CRYPTO_AES_NIST_GMAC: 1380 ses->swcr_process = swcr_gmac; 1381 break; 1382 case CRYPTO_AES_CCM_CBC_MAC: 1383 ses->swcr_process = swcr_ccm_cbc_mac; 1384 break; 1385 default: 1386 ses->swcr_process = swcr_authcompute; 1387 } 1388 } 1389 1390 return (0); 1391 } 1392 1393 static int 1394 swcr_setup_aead(struct swcr_session *ses, 1395 const struct crypto_session_params *csp) 1396 { 1397 struct swcr_auth *swa; 1398 int error; 1399 1400 error = swcr_setup_cipher(ses, csp); 1401 if (error) 1402 return (error); 1403 1404 swa = &ses->swcr_auth; 1405 if (csp->csp_auth_mlen == 0) 1406 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1407 else 1408 swa->sw_mlen = csp->csp_auth_mlen; 1409 return (0); 1410 } 1411 1412 static bool 1413 swcr_auth_supported(const struct crypto_session_params *csp) 1414 { 1415 const struct auth_hash *axf; 1416 1417 axf = crypto_auth_hash(csp); 1418 if (axf == NULL) 1419 return (false); 1420 switch (csp->csp_auth_alg) { 1421 case CRYPTO_SHA1_HMAC: 1422 case CRYPTO_SHA2_224_HMAC: 1423 case CRYPTO_SHA2_256_HMAC: 1424 case CRYPTO_SHA2_384_HMAC: 1425 case CRYPTO_SHA2_512_HMAC: 1426 case CRYPTO_NULL_HMAC: 1427 case CRYPTO_RIPEMD160_HMAC: 1428 break; 1429 case CRYPTO_AES_NIST_GMAC: 1430 switch (csp->csp_auth_klen * 8) { 1431 case 128: 1432 case 192: 1433 case 256: 1434 break; 1435 default: 1436 return (false); 1437 } 1438 if (csp->csp_auth_key == NULL) 1439 return (false); 1440 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1441 return (false); 1442 break; 1443 case CRYPTO_POLY1305: 1444 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1445 return (false); 1446 break; 1447 case CRYPTO_AES_CCM_CBC_MAC: 1448 switch (csp->csp_auth_klen * 8) { 1449 case 128: 1450 case 192: 1451 case 256: 1452 break; 1453 default: 1454 return (false); 1455 } 1456 if (csp->csp_auth_key == NULL) 1457 return (false); 1458 break; 1459 } 1460 return (true); 1461 } 1462 1463 static bool 1464 swcr_cipher_supported(const struct crypto_session_params *csp) 1465 { 1466 const struct enc_xform *txf; 1467 1468 txf = crypto_cipher(csp); 1469 if (txf == NULL) 1470 return (false); 1471 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1472 txf->ivsize != csp->csp_ivlen) 1473 return (false); 1474 return (true); 1475 } 1476 1477 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1478 1479 static int 1480 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1481 { 1482 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1483 return (EINVAL); 1484 switch (csp->csp_mode) { 1485 case CSP_MODE_COMPRESS: 1486 switch (csp->csp_cipher_alg) { 1487 case CRYPTO_DEFLATE_COMP: 1488 break; 1489 default: 1490 return (EINVAL); 1491 } 1492 break; 1493 case CSP_MODE_CIPHER: 1494 switch (csp->csp_cipher_alg) { 1495 case CRYPTO_AES_NIST_GCM_16: 1496 case CRYPTO_AES_CCM_16: 1497 case CRYPTO_CHACHA20_POLY1305: 1498 case CRYPTO_XCHACHA20_POLY1305: 1499 return (EINVAL); 1500 default: 1501 if (!swcr_cipher_supported(csp)) 1502 return (EINVAL); 1503 break; 1504 } 1505 break; 1506 case CSP_MODE_DIGEST: 1507 if (!swcr_auth_supported(csp)) 1508 return (EINVAL); 1509 break; 1510 case CSP_MODE_AEAD: 1511 switch (csp->csp_cipher_alg) { 1512 case CRYPTO_AES_NIST_GCM_16: 1513 case CRYPTO_AES_CCM_16: 1514 switch (csp->csp_cipher_klen * 8) { 1515 case 128: 1516 case 192: 1517 case 256: 1518 break; 1519 default: 1520 return (EINVAL); 1521 } 1522 break; 1523 case CRYPTO_CHACHA20_POLY1305: 1524 case CRYPTO_XCHACHA20_POLY1305: 1525 break; 1526 default: 1527 return (EINVAL); 1528 } 1529 break; 1530 case CSP_MODE_ETA: 1531 /* AEAD algorithms cannot be used for EtA. */ 1532 switch (csp->csp_cipher_alg) { 1533 case CRYPTO_AES_NIST_GCM_16: 1534 case CRYPTO_AES_CCM_16: 1535 case CRYPTO_CHACHA20_POLY1305: 1536 case CRYPTO_XCHACHA20_POLY1305: 1537 return (EINVAL); 1538 } 1539 switch (csp->csp_auth_alg) { 1540 case CRYPTO_AES_NIST_GMAC: 1541 case CRYPTO_AES_CCM_CBC_MAC: 1542 return (EINVAL); 1543 } 1544 1545 if (!swcr_cipher_supported(csp) || 1546 !swcr_auth_supported(csp)) 1547 return (EINVAL); 1548 break; 1549 default: 1550 return (EINVAL); 1551 } 1552 1553 return (CRYPTODEV_PROBE_SOFTWARE); 1554 } 1555 1556 /* 1557 * Generate a new software session. 1558 */ 1559 static int 1560 swcr_newsession(device_t dev, crypto_session_t cses, 1561 const struct crypto_session_params *csp) 1562 { 1563 struct swcr_session *ses; 1564 const struct comp_algo *cxf; 1565 int error; 1566 1567 ses = crypto_get_driver_session(cses); 1568 1569 error = 0; 1570 switch (csp->csp_mode) { 1571 case CSP_MODE_COMPRESS: 1572 switch (csp->csp_cipher_alg) { 1573 case CRYPTO_DEFLATE_COMP: 1574 cxf = &comp_algo_deflate; 1575 break; 1576 #ifdef INVARIANTS 1577 default: 1578 panic("bad compression algo"); 1579 #endif 1580 } 1581 ses->swcr_compdec.sw_cxf = cxf; 1582 ses->swcr_process = swcr_compdec; 1583 break; 1584 case CSP_MODE_CIPHER: 1585 switch (csp->csp_cipher_alg) { 1586 case CRYPTO_NULL_CBC: 1587 ses->swcr_process = swcr_null; 1588 break; 1589 #ifdef INVARIANTS 1590 case CRYPTO_AES_NIST_GCM_16: 1591 case CRYPTO_AES_CCM_16: 1592 case CRYPTO_CHACHA20_POLY1305: 1593 case CRYPTO_XCHACHA20_POLY1305: 1594 panic("bad cipher algo"); 1595 #endif 1596 default: 1597 error = swcr_setup_cipher(ses, csp); 1598 if (error == 0) 1599 ses->swcr_process = swcr_encdec; 1600 } 1601 break; 1602 case CSP_MODE_DIGEST: 1603 error = swcr_setup_auth(ses, csp); 1604 break; 1605 case CSP_MODE_AEAD: 1606 switch (csp->csp_cipher_alg) { 1607 case CRYPTO_AES_NIST_GCM_16: 1608 error = swcr_setup_aead(ses, csp); 1609 if (error == 0) 1610 ses->swcr_process = swcr_gcm; 1611 break; 1612 case CRYPTO_AES_CCM_16: 1613 error = swcr_setup_aead(ses, csp); 1614 if (error == 0) 1615 ses->swcr_process = swcr_ccm; 1616 break; 1617 case CRYPTO_CHACHA20_POLY1305: 1618 case CRYPTO_XCHACHA20_POLY1305: 1619 error = swcr_setup_aead(ses, csp); 1620 if (error == 0) 1621 ses->swcr_process = swcr_chacha20_poly1305; 1622 break; 1623 #ifdef INVARIANTS 1624 default: 1625 panic("bad aead algo"); 1626 #endif 1627 } 1628 break; 1629 case CSP_MODE_ETA: 1630 #ifdef INVARIANTS 1631 switch (csp->csp_cipher_alg) { 1632 case CRYPTO_AES_NIST_GCM_16: 1633 case CRYPTO_AES_CCM_16: 1634 case CRYPTO_CHACHA20_POLY1305: 1635 case CRYPTO_XCHACHA20_POLY1305: 1636 panic("bad eta cipher algo"); 1637 } 1638 switch (csp->csp_auth_alg) { 1639 case CRYPTO_AES_NIST_GMAC: 1640 case CRYPTO_AES_CCM_CBC_MAC: 1641 panic("bad eta auth algo"); 1642 } 1643 #endif 1644 1645 error = swcr_setup_auth(ses, csp); 1646 if (error) 1647 break; 1648 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1649 /* Effectively degrade to digest mode. */ 1650 ses->swcr_process = swcr_authcompute; 1651 break; 1652 } 1653 1654 error = swcr_setup_cipher(ses, csp); 1655 if (error == 0) 1656 ses->swcr_process = swcr_eta; 1657 break; 1658 default: 1659 error = EINVAL; 1660 } 1661 1662 if (error) 1663 swcr_freesession(dev, cses); 1664 return (error); 1665 } 1666 1667 static void 1668 swcr_freesession(device_t dev, crypto_session_t cses) 1669 { 1670 struct swcr_session *ses; 1671 1672 ses = crypto_get_driver_session(cses); 1673 1674 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1675 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1676 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1677 } 1678 1679 /* 1680 * Process a software request. 1681 */ 1682 static int 1683 swcr_process(device_t dev, struct cryptop *crp, int hint) 1684 { 1685 struct swcr_session *ses; 1686 1687 ses = crypto_get_driver_session(crp->crp_session); 1688 1689 crp->crp_etype = ses->swcr_process(ses, crp); 1690 1691 crypto_done(crp); 1692 return (0); 1693 } 1694 1695 static void 1696 swcr_identify(driver_t *drv, device_t parent) 1697 { 1698 /* NB: order 10 is so we get attached after h/w devices */ 1699 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1700 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1701 panic("cryptosoft: could not attach"); 1702 } 1703 1704 static int 1705 swcr_probe(device_t dev) 1706 { 1707 device_set_desc(dev, "software crypto"); 1708 device_quiet(dev); 1709 return (BUS_PROBE_NOWILDCARD); 1710 } 1711 1712 static int 1713 swcr_attach(device_t dev) 1714 { 1715 1716 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1717 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1718 if (swcr_id < 0) { 1719 device_printf(dev, "cannot initialize!"); 1720 return (ENXIO); 1721 } 1722 1723 return (0); 1724 } 1725 1726 static int 1727 swcr_detach(device_t dev) 1728 { 1729 crypto_unregister_all(swcr_id); 1730 return 0; 1731 } 1732 1733 static device_method_t swcr_methods[] = { 1734 DEVMETHOD(device_identify, swcr_identify), 1735 DEVMETHOD(device_probe, swcr_probe), 1736 DEVMETHOD(device_attach, swcr_attach), 1737 DEVMETHOD(device_detach, swcr_detach), 1738 1739 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1740 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1741 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1742 DEVMETHOD(cryptodev_process, swcr_process), 1743 1744 {0, 0}, 1745 }; 1746 1747 static driver_t swcr_driver = { 1748 "cryptosoft", 1749 swcr_methods, 1750 0, /* NB: no softc */ 1751 }; 1752 static devclass_t swcr_devclass; 1753 1754 /* 1755 * NB: We explicitly reference the crypto module so we 1756 * get the necessary ordering when built as a loadable 1757 * module. This is required because we bundle the crypto 1758 * module code together with the cryptosoft driver (otherwise 1759 * normal module dependencies would handle things). 1760 */ 1761 extern int crypto_modevent(struct module *, int, void *); 1762 /* XXX where to attach */ 1763 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1764 MODULE_VERSION(cryptosoft, 1); 1765 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1766