1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/lock.h> 48 #include <sys/rwlock.h> 49 #include <sys/endian.h> 50 #include <sys/limits.h> 51 #include <sys/mutex.h> 52 53 #include <crypto/sha1.h> 54 #include <opencrypto/rmd160.h> 55 56 #include <opencrypto/cryptodev.h> 57 #include <opencrypto/xform.h> 58 59 #include <sys/kobj.h> 60 #include <sys/bus.h> 61 #include "cryptodev_if.h" 62 63 struct swcr_auth { 64 void *sw_ictx; 65 void *sw_octx; 66 const struct auth_hash *sw_axf; 67 uint16_t sw_mlen; 68 }; 69 70 struct swcr_encdec { 71 void *sw_ctx; 72 const struct enc_xform *sw_exf; 73 }; 74 75 struct swcr_compdec { 76 const struct comp_algo *sw_cxf; 77 }; 78 79 struct swcr_session { 80 struct mtx swcr_lock; 81 int (*swcr_process)(struct swcr_session *, struct cryptop *); 82 83 struct swcr_auth swcr_auth; 84 struct swcr_encdec swcr_encdec; 85 struct swcr_compdec swcr_compdec; 86 }; 87 88 static int32_t swcr_id; 89 90 static void swcr_freesession(device_t dev, crypto_session_t cses); 91 92 /* Used for CRYPTO_NULL_CBC. */ 93 static int 94 swcr_null(struct swcr_session *ses, struct cryptop *crp) 95 { 96 97 return (0); 98 } 99 100 /* 101 * Apply a symmetric encryption/decryption algorithm. 102 */ 103 static int 104 swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 105 { 106 unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 107 unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 108 const struct crypto_session_params *csp; 109 const struct enc_xform *exf; 110 const struct swcr_encdec *sw; 111 void *ctx; 112 size_t inlen, outlen; 113 int i, blks, resid; 114 struct crypto_buffer_cursor cc_in, cc_out; 115 const unsigned char *inblk; 116 unsigned char *outblk; 117 int error; 118 bool encrypting; 119 120 error = 0; 121 122 sw = &ses->swcr_encdec; 123 exf = sw->sw_exf; 124 csp = crypto_get_params(crp->crp_session); 125 126 if (exf->native_blocksize == 0) { 127 /* Check for non-padded data */ 128 if ((crp->crp_payload_length % exf->blocksize) != 0) 129 return (EINVAL); 130 131 blks = exf->blocksize; 132 } else 133 blks = exf->native_blocksize; 134 135 if (exf == &enc_xform_aes_icm && 136 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 137 return (EINVAL); 138 139 ctx = __builtin_alloca(exf->ctxsize); 140 if (crp->crp_cipher_key != NULL) { 141 error = exf->setkey(ctx, crp->crp_cipher_key, 142 csp->csp_cipher_klen); 143 if (error) 144 return (error); 145 } else 146 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 147 148 crypto_read_iv(crp, iv); 149 150 if (exf->reinit) { 151 /* 152 * xforms that provide a reinit method perform all IV 153 * handling themselves. 154 */ 155 exf->reinit(ctx, iv, csp->csp_ivlen); 156 } 157 158 ivp = iv; 159 160 crypto_cursor_init(&cc_in, &crp->crp_buf); 161 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 162 inblk = crypto_cursor_segment(&cc_in, &inlen); 163 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 164 crypto_cursor_init(&cc_out, &crp->crp_obuf); 165 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 166 } else 167 cc_out = cc_in; 168 outblk = crypto_cursor_segment(&cc_out, &outlen); 169 170 resid = crp->crp_payload_length; 171 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 172 173 /* 174 * Loop through encrypting blocks. 'inlen' is the remaining 175 * length of the current segment in the input buffer. 176 * 'outlen' is the remaining length of current segment in the 177 * output buffer. 178 */ 179 while (resid >= blks) { 180 /* 181 * If the current block is not contained within the 182 * current input/output segment, use 'blk' as a local 183 * buffer. 184 */ 185 if (inlen < blks) { 186 crypto_cursor_copydata(&cc_in, blks, blk); 187 inblk = blk; 188 } 189 if (outlen < blks) 190 outblk = blk; 191 192 /* 193 * Ciphers without a 'reinit' hook are assumed to be 194 * used in CBC mode where the chaining is done here. 195 */ 196 if (exf->reinit != NULL) { 197 if (encrypting) 198 exf->encrypt(ctx, inblk, outblk); 199 else 200 exf->decrypt(ctx, inblk, outblk); 201 } else if (encrypting) { 202 /* XOR with previous block */ 203 for (i = 0; i < blks; i++) 204 outblk[i] = inblk[i] ^ ivp[i]; 205 206 exf->encrypt(ctx, outblk, outblk); 207 208 /* 209 * Keep encrypted block for XOR'ing 210 * with next block 211 */ 212 memcpy(iv, outblk, blks); 213 ivp = iv; 214 } else { /* decrypt */ 215 /* 216 * Keep encrypted block for XOR'ing 217 * with next block 218 */ 219 nivp = (ivp == iv) ? iv2 : iv; 220 memcpy(nivp, inblk, blks); 221 222 exf->decrypt(ctx, inblk, outblk); 223 224 /* XOR with previous block */ 225 for (i = 0; i < blks; i++) 226 outblk[i] ^= ivp[i]; 227 228 ivp = nivp; 229 } 230 231 if (inlen < blks) { 232 inblk = crypto_cursor_segment(&cc_in, &inlen); 233 } else { 234 crypto_cursor_advance(&cc_in, blks); 235 inlen -= blks; 236 inblk += blks; 237 } 238 239 if (outlen < blks) { 240 crypto_cursor_copyback(&cc_out, blks, blk); 241 outblk = crypto_cursor_segment(&cc_out, &outlen); 242 } else { 243 crypto_cursor_advance(&cc_out, blks); 244 outlen -= blks; 245 outblk += blks; 246 } 247 248 resid -= blks; 249 } 250 251 /* Handle trailing partial block for stream ciphers. */ 252 if (resid > 0) { 253 KASSERT(exf->native_blocksize != 0, 254 ("%s: partial block of %d bytes for cipher %s", 255 __func__, i, exf->name)); 256 KASSERT(exf->reinit != NULL, 257 ("%s: partial block cipher %s without reinit hook", 258 __func__, exf->name)); 259 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 260 261 inblk = crypto_cursor_segment(&cc_in, &inlen); 262 outblk = crypto_cursor_segment(&cc_out, &outlen); 263 if (inlen < resid) { 264 crypto_cursor_copydata(&cc_in, resid, blk); 265 inblk = blk; 266 } 267 if (outlen < resid) 268 outblk = blk; 269 if (encrypting) 270 exf->encrypt_last(ctx, inblk, outblk, 271 resid); 272 else 273 exf->decrypt_last(ctx, inblk, outblk, 274 resid); 275 if (outlen < resid) 276 crypto_cursor_copyback(&cc_out, resid, blk); 277 } 278 279 explicit_bzero(ctx, exf->ctxsize); 280 explicit_bzero(blk, sizeof(blk)); 281 explicit_bzero(iv, sizeof(iv)); 282 explicit_bzero(iv2, sizeof(iv2)); 283 return (0); 284 } 285 286 static void 287 swcr_authprepare(const struct auth_hash *axf, struct swcr_auth *sw, 288 const uint8_t *key, int klen) 289 { 290 291 switch (axf->type) { 292 case CRYPTO_SHA1_HMAC: 293 case CRYPTO_SHA2_224_HMAC: 294 case CRYPTO_SHA2_256_HMAC: 295 case CRYPTO_SHA2_384_HMAC: 296 case CRYPTO_SHA2_512_HMAC: 297 case CRYPTO_NULL_HMAC: 298 case CRYPTO_RIPEMD160_HMAC: 299 hmac_init_ipad(axf, key, klen, sw->sw_ictx); 300 hmac_init_opad(axf, key, klen, sw->sw_octx); 301 break; 302 case CRYPTO_POLY1305: 303 case CRYPTO_BLAKE2B: 304 case CRYPTO_BLAKE2S: 305 axf->Setkey(sw->sw_ictx, key, klen); 306 axf->Init(sw->sw_ictx); 307 break; 308 default: 309 panic("%s: algorithm %d doesn't use keys", __func__, axf->type); 310 } 311 } 312 313 /* 314 * Compute or verify hash. 315 */ 316 static int 317 swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 318 { 319 u_char aalg[HASH_MAX_LEN]; 320 const struct crypto_session_params *csp; 321 struct swcr_auth *sw; 322 const struct auth_hash *axf; 323 union authctx ctx; 324 int err; 325 326 sw = &ses->swcr_auth; 327 328 axf = sw->sw_axf; 329 330 csp = crypto_get_params(crp->crp_session); 331 if (crp->crp_auth_key != NULL) { 332 swcr_authprepare(axf, sw, crp->crp_auth_key, 333 csp->csp_auth_klen); 334 } 335 336 bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 337 338 if (crp->crp_aad != NULL) 339 err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 340 else 341 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 342 axf->Update, &ctx); 343 if (err) 344 goto out; 345 346 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 347 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 348 err = crypto_apply_buf(&crp->crp_obuf, 349 crp->crp_payload_output_start, crp->crp_payload_length, 350 axf->Update, &ctx); 351 else 352 err = crypto_apply(crp, crp->crp_payload_start, 353 crp->crp_payload_length, axf->Update, &ctx); 354 if (err) 355 goto out; 356 357 if (csp->csp_flags & CSP_F_ESN) 358 axf->Update(&ctx, crp->crp_esn, 4); 359 360 axf->Final(aalg, &ctx); 361 if (sw->sw_octx != NULL) { 362 bcopy(sw->sw_octx, &ctx, axf->ctxsize); 363 axf->Update(&ctx, aalg, axf->hashsize); 364 axf->Final(aalg, &ctx); 365 } 366 367 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 368 u_char uaalg[HASH_MAX_LEN]; 369 370 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 371 if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 372 err = EBADMSG; 373 explicit_bzero(uaalg, sizeof(uaalg)); 374 } else { 375 /* Inject the authentication data */ 376 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 377 } 378 explicit_bzero(aalg, sizeof(aalg)); 379 out: 380 explicit_bzero(&ctx, sizeof(ctx)); 381 return (err); 382 } 383 384 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 385 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 386 387 static int 388 swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 389 { 390 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 391 u_char *blk = (u_char *)blkbuf; 392 u_char tag[GMAC_DIGEST_LEN]; 393 u_char iv[AES_BLOCK_LEN]; 394 struct crypto_buffer_cursor cc; 395 const u_char *inblk; 396 union authctx ctx; 397 struct swcr_auth *swa; 398 const struct auth_hash *axf; 399 uint32_t *blkp; 400 size_t len; 401 int blksz, error, ivlen, resid; 402 403 swa = &ses->swcr_auth; 404 axf = swa->sw_axf; 405 406 bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 407 blksz = GMAC_BLOCK_LEN; 408 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 409 __func__)); 410 411 /* Initialize the IV */ 412 ivlen = AES_GCM_IV_LEN; 413 crypto_read_iv(crp, iv); 414 415 axf->Reinit(&ctx, iv, ivlen); 416 crypto_cursor_init(&cc, &crp->crp_buf); 417 crypto_cursor_advance(&cc, crp->crp_payload_start); 418 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 419 inblk = crypto_cursor_segment(&cc, &len); 420 if (len >= blksz) { 421 len = rounddown(MIN(len, resid), blksz); 422 crypto_cursor_advance(&cc, len); 423 } else { 424 len = blksz; 425 crypto_cursor_copydata(&cc, len, blk); 426 inblk = blk; 427 } 428 axf->Update(&ctx, inblk, len); 429 } 430 if (resid > 0) { 431 memset(blk, 0, blksz); 432 crypto_cursor_copydata(&cc, resid, blk); 433 axf->Update(&ctx, blk, blksz); 434 } 435 436 /* length block */ 437 memset(blk, 0, blksz); 438 blkp = (uint32_t *)blk + 1; 439 *blkp = htobe32(crp->crp_payload_length * 8); 440 axf->Update(&ctx, blk, blksz); 441 442 /* Finalize MAC */ 443 axf->Final(tag, &ctx); 444 445 error = 0; 446 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 447 u_char tag2[GMAC_DIGEST_LEN]; 448 449 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 450 tag2); 451 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 452 error = EBADMSG; 453 explicit_bzero(tag2, sizeof(tag2)); 454 } else { 455 /* Inject the authentication data */ 456 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 457 } 458 explicit_bzero(blkbuf, sizeof(blkbuf)); 459 explicit_bzero(tag, sizeof(tag)); 460 explicit_bzero(iv, sizeof(iv)); 461 return (error); 462 } 463 464 static int 465 swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 466 { 467 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 468 u_char *blk = (u_char *)blkbuf; 469 u_char tag[GMAC_DIGEST_LEN]; 470 struct crypto_buffer_cursor cc_in, cc_out; 471 const u_char *inblk; 472 u_char *outblk; 473 const struct swcr_auth *swa; 474 const struct swcr_encdec *swe; 475 const struct enc_xform *exf; 476 void *ctx; 477 uint32_t *blkp; 478 size_t len; 479 int blksz, error, ivlen, r, resid; 480 481 swa = &ses->swcr_auth; 482 swe = &ses->swcr_encdec; 483 exf = swe->sw_exf; 484 blksz = GMAC_BLOCK_LEN; 485 KASSERT(blksz == exf->native_blocksize, 486 ("%s: blocksize mismatch", __func__)); 487 488 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 489 return (EINVAL); 490 491 ivlen = AES_GCM_IV_LEN; 492 493 ctx = __builtin_alloca(exf->ctxsize); 494 if (crp->crp_cipher_key != NULL) 495 exf->setkey(ctx, crp->crp_cipher_key, 496 crypto_get_params(crp->crp_session)->csp_cipher_klen); 497 else 498 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 499 exf->reinit(ctx, crp->crp_iv, ivlen); 500 501 /* Supply MAC with AAD */ 502 if (crp->crp_aad != NULL) { 503 len = rounddown(crp->crp_aad_length, blksz); 504 if (len != 0) 505 exf->update(ctx, crp->crp_aad, len); 506 if (crp->crp_aad_length != len) { 507 memset(blk, 0, blksz); 508 memcpy(blk, (char *)crp->crp_aad + len, 509 crp->crp_aad_length - len); 510 exf->update(ctx, blk, blksz); 511 } 512 } else { 513 crypto_cursor_init(&cc_in, &crp->crp_buf); 514 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 515 for (resid = crp->crp_aad_length; resid >= blksz; 516 resid -= len) { 517 inblk = crypto_cursor_segment(&cc_in, &len); 518 if (len >= blksz) { 519 len = rounddown(MIN(len, resid), blksz); 520 crypto_cursor_advance(&cc_in, len); 521 } else { 522 len = blksz; 523 crypto_cursor_copydata(&cc_in, len, blk); 524 inblk = blk; 525 } 526 exf->update(ctx, inblk, len); 527 } 528 if (resid > 0) { 529 memset(blk, 0, blksz); 530 crypto_cursor_copydata(&cc_in, resid, blk); 531 exf->update(ctx, blk, blksz); 532 } 533 } 534 535 /* Do encryption with MAC */ 536 crypto_cursor_init(&cc_in, &crp->crp_buf); 537 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 538 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 539 crypto_cursor_init(&cc_out, &crp->crp_obuf); 540 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 541 } else 542 cc_out = cc_in; 543 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 544 inblk = crypto_cursor_segment(&cc_in, &len); 545 if (len < blksz) { 546 crypto_cursor_copydata(&cc_in, blksz, blk); 547 inblk = blk; 548 } else { 549 crypto_cursor_advance(&cc_in, blksz); 550 } 551 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 552 outblk = crypto_cursor_segment(&cc_out, &len); 553 if (len < blksz) 554 outblk = blk; 555 exf->encrypt(ctx, inblk, outblk); 556 exf->update(ctx, outblk, blksz); 557 if (outblk == blk) 558 crypto_cursor_copyback(&cc_out, blksz, blk); 559 else 560 crypto_cursor_advance(&cc_out, blksz); 561 } else { 562 exf->update(ctx, inblk, blksz); 563 } 564 } 565 if (resid > 0) { 566 crypto_cursor_copydata(&cc_in, resid, blk); 567 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 568 exf->encrypt_last(ctx, blk, blk, resid); 569 crypto_cursor_copyback(&cc_out, resid, blk); 570 } 571 exf->update(ctx, blk, resid); 572 } 573 574 /* length block */ 575 memset(blk, 0, blksz); 576 blkp = (uint32_t *)blk + 1; 577 *blkp = htobe32(crp->crp_aad_length * 8); 578 blkp = (uint32_t *)blk + 3; 579 *blkp = htobe32(crp->crp_payload_length * 8); 580 exf->update(ctx, blk, blksz); 581 582 /* Finalize MAC */ 583 exf->final(tag, ctx); 584 585 /* Validate tag */ 586 error = 0; 587 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 588 u_char tag2[GMAC_DIGEST_LEN]; 589 590 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 591 592 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 593 explicit_bzero(tag2, sizeof(tag2)); 594 if (r != 0) { 595 error = EBADMSG; 596 goto out; 597 } 598 599 /* tag matches, decrypt data */ 600 crypto_cursor_init(&cc_in, &crp->crp_buf); 601 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 602 for (resid = crp->crp_payload_length; resid > blksz; 603 resid -= blksz) { 604 inblk = crypto_cursor_segment(&cc_in, &len); 605 if (len < blksz) { 606 crypto_cursor_copydata(&cc_in, blksz, blk); 607 inblk = blk; 608 } else 609 crypto_cursor_advance(&cc_in, blksz); 610 outblk = crypto_cursor_segment(&cc_out, &len); 611 if (len < blksz) 612 outblk = blk; 613 exf->decrypt(ctx, inblk, outblk); 614 if (outblk == blk) 615 crypto_cursor_copyback(&cc_out, blksz, blk); 616 else 617 crypto_cursor_advance(&cc_out, blksz); 618 } 619 if (resid > 0) { 620 crypto_cursor_copydata(&cc_in, resid, blk); 621 exf->decrypt_last(ctx, blk, blk, resid); 622 crypto_cursor_copyback(&cc_out, resid, blk); 623 } 624 } else { 625 /* Inject the authentication data */ 626 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 627 } 628 629 out: 630 explicit_bzero(ctx, exf->ctxsize); 631 explicit_bzero(blkbuf, sizeof(blkbuf)); 632 explicit_bzero(tag, sizeof(tag)); 633 634 return (error); 635 } 636 637 static void 638 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 639 u_int data_length, u_int tag_length, uint8_t *b0) 640 { 641 uint8_t *bp; 642 uint8_t flags, L; 643 644 KASSERT(nonce_length >= 7 && nonce_length <= 13, 645 ("nonce_length must be between 7 and 13 bytes")); 646 647 /* 648 * Need to determine the L field value. This is the number of 649 * bytes needed to specify the length of the message; the length 650 * is whatever is left in the 16 bytes after specifying flags and 651 * the nonce. 652 */ 653 L = 15 - nonce_length; 654 655 flags = ((aad_length > 0) << 6) + 656 (((tag_length - 2) / 2) << 3) + 657 L - 1; 658 659 /* 660 * Now we need to set up the first block, which has flags, nonce, 661 * and the message length. 662 */ 663 b0[0] = flags; 664 memcpy(b0 + 1, nonce, nonce_length); 665 bp = b0 + 1 + nonce_length; 666 667 /* Need to copy L' [aka L-1] bytes of data_length */ 668 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 669 *dst = data_length; 670 data_length >>= 8; 671 } 672 } 673 674 /* NB: OCF only supports AAD lengths < 2^32. */ 675 static int 676 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 677 { 678 if (aad_length < ((1 << 16) - (1 << 8))) { 679 be16enc(blk, aad_length); 680 return (sizeof(uint16_t)); 681 } else { 682 blk[0] = 0xff; 683 blk[1] = 0xfe; 684 be32enc(blk + 2, aad_length); 685 return (2 + sizeof(uint32_t)); 686 } 687 } 688 689 static int 690 swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 691 { 692 u_char iv[AES_BLOCK_LEN]; 693 u_char blk[CCM_CBC_BLOCK_LEN]; 694 u_char tag[AES_CBC_MAC_HASH_LEN]; 695 union authctx ctx; 696 const struct crypto_session_params *csp; 697 struct swcr_auth *swa; 698 const struct auth_hash *axf; 699 int error, ivlen, len; 700 701 csp = crypto_get_params(crp->crp_session); 702 swa = &ses->swcr_auth; 703 axf = swa->sw_axf; 704 705 bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 706 707 /* Initialize the IV */ 708 ivlen = csp->csp_ivlen; 709 crypto_read_iv(crp, iv); 710 711 /* Supply MAC with IV */ 712 axf->Reinit(&ctx, crp->crp_iv, ivlen); 713 714 /* Supply MAC with b0. */ 715 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 716 swa->sw_mlen, blk); 717 axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 718 719 len = build_ccm_aad_length(crp->crp_payload_length, blk); 720 axf->Update(&ctx, blk, len); 721 722 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 723 axf->Update, &ctx); 724 725 /* Finalize MAC */ 726 axf->Final(tag, &ctx); 727 728 error = 0; 729 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 730 u_char tag2[AES_CBC_MAC_HASH_LEN]; 731 732 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 733 tag2); 734 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 735 error = EBADMSG; 736 explicit_bzero(tag2, sizeof(tag)); 737 } else { 738 /* Inject the authentication data */ 739 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 740 } 741 explicit_bzero(tag, sizeof(tag)); 742 explicit_bzero(blk, sizeof(blk)); 743 explicit_bzero(iv, sizeof(iv)); 744 return (error); 745 } 746 747 static int 748 swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 749 { 750 const struct crypto_session_params *csp; 751 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 752 u_char *blk = (u_char *)blkbuf; 753 u_char tag[AES_CBC_MAC_HASH_LEN]; 754 struct crypto_buffer_cursor cc_in, cc_out; 755 const u_char *inblk; 756 u_char *outblk; 757 const struct swcr_auth *swa; 758 const struct swcr_encdec *swe; 759 const struct enc_xform *exf; 760 void *ctx; 761 size_t len; 762 int blksz, error, ivlen, r, resid; 763 764 csp = crypto_get_params(crp->crp_session); 765 swa = &ses->swcr_auth; 766 swe = &ses->swcr_encdec; 767 exf = swe->sw_exf; 768 blksz = AES_BLOCK_LEN; 769 KASSERT(blksz == exf->native_blocksize, 770 ("%s: blocksize mismatch", __func__)); 771 772 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 773 return (EMSGSIZE); 774 775 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 776 return (EINVAL); 777 778 ivlen = csp->csp_ivlen; 779 780 ctx = __builtin_alloca(exf->ctxsize); 781 if (crp->crp_cipher_key != NULL) 782 exf->setkey(ctx, crp->crp_cipher_key, 783 crypto_get_params(crp->crp_session)->csp_cipher_klen); 784 else 785 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 786 exf->reinit(ctx, crp->crp_iv, ivlen); 787 788 /* Supply MAC with b0. */ 789 _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 790 "blkbuf too small for b0"); 791 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 792 crp->crp_payload_length, swa->sw_mlen, blk); 793 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 794 795 /* Supply MAC with AAD */ 796 if (crp->crp_aad_length != 0) { 797 len = build_ccm_aad_length(crp->crp_aad_length, blk); 798 exf->update(ctx, blk, len); 799 if (crp->crp_aad != NULL) 800 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 801 else 802 crypto_apply(crp, crp->crp_aad_start, 803 crp->crp_aad_length, exf->update, ctx); 804 805 /* Pad the AAD (including length field) to a full block. */ 806 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 807 if (len != 0) { 808 len = CCM_CBC_BLOCK_LEN - len; 809 memset(blk, 0, CCM_CBC_BLOCK_LEN); 810 exf->update(ctx, blk, len); 811 } 812 } 813 814 /* Do encryption/decryption with MAC */ 815 crypto_cursor_init(&cc_in, &crp->crp_buf); 816 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 817 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 818 crypto_cursor_init(&cc_out, &crp->crp_obuf); 819 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 820 } else 821 cc_out = cc_in; 822 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 823 inblk = crypto_cursor_segment(&cc_in, &len); 824 if (len < blksz) { 825 crypto_cursor_copydata(&cc_in, blksz, blk); 826 inblk = blk; 827 } else 828 crypto_cursor_advance(&cc_in, blksz); 829 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 830 outblk = crypto_cursor_segment(&cc_out, &len); 831 if (len < blksz) 832 outblk = blk; 833 exf->update(ctx, inblk, blksz); 834 exf->encrypt(ctx, inblk, outblk); 835 if (outblk == blk) 836 crypto_cursor_copyback(&cc_out, blksz, blk); 837 else 838 crypto_cursor_advance(&cc_out, blksz); 839 } else { 840 /* 841 * One of the problems with CCM+CBC is that 842 * the authentication is done on the 843 * unencrypted data. As a result, we have to 844 * decrypt the data twice: once to generate 845 * the tag and a second time after the tag is 846 * verified. 847 */ 848 exf->decrypt(ctx, inblk, blk); 849 exf->update(ctx, blk, blksz); 850 } 851 } 852 if (resid > 0) { 853 crypto_cursor_copydata(&cc_in, resid, blk); 854 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 855 exf->update(ctx, blk, resid); 856 exf->encrypt_last(ctx, blk, blk, resid); 857 crypto_cursor_copyback(&cc_out, resid, blk); 858 } else { 859 exf->decrypt_last(ctx, blk, blk, resid); 860 exf->update(ctx, blk, resid); 861 } 862 } 863 864 /* Finalize MAC */ 865 exf->final(tag, ctx); 866 867 /* Validate tag */ 868 error = 0; 869 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 870 u_char tag2[AES_CBC_MAC_HASH_LEN]; 871 872 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 873 tag2); 874 875 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 876 explicit_bzero(tag2, sizeof(tag2)); 877 if (r != 0) { 878 error = EBADMSG; 879 goto out; 880 } 881 882 /* tag matches, decrypt data */ 883 exf->reinit(ctx, crp->crp_iv, ivlen); 884 crypto_cursor_init(&cc_in, &crp->crp_buf); 885 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 886 for (resid = crp->crp_payload_length; resid > blksz; 887 resid -= blksz) { 888 inblk = crypto_cursor_segment(&cc_in, &len); 889 if (len < blksz) { 890 crypto_cursor_copydata(&cc_in, blksz, blk); 891 inblk = blk; 892 } else 893 crypto_cursor_advance(&cc_in, blksz); 894 outblk = crypto_cursor_segment(&cc_out, &len); 895 if (len < blksz) 896 outblk = blk; 897 exf->decrypt(ctx, inblk, outblk); 898 if (outblk == blk) 899 crypto_cursor_copyback(&cc_out, blksz, blk); 900 else 901 crypto_cursor_advance(&cc_out, blksz); 902 } 903 if (resid > 0) { 904 crypto_cursor_copydata(&cc_in, resid, blk); 905 exf->decrypt_last(ctx, blk, blk, resid); 906 crypto_cursor_copyback(&cc_out, resid, blk); 907 } 908 } else { 909 /* Inject the authentication data */ 910 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 911 } 912 913 out: 914 explicit_bzero(ctx, exf->ctxsize); 915 explicit_bzero(blkbuf, sizeof(blkbuf)); 916 explicit_bzero(tag, sizeof(tag)); 917 return (error); 918 } 919 920 static int 921 swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) 922 { 923 const struct crypto_session_params *csp; 924 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 925 u_char *blk = (u_char *)blkbuf; 926 u_char tag[POLY1305_HASH_LEN]; 927 struct crypto_buffer_cursor cc_in, cc_out; 928 const u_char *inblk; 929 u_char *outblk; 930 uint64_t *blkp; 931 const struct swcr_auth *swa; 932 const struct swcr_encdec *swe; 933 const struct enc_xform *exf; 934 void *ctx; 935 size_t len; 936 int blksz, error, r, resid; 937 938 swa = &ses->swcr_auth; 939 swe = &ses->swcr_encdec; 940 exf = swe->sw_exf; 941 blksz = exf->native_blocksize; 942 KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 943 944 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 945 return (EINVAL); 946 947 csp = crypto_get_params(crp->crp_session); 948 949 ctx = __builtin_alloca(exf->ctxsize); 950 if (crp->crp_cipher_key != NULL) 951 exf->setkey(ctx, crp->crp_cipher_key, 952 csp->csp_cipher_klen); 953 else 954 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 955 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 956 957 /* Supply MAC with AAD */ 958 if (crp->crp_aad != NULL) 959 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 960 else 961 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 962 exf->update, ctx); 963 if (crp->crp_aad_length % 16 != 0) { 964 /* padding1 */ 965 memset(blk, 0, 16); 966 exf->update(ctx, blk, 16 - crp->crp_aad_length % 16); 967 } 968 969 /* Do encryption with MAC */ 970 crypto_cursor_init(&cc_in, &crp->crp_buf); 971 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 972 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 973 crypto_cursor_init(&cc_out, &crp->crp_obuf); 974 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 975 } else 976 cc_out = cc_in; 977 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 978 inblk = crypto_cursor_segment(&cc_in, &len); 979 if (len < blksz) { 980 crypto_cursor_copydata(&cc_in, blksz, blk); 981 inblk = blk; 982 } else 983 crypto_cursor_advance(&cc_in, blksz); 984 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 985 outblk = crypto_cursor_segment(&cc_out, &len); 986 if (len < blksz) 987 outblk = blk; 988 exf->encrypt(ctx, inblk, outblk); 989 exf->update(ctx, outblk, blksz); 990 if (outblk == blk) 991 crypto_cursor_copyback(&cc_out, blksz, blk); 992 else 993 crypto_cursor_advance(&cc_out, blksz); 994 } else { 995 exf->update(ctx, inblk, blksz); 996 } 997 } 998 if (resid > 0) { 999 crypto_cursor_copydata(&cc_in, resid, blk); 1000 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1001 exf->encrypt_last(ctx, blk, blk, resid); 1002 crypto_cursor_copyback(&cc_out, resid, blk); 1003 } 1004 exf->update(ctx, blk, resid); 1005 if (resid % 16 != 0) { 1006 /* padding2 */ 1007 memset(blk, 0, 16); 1008 exf->update(ctx, blk, 16 - resid % 16); 1009 } 1010 } 1011 1012 /* lengths */ 1013 blkp = (uint64_t *)blk; 1014 blkp[0] = htole64(crp->crp_aad_length); 1015 blkp[1] = htole64(crp->crp_payload_length); 1016 exf->update(ctx, blk, sizeof(uint64_t) * 2); 1017 1018 /* Finalize MAC */ 1019 exf->final(tag, ctx); 1020 1021 /* Validate tag */ 1022 error = 0; 1023 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1024 u_char tag2[POLY1305_HASH_LEN]; 1025 1026 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 1027 1028 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 1029 explicit_bzero(tag2, sizeof(tag2)); 1030 if (r != 0) { 1031 error = EBADMSG; 1032 goto out; 1033 } 1034 1035 /* tag matches, decrypt data */ 1036 crypto_cursor_init(&cc_in, &crp->crp_buf); 1037 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1038 for (resid = crp->crp_payload_length; resid > blksz; 1039 resid -= blksz) { 1040 inblk = crypto_cursor_segment(&cc_in, &len); 1041 if (len < blksz) { 1042 crypto_cursor_copydata(&cc_in, blksz, blk); 1043 inblk = blk; 1044 } else 1045 crypto_cursor_advance(&cc_in, blksz); 1046 outblk = crypto_cursor_segment(&cc_out, &len); 1047 if (len < blksz) 1048 outblk = blk; 1049 exf->decrypt(ctx, inblk, outblk); 1050 if (outblk == blk) 1051 crypto_cursor_copyback(&cc_out, blksz, blk); 1052 else 1053 crypto_cursor_advance(&cc_out, blksz); 1054 } 1055 if (resid > 0) { 1056 crypto_cursor_copydata(&cc_in, resid, blk); 1057 exf->decrypt_last(ctx, blk, blk, resid); 1058 crypto_cursor_copyback(&cc_out, resid, blk); 1059 } 1060 } else { 1061 /* Inject the authentication data */ 1062 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 1063 } 1064 1065 out: 1066 explicit_bzero(ctx, exf->ctxsize); 1067 explicit_bzero(blkbuf, sizeof(blkbuf)); 1068 explicit_bzero(tag, sizeof(tag)); 1069 return (error); 1070 } 1071 1072 /* 1073 * Apply a cipher and a digest to perform EtA. 1074 */ 1075 static int 1076 swcr_eta(struct swcr_session *ses, struct cryptop *crp) 1077 { 1078 int error; 1079 1080 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1081 error = swcr_encdec(ses, crp); 1082 if (error == 0) 1083 error = swcr_authcompute(ses, crp); 1084 } else { 1085 error = swcr_authcompute(ses, crp); 1086 if (error == 0) 1087 error = swcr_encdec(ses, crp); 1088 } 1089 return (error); 1090 } 1091 1092 /* 1093 * Apply a compression/decompression algorithm 1094 */ 1095 static int 1096 swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 1097 { 1098 const struct comp_algo *cxf; 1099 uint8_t *data, *out; 1100 int adj; 1101 uint32_t result; 1102 1103 cxf = ses->swcr_compdec.sw_cxf; 1104 1105 /* We must handle the whole buffer of data in one time 1106 * then if there is not all the data in the mbuf, we must 1107 * copy in a buffer. 1108 */ 1109 1110 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1111 if (data == NULL) 1112 return (EINVAL); 1113 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1114 data); 1115 1116 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1117 result = cxf->compress(data, crp->crp_payload_length, &out); 1118 else 1119 result = cxf->decompress(data, crp->crp_payload_length, &out); 1120 1121 free(data, M_CRYPTO_DATA); 1122 if (result == 0) 1123 return (EINVAL); 1124 crp->crp_olen = result; 1125 1126 /* Check the compressed size when doing compression */ 1127 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1128 if (result >= crp->crp_payload_length) { 1129 /* Compression was useless, we lost time */ 1130 free(out, M_CRYPTO_DATA); 1131 return (0); 1132 } 1133 } 1134 1135 /* Copy back the (de)compressed data. m_copyback is 1136 * extending the mbuf as necessary. 1137 */ 1138 crypto_copyback(crp, crp->crp_payload_start, result, out); 1139 if (result < crp->crp_payload_length) { 1140 switch (crp->crp_buf.cb_type) { 1141 case CRYPTO_BUF_MBUF: 1142 case CRYPTO_BUF_SINGLE_MBUF: 1143 adj = result - crp->crp_payload_length; 1144 m_adj(crp->crp_buf.cb_mbuf, adj); 1145 break; 1146 case CRYPTO_BUF_UIO: { 1147 struct uio *uio = crp->crp_buf.cb_uio; 1148 int ind; 1149 1150 adj = crp->crp_payload_length - result; 1151 ind = uio->uio_iovcnt - 1; 1152 1153 while (adj > 0 && ind >= 0) { 1154 if (adj < uio->uio_iov[ind].iov_len) { 1155 uio->uio_iov[ind].iov_len -= adj; 1156 break; 1157 } 1158 1159 adj -= uio->uio_iov[ind].iov_len; 1160 uio->uio_iov[ind].iov_len = 0; 1161 ind--; 1162 uio->uio_iovcnt--; 1163 } 1164 } 1165 break; 1166 case CRYPTO_BUF_VMPAGE: 1167 adj = crp->crp_payload_length - result; 1168 crp->crp_buf.cb_vm_page_len -= adj; 1169 break; 1170 default: 1171 break; 1172 } 1173 } 1174 free(out, M_CRYPTO_DATA); 1175 return 0; 1176 } 1177 1178 static int 1179 swcr_setup_cipher(struct swcr_session *ses, 1180 const struct crypto_session_params *csp) 1181 { 1182 struct swcr_encdec *swe; 1183 const struct enc_xform *txf; 1184 int error; 1185 1186 swe = &ses->swcr_encdec; 1187 txf = crypto_cipher(csp); 1188 if (csp->csp_cipher_key != NULL) { 1189 if (txf->ctxsize != 0) { 1190 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1191 M_NOWAIT); 1192 if (swe->sw_ctx == NULL) 1193 return (ENOMEM); 1194 } 1195 error = txf->setkey(swe->sw_ctx, 1196 csp->csp_cipher_key, csp->csp_cipher_klen); 1197 if (error) 1198 return (error); 1199 } 1200 swe->sw_exf = txf; 1201 return (0); 1202 } 1203 1204 static int 1205 swcr_setup_auth(struct swcr_session *ses, 1206 const struct crypto_session_params *csp) 1207 { 1208 struct swcr_auth *swa; 1209 const struct auth_hash *axf; 1210 1211 swa = &ses->swcr_auth; 1212 1213 axf = crypto_auth_hash(csp); 1214 swa->sw_axf = axf; 1215 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1216 return (EINVAL); 1217 if (csp->csp_auth_mlen == 0) 1218 swa->sw_mlen = axf->hashsize; 1219 else 1220 swa->sw_mlen = csp->csp_auth_mlen; 1221 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1222 if (swa->sw_ictx == NULL) 1223 return (ENOBUFS); 1224 1225 switch (csp->csp_auth_alg) { 1226 case CRYPTO_SHA1_HMAC: 1227 case CRYPTO_SHA2_224_HMAC: 1228 case CRYPTO_SHA2_256_HMAC: 1229 case CRYPTO_SHA2_384_HMAC: 1230 case CRYPTO_SHA2_512_HMAC: 1231 case CRYPTO_NULL_HMAC: 1232 case CRYPTO_RIPEMD160_HMAC: 1233 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1234 M_NOWAIT); 1235 if (swa->sw_octx == NULL) 1236 return (ENOBUFS); 1237 1238 if (csp->csp_auth_key != NULL) { 1239 swcr_authprepare(axf, swa, csp->csp_auth_key, 1240 csp->csp_auth_klen); 1241 } 1242 1243 if (csp->csp_mode == CSP_MODE_DIGEST) 1244 ses->swcr_process = swcr_authcompute; 1245 break; 1246 case CRYPTO_SHA1: 1247 case CRYPTO_SHA2_224: 1248 case CRYPTO_SHA2_256: 1249 case CRYPTO_SHA2_384: 1250 case CRYPTO_SHA2_512: 1251 axf->Init(swa->sw_ictx); 1252 if (csp->csp_mode == CSP_MODE_DIGEST) 1253 ses->swcr_process = swcr_authcompute; 1254 break; 1255 case CRYPTO_AES_NIST_GMAC: 1256 axf->Init(swa->sw_ictx); 1257 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1258 csp->csp_auth_klen); 1259 if (csp->csp_mode == CSP_MODE_DIGEST) 1260 ses->swcr_process = swcr_gmac; 1261 break; 1262 case CRYPTO_POLY1305: 1263 case CRYPTO_BLAKE2B: 1264 case CRYPTO_BLAKE2S: 1265 /* 1266 * Blake2b and Blake2s support an optional key but do 1267 * not require one. 1268 */ 1269 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) 1270 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1271 csp->csp_auth_klen); 1272 axf->Init(swa->sw_ictx); 1273 if (csp->csp_mode == CSP_MODE_DIGEST) 1274 ses->swcr_process = swcr_authcompute; 1275 break; 1276 case CRYPTO_AES_CCM_CBC_MAC: 1277 axf->Init(swa->sw_ictx); 1278 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1279 csp->csp_auth_klen); 1280 if (csp->csp_mode == CSP_MODE_DIGEST) 1281 ses->swcr_process = swcr_ccm_cbc_mac; 1282 break; 1283 } 1284 1285 return (0); 1286 } 1287 1288 static int 1289 swcr_setup_aead(struct swcr_session *ses, 1290 const struct crypto_session_params *csp) 1291 { 1292 struct swcr_auth *swa; 1293 int error; 1294 1295 error = swcr_setup_cipher(ses, csp); 1296 if (error) 1297 return (error); 1298 1299 swa = &ses->swcr_auth; 1300 if (csp->csp_auth_mlen == 0) 1301 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1302 else 1303 swa->sw_mlen = csp->csp_auth_mlen; 1304 return (0); 1305 } 1306 1307 static bool 1308 swcr_auth_supported(const struct crypto_session_params *csp) 1309 { 1310 const struct auth_hash *axf; 1311 1312 axf = crypto_auth_hash(csp); 1313 if (axf == NULL) 1314 return (false); 1315 switch (csp->csp_auth_alg) { 1316 case CRYPTO_SHA1_HMAC: 1317 case CRYPTO_SHA2_224_HMAC: 1318 case CRYPTO_SHA2_256_HMAC: 1319 case CRYPTO_SHA2_384_HMAC: 1320 case CRYPTO_SHA2_512_HMAC: 1321 case CRYPTO_NULL_HMAC: 1322 case CRYPTO_RIPEMD160_HMAC: 1323 break; 1324 case CRYPTO_AES_NIST_GMAC: 1325 switch (csp->csp_auth_klen * 8) { 1326 case 128: 1327 case 192: 1328 case 256: 1329 break; 1330 default: 1331 return (false); 1332 } 1333 if (csp->csp_auth_key == NULL) 1334 return (false); 1335 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1336 return (false); 1337 break; 1338 case CRYPTO_POLY1305: 1339 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1340 return (false); 1341 break; 1342 case CRYPTO_AES_CCM_CBC_MAC: 1343 switch (csp->csp_auth_klen * 8) { 1344 case 128: 1345 case 192: 1346 case 256: 1347 break; 1348 default: 1349 return (false); 1350 } 1351 if (csp->csp_auth_key == NULL) 1352 return (false); 1353 break; 1354 } 1355 return (true); 1356 } 1357 1358 static bool 1359 swcr_cipher_supported(const struct crypto_session_params *csp) 1360 { 1361 const struct enc_xform *txf; 1362 1363 txf = crypto_cipher(csp); 1364 if (txf == NULL) 1365 return (false); 1366 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1367 txf->ivsize != csp->csp_ivlen) 1368 return (false); 1369 return (true); 1370 } 1371 1372 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1373 1374 static int 1375 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1376 { 1377 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1378 return (EINVAL); 1379 switch (csp->csp_mode) { 1380 case CSP_MODE_COMPRESS: 1381 switch (csp->csp_cipher_alg) { 1382 case CRYPTO_DEFLATE_COMP: 1383 break; 1384 default: 1385 return (EINVAL); 1386 } 1387 break; 1388 case CSP_MODE_CIPHER: 1389 switch (csp->csp_cipher_alg) { 1390 case CRYPTO_AES_NIST_GCM_16: 1391 case CRYPTO_AES_CCM_16: 1392 case CRYPTO_CHACHA20_POLY1305: 1393 return (EINVAL); 1394 default: 1395 if (!swcr_cipher_supported(csp)) 1396 return (EINVAL); 1397 break; 1398 } 1399 break; 1400 case CSP_MODE_DIGEST: 1401 if (!swcr_auth_supported(csp)) 1402 return (EINVAL); 1403 break; 1404 case CSP_MODE_AEAD: 1405 switch (csp->csp_cipher_alg) { 1406 case CRYPTO_AES_NIST_GCM_16: 1407 case CRYPTO_AES_CCM_16: 1408 switch (csp->csp_cipher_klen * 8) { 1409 case 128: 1410 case 192: 1411 case 256: 1412 break; 1413 default: 1414 return (EINVAL); 1415 } 1416 break; 1417 case CRYPTO_CHACHA20_POLY1305: 1418 break; 1419 default: 1420 return (EINVAL); 1421 } 1422 break; 1423 case CSP_MODE_ETA: 1424 /* AEAD algorithms cannot be used for EtA. */ 1425 switch (csp->csp_cipher_alg) { 1426 case CRYPTO_AES_NIST_GCM_16: 1427 case CRYPTO_AES_CCM_16: 1428 case CRYPTO_CHACHA20_POLY1305: 1429 return (EINVAL); 1430 } 1431 switch (csp->csp_auth_alg) { 1432 case CRYPTO_AES_NIST_GMAC: 1433 case CRYPTO_AES_CCM_CBC_MAC: 1434 return (EINVAL); 1435 } 1436 1437 if (!swcr_cipher_supported(csp) || 1438 !swcr_auth_supported(csp)) 1439 return (EINVAL); 1440 break; 1441 default: 1442 return (EINVAL); 1443 } 1444 1445 return (CRYPTODEV_PROBE_SOFTWARE); 1446 } 1447 1448 /* 1449 * Generate a new software session. 1450 */ 1451 static int 1452 swcr_newsession(device_t dev, crypto_session_t cses, 1453 const struct crypto_session_params *csp) 1454 { 1455 struct swcr_session *ses; 1456 const struct comp_algo *cxf; 1457 int error; 1458 1459 ses = crypto_get_driver_session(cses); 1460 mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1461 1462 error = 0; 1463 switch (csp->csp_mode) { 1464 case CSP_MODE_COMPRESS: 1465 switch (csp->csp_cipher_alg) { 1466 case CRYPTO_DEFLATE_COMP: 1467 cxf = &comp_algo_deflate; 1468 break; 1469 #ifdef INVARIANTS 1470 default: 1471 panic("bad compression algo"); 1472 #endif 1473 } 1474 ses->swcr_compdec.sw_cxf = cxf; 1475 ses->swcr_process = swcr_compdec; 1476 break; 1477 case CSP_MODE_CIPHER: 1478 switch (csp->csp_cipher_alg) { 1479 case CRYPTO_NULL_CBC: 1480 ses->swcr_process = swcr_null; 1481 break; 1482 #ifdef INVARIANTS 1483 case CRYPTO_AES_NIST_GCM_16: 1484 case CRYPTO_AES_CCM_16: 1485 case CRYPTO_CHACHA20_POLY1305: 1486 panic("bad cipher algo"); 1487 #endif 1488 default: 1489 error = swcr_setup_cipher(ses, csp); 1490 if (error == 0) 1491 ses->swcr_process = swcr_encdec; 1492 } 1493 break; 1494 case CSP_MODE_DIGEST: 1495 error = swcr_setup_auth(ses, csp); 1496 break; 1497 case CSP_MODE_AEAD: 1498 switch (csp->csp_cipher_alg) { 1499 case CRYPTO_AES_NIST_GCM_16: 1500 error = swcr_setup_aead(ses, csp); 1501 if (error == 0) 1502 ses->swcr_process = swcr_gcm; 1503 break; 1504 case CRYPTO_AES_CCM_16: 1505 error = swcr_setup_aead(ses, csp); 1506 if (error == 0) 1507 ses->swcr_process = swcr_ccm; 1508 break; 1509 case CRYPTO_CHACHA20_POLY1305: 1510 error = swcr_setup_aead(ses, csp); 1511 if (error == 0) 1512 ses->swcr_process = swcr_chacha20_poly1305; 1513 break; 1514 #ifdef INVARIANTS 1515 default: 1516 panic("bad aead algo"); 1517 #endif 1518 } 1519 break; 1520 case CSP_MODE_ETA: 1521 #ifdef INVARIANTS 1522 switch (csp->csp_cipher_alg) { 1523 case CRYPTO_AES_NIST_GCM_16: 1524 case CRYPTO_AES_CCM_16: 1525 case CRYPTO_CHACHA20_POLY1305: 1526 panic("bad eta cipher algo"); 1527 } 1528 switch (csp->csp_auth_alg) { 1529 case CRYPTO_AES_NIST_GMAC: 1530 case CRYPTO_AES_CCM_CBC_MAC: 1531 panic("bad eta auth algo"); 1532 } 1533 #endif 1534 1535 error = swcr_setup_auth(ses, csp); 1536 if (error) 1537 break; 1538 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1539 /* Effectively degrade to digest mode. */ 1540 ses->swcr_process = swcr_authcompute; 1541 break; 1542 } 1543 1544 error = swcr_setup_cipher(ses, csp); 1545 if (error == 0) 1546 ses->swcr_process = swcr_eta; 1547 break; 1548 default: 1549 error = EINVAL; 1550 } 1551 1552 if (error) 1553 swcr_freesession(dev, cses); 1554 return (error); 1555 } 1556 1557 static void 1558 swcr_freesession(device_t dev, crypto_session_t cses) 1559 { 1560 struct swcr_session *ses; 1561 1562 ses = crypto_get_driver_session(cses); 1563 1564 mtx_destroy(&ses->swcr_lock); 1565 1566 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1567 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1568 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1569 } 1570 1571 /* 1572 * Process a software request. 1573 */ 1574 static int 1575 swcr_process(device_t dev, struct cryptop *crp, int hint) 1576 { 1577 struct swcr_session *ses; 1578 1579 ses = crypto_get_driver_session(crp->crp_session); 1580 mtx_lock(&ses->swcr_lock); 1581 1582 crp->crp_etype = ses->swcr_process(ses, crp); 1583 1584 mtx_unlock(&ses->swcr_lock); 1585 crypto_done(crp); 1586 return (0); 1587 } 1588 1589 static void 1590 swcr_identify(driver_t *drv, device_t parent) 1591 { 1592 /* NB: order 10 is so we get attached after h/w devices */ 1593 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1594 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1595 panic("cryptosoft: could not attach"); 1596 } 1597 1598 static int 1599 swcr_probe(device_t dev) 1600 { 1601 device_set_desc(dev, "software crypto"); 1602 device_quiet(dev); 1603 return (BUS_PROBE_NOWILDCARD); 1604 } 1605 1606 static int 1607 swcr_attach(device_t dev) 1608 { 1609 1610 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1611 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1612 if (swcr_id < 0) { 1613 device_printf(dev, "cannot initialize!"); 1614 return (ENXIO); 1615 } 1616 1617 return (0); 1618 } 1619 1620 static int 1621 swcr_detach(device_t dev) 1622 { 1623 crypto_unregister_all(swcr_id); 1624 return 0; 1625 } 1626 1627 static device_method_t swcr_methods[] = { 1628 DEVMETHOD(device_identify, swcr_identify), 1629 DEVMETHOD(device_probe, swcr_probe), 1630 DEVMETHOD(device_attach, swcr_attach), 1631 DEVMETHOD(device_detach, swcr_detach), 1632 1633 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1634 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1635 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1636 DEVMETHOD(cryptodev_process, swcr_process), 1637 1638 {0, 0}, 1639 }; 1640 1641 static driver_t swcr_driver = { 1642 "cryptosoft", 1643 swcr_methods, 1644 0, /* NB: no softc */ 1645 }; 1646 static devclass_t swcr_devclass; 1647 1648 /* 1649 * NB: We explicitly reference the crypto module so we 1650 * get the necessary ordering when built as a loadable 1651 * module. This is required because we bundle the crypto 1652 * module code together with the cryptosoft driver (otherwise 1653 * normal module dependencies would handle things). 1654 */ 1655 extern int crypto_modevent(struct module *, int, void *); 1656 /* XXX where to attach */ 1657 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1658 MODULE_VERSION(cryptosoft, 1); 1659 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1660