1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/lock.h> 48 #include <sys/rwlock.h> 49 #include <sys/endian.h> 50 #include <sys/limits.h> 51 #include <sys/mutex.h> 52 53 #include <crypto/sha1.h> 54 #include <opencrypto/rmd160.h> 55 56 #include <opencrypto/cryptodev.h> 57 #include <opencrypto/xform.h> 58 59 #include <sys/kobj.h> 60 #include <sys/bus.h> 61 #include "cryptodev_if.h" 62 63 struct swcr_auth { 64 void *sw_ictx; 65 void *sw_octx; 66 const struct auth_hash *sw_axf; 67 uint16_t sw_mlen; 68 }; 69 70 struct swcr_encdec { 71 void *sw_kschedule; 72 const struct enc_xform *sw_exf; 73 }; 74 75 struct swcr_compdec { 76 const struct comp_algo *sw_cxf; 77 }; 78 79 struct swcr_session { 80 struct mtx swcr_lock; 81 int (*swcr_process)(struct swcr_session *, struct cryptop *); 82 83 struct swcr_auth swcr_auth; 84 struct swcr_encdec swcr_encdec; 85 struct swcr_compdec swcr_compdec; 86 }; 87 88 static int32_t swcr_id; 89 90 static void swcr_freesession(device_t dev, crypto_session_t cses); 91 92 /* Used for CRYPTO_NULL_CBC. */ 93 static int 94 swcr_null(struct swcr_session *ses, struct cryptop *crp) 95 { 96 97 return (0); 98 } 99 100 /* 101 * Apply a symmetric encryption/decryption algorithm. 102 */ 103 static int 104 swcr_encdec(struct swcr_session *ses, struct cryptop *crp) 105 { 106 unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 107 unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 108 const struct crypto_session_params *csp; 109 const struct enc_xform *exf; 110 struct swcr_encdec *sw; 111 size_t inlen, outlen; 112 int i, blks, resid; 113 struct crypto_buffer_cursor cc_in, cc_out; 114 const unsigned char *inblk; 115 unsigned char *outblk; 116 int error; 117 bool encrypting; 118 119 error = 0; 120 121 sw = &ses->swcr_encdec; 122 exf = sw->sw_exf; 123 csp = crypto_get_params(crp->crp_session); 124 125 if (exf->native_blocksize == 0) { 126 /* Check for non-padded data */ 127 if ((crp->crp_payload_length % exf->blocksize) != 0) 128 return (EINVAL); 129 130 blks = exf->blocksize; 131 } else 132 blks = exf->native_blocksize; 133 134 if (exf == &enc_xform_aes_icm && 135 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 136 return (EINVAL); 137 138 if (crp->crp_cipher_key != NULL) { 139 error = exf->setkey(sw->sw_kschedule, 140 crp->crp_cipher_key, csp->csp_cipher_klen); 141 if (error) 142 return (error); 143 } 144 145 crypto_read_iv(crp, iv); 146 147 if (exf->reinit) { 148 /* 149 * xforms that provide a reinit method perform all IV 150 * handling themselves. 151 */ 152 exf->reinit(sw->sw_kschedule, iv, csp->csp_ivlen); 153 } 154 155 ivp = iv; 156 157 crypto_cursor_init(&cc_in, &crp->crp_buf); 158 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 159 inblk = crypto_cursor_segment(&cc_in, &inlen); 160 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 161 crypto_cursor_init(&cc_out, &crp->crp_obuf); 162 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 163 } else 164 cc_out = cc_in; 165 outblk = crypto_cursor_segment(&cc_out, &outlen); 166 167 resid = crp->crp_payload_length; 168 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 169 170 /* 171 * Loop through encrypting blocks. 'inlen' is the remaining 172 * length of the current segment in the input buffer. 173 * 'outlen' is the remaining length of current segment in the 174 * output buffer. 175 */ 176 while (resid >= blks) { 177 /* 178 * If the current block is not contained within the 179 * current input/output segment, use 'blk' as a local 180 * buffer. 181 */ 182 if (inlen < blks) { 183 crypto_cursor_copydata(&cc_in, blks, blk); 184 inblk = blk; 185 } 186 if (outlen < blks) 187 outblk = blk; 188 189 /* 190 * Ciphers without a 'reinit' hook are assumed to be 191 * used in CBC mode where the chaining is done here. 192 */ 193 if (exf->reinit != NULL) { 194 if (encrypting) 195 exf->encrypt(sw->sw_kschedule, inblk, outblk); 196 else 197 exf->decrypt(sw->sw_kschedule, inblk, outblk); 198 } else if (encrypting) { 199 /* XOR with previous block */ 200 for (i = 0; i < blks; i++) 201 outblk[i] = inblk[i] ^ ivp[i]; 202 203 exf->encrypt(sw->sw_kschedule, outblk, outblk); 204 205 /* 206 * Keep encrypted block for XOR'ing 207 * with next block 208 */ 209 memcpy(iv, outblk, blks); 210 ivp = iv; 211 } else { /* decrypt */ 212 /* 213 * Keep encrypted block for XOR'ing 214 * with next block 215 */ 216 nivp = (ivp == iv) ? iv2 : iv; 217 memcpy(nivp, inblk, blks); 218 219 exf->decrypt(sw->sw_kschedule, inblk, outblk); 220 221 /* XOR with previous block */ 222 for (i = 0; i < blks; i++) 223 outblk[i] ^= ivp[i]; 224 225 ivp = nivp; 226 } 227 228 if (inlen < blks) { 229 inblk = crypto_cursor_segment(&cc_in, &inlen); 230 } else { 231 crypto_cursor_advance(&cc_in, blks); 232 inlen -= blks; 233 inblk += blks; 234 } 235 236 if (outlen < blks) { 237 crypto_cursor_copyback(&cc_out, blks, blk); 238 outblk = crypto_cursor_segment(&cc_out, &outlen); 239 } else { 240 crypto_cursor_advance(&cc_out, blks); 241 outlen -= blks; 242 outblk += blks; 243 } 244 245 resid -= blks; 246 } 247 248 /* Handle trailing partial block for stream ciphers. */ 249 if (resid > 0) { 250 KASSERT(exf->native_blocksize != 0, 251 ("%s: partial block of %d bytes for cipher %s", 252 __func__, i, exf->name)); 253 KASSERT(exf->reinit != NULL, 254 ("%s: partial block cipher %s without reinit hook", 255 __func__, exf->name)); 256 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 257 258 inblk = crypto_cursor_segment(&cc_in, &inlen); 259 outblk = crypto_cursor_segment(&cc_out, &outlen); 260 if (inlen < resid) { 261 crypto_cursor_copydata(&cc_in, resid, blk); 262 inblk = blk; 263 } 264 if (outlen < resid) 265 outblk = blk; 266 if (encrypting) 267 exf->encrypt_last(sw->sw_kschedule, inblk, outblk, 268 resid); 269 else 270 exf->decrypt_last(sw->sw_kschedule, inblk, outblk, 271 resid); 272 if (outlen < resid) 273 crypto_cursor_copyback(&cc_out, resid, blk); 274 } 275 276 explicit_bzero(blk, sizeof(blk)); 277 explicit_bzero(iv, sizeof(iv)); 278 explicit_bzero(iv2, sizeof(iv2)); 279 return (0); 280 } 281 282 static void 283 swcr_authprepare(const struct auth_hash *axf, struct swcr_auth *sw, 284 const uint8_t *key, int klen) 285 { 286 287 switch (axf->type) { 288 case CRYPTO_SHA1_HMAC: 289 case CRYPTO_SHA2_224_HMAC: 290 case CRYPTO_SHA2_256_HMAC: 291 case CRYPTO_SHA2_384_HMAC: 292 case CRYPTO_SHA2_512_HMAC: 293 case CRYPTO_NULL_HMAC: 294 case CRYPTO_RIPEMD160_HMAC: 295 hmac_init_ipad(axf, key, klen, sw->sw_ictx); 296 hmac_init_opad(axf, key, klen, sw->sw_octx); 297 break; 298 case CRYPTO_POLY1305: 299 case CRYPTO_BLAKE2B: 300 case CRYPTO_BLAKE2S: 301 axf->Setkey(sw->sw_ictx, key, klen); 302 axf->Init(sw->sw_ictx); 303 break; 304 default: 305 panic("%s: algorithm %d doesn't use keys", __func__, axf->type); 306 } 307 } 308 309 /* 310 * Compute or verify hash. 311 */ 312 static int 313 swcr_authcompute(struct swcr_session *ses, struct cryptop *crp) 314 { 315 u_char aalg[HASH_MAX_LEN]; 316 const struct crypto_session_params *csp; 317 struct swcr_auth *sw; 318 const struct auth_hash *axf; 319 union authctx ctx; 320 int err; 321 322 sw = &ses->swcr_auth; 323 324 axf = sw->sw_axf; 325 326 csp = crypto_get_params(crp->crp_session); 327 if (crp->crp_auth_key != NULL) { 328 swcr_authprepare(axf, sw, crp->crp_auth_key, 329 csp->csp_auth_klen); 330 } 331 332 bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 333 334 if (crp->crp_aad != NULL) 335 err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 336 else 337 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 338 axf->Update, &ctx); 339 if (err) 340 goto out; 341 342 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 343 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 344 err = crypto_apply_buf(&crp->crp_obuf, 345 crp->crp_payload_output_start, crp->crp_payload_length, 346 axf->Update, &ctx); 347 else 348 err = crypto_apply(crp, crp->crp_payload_start, 349 crp->crp_payload_length, axf->Update, &ctx); 350 if (err) 351 goto out; 352 353 if (csp->csp_flags & CSP_F_ESN) 354 axf->Update(&ctx, crp->crp_esn, 4); 355 356 axf->Final(aalg, &ctx); 357 if (sw->sw_octx != NULL) { 358 bcopy(sw->sw_octx, &ctx, axf->ctxsize); 359 axf->Update(&ctx, aalg, axf->hashsize); 360 axf->Final(aalg, &ctx); 361 } 362 363 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 364 u_char uaalg[HASH_MAX_LEN]; 365 366 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 367 if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 368 err = EBADMSG; 369 explicit_bzero(uaalg, sizeof(uaalg)); 370 } else { 371 /* Inject the authentication data */ 372 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 373 } 374 explicit_bzero(aalg, sizeof(aalg)); 375 out: 376 explicit_bzero(&ctx, sizeof(ctx)); 377 return (err); 378 } 379 380 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 381 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 382 383 static int 384 swcr_gmac(struct swcr_session *ses, struct cryptop *crp) 385 { 386 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 387 u_char *blk = (u_char *)blkbuf; 388 u_char tag[GMAC_DIGEST_LEN]; 389 u_char iv[AES_BLOCK_LEN]; 390 struct crypto_buffer_cursor cc; 391 const u_char *inblk; 392 union authctx ctx; 393 struct swcr_auth *swa; 394 const struct auth_hash *axf; 395 uint32_t *blkp; 396 size_t len; 397 int blksz, error, ivlen, resid; 398 399 swa = &ses->swcr_auth; 400 axf = swa->sw_axf; 401 402 bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 403 blksz = GMAC_BLOCK_LEN; 404 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 405 __func__)); 406 407 /* Initialize the IV */ 408 ivlen = AES_GCM_IV_LEN; 409 crypto_read_iv(crp, iv); 410 411 axf->Reinit(&ctx, iv, ivlen); 412 crypto_cursor_init(&cc, &crp->crp_buf); 413 crypto_cursor_advance(&cc, crp->crp_payload_start); 414 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 415 inblk = crypto_cursor_segment(&cc, &len); 416 if (len >= blksz) { 417 len = rounddown(MIN(len, resid), blksz); 418 crypto_cursor_advance(&cc, len); 419 } else { 420 len = blksz; 421 crypto_cursor_copydata(&cc, len, blk); 422 inblk = blk; 423 } 424 axf->Update(&ctx, inblk, len); 425 } 426 if (resid > 0) { 427 memset(blk, 0, blksz); 428 crypto_cursor_copydata(&cc, resid, blk); 429 axf->Update(&ctx, blk, blksz); 430 } 431 432 /* length block */ 433 memset(blk, 0, blksz); 434 blkp = (uint32_t *)blk + 1; 435 *blkp = htobe32(crp->crp_payload_length * 8); 436 axf->Update(&ctx, blk, blksz); 437 438 /* Finalize MAC */ 439 axf->Final(tag, &ctx); 440 441 error = 0; 442 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 443 u_char tag2[GMAC_DIGEST_LEN]; 444 445 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 446 tag2); 447 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 448 error = EBADMSG; 449 explicit_bzero(tag2, sizeof(tag2)); 450 } else { 451 /* Inject the authentication data */ 452 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 453 } 454 explicit_bzero(blkbuf, sizeof(blkbuf)); 455 explicit_bzero(tag, sizeof(tag)); 456 explicit_bzero(iv, sizeof(iv)); 457 return (error); 458 } 459 460 static int 461 swcr_gcm(struct swcr_session *ses, struct cryptop *crp) 462 { 463 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 464 u_char *blk = (u_char *)blkbuf; 465 u_char tag[GMAC_DIGEST_LEN]; 466 struct crypto_buffer_cursor cc_in, cc_out; 467 const u_char *inblk; 468 u_char *outblk; 469 struct swcr_auth *swa; 470 struct swcr_encdec *swe; 471 const struct enc_xform *exf; 472 uint32_t *blkp; 473 size_t len; 474 int blksz, error, ivlen, r, resid; 475 476 swa = &ses->swcr_auth; 477 swe = &ses->swcr_encdec; 478 exf = swe->sw_exf; 479 blksz = GMAC_BLOCK_LEN; 480 KASSERT(blksz == exf->native_blocksize, 481 ("%s: blocksize mismatch", __func__)); 482 483 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 484 return (EINVAL); 485 486 ivlen = AES_GCM_IV_LEN; 487 488 /* Supply cipher with nonce. */ 489 if (crp->crp_cipher_key != NULL) 490 exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 491 crypto_get_params(crp->crp_session)->csp_cipher_klen); 492 exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 493 494 /* Supply MAC with AAD */ 495 if (crp->crp_aad != NULL) { 496 len = rounddown(crp->crp_aad_length, blksz); 497 if (len != 0) 498 exf->update(swe->sw_kschedule, crp->crp_aad, len); 499 if (crp->crp_aad_length != len) { 500 memset(blk, 0, blksz); 501 memcpy(blk, (char *)crp->crp_aad + len, 502 crp->crp_aad_length - len); 503 exf->update(swe->sw_kschedule, blk, blksz); 504 } 505 } else { 506 crypto_cursor_init(&cc_in, &crp->crp_buf); 507 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 508 for (resid = crp->crp_aad_length; resid >= blksz; 509 resid -= len) { 510 inblk = crypto_cursor_segment(&cc_in, &len); 511 if (len >= blksz) { 512 len = rounddown(MIN(len, resid), blksz); 513 crypto_cursor_advance(&cc_in, len); 514 } else { 515 len = blksz; 516 crypto_cursor_copydata(&cc_in, len, blk); 517 inblk = blk; 518 } 519 exf->update(swe->sw_kschedule, inblk, len); 520 } 521 if (resid > 0) { 522 memset(blk, 0, blksz); 523 crypto_cursor_copydata(&cc_in, resid, blk); 524 exf->update(swe->sw_kschedule, blk, blksz); 525 } 526 } 527 528 /* Do encryption with MAC */ 529 crypto_cursor_init(&cc_in, &crp->crp_buf); 530 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 531 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 532 crypto_cursor_init(&cc_out, &crp->crp_obuf); 533 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 534 } else 535 cc_out = cc_in; 536 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 537 inblk = crypto_cursor_segment(&cc_in, &len); 538 if (len < blksz) { 539 crypto_cursor_copydata(&cc_in, blksz, blk); 540 inblk = blk; 541 } else { 542 crypto_cursor_advance(&cc_in, blksz); 543 } 544 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 545 outblk = crypto_cursor_segment(&cc_out, &len); 546 if (len < blksz) 547 outblk = blk; 548 exf->encrypt(swe->sw_kschedule, inblk, outblk); 549 exf->update(swe->sw_kschedule, outblk, blksz); 550 if (outblk == blk) 551 crypto_cursor_copyback(&cc_out, blksz, blk); 552 else 553 crypto_cursor_advance(&cc_out, blksz); 554 } else { 555 exf->update(swe->sw_kschedule, inblk, blksz); 556 } 557 } 558 if (resid > 0) { 559 crypto_cursor_copydata(&cc_in, resid, blk); 560 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 561 exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 562 crypto_cursor_copyback(&cc_out, resid, blk); 563 } 564 exf->update(swe->sw_kschedule, blk, resid); 565 } 566 567 /* length block */ 568 memset(blk, 0, blksz); 569 blkp = (uint32_t *)blk + 1; 570 *blkp = htobe32(crp->crp_aad_length * 8); 571 blkp = (uint32_t *)blk + 3; 572 *blkp = htobe32(crp->crp_payload_length * 8); 573 exf->update(swe->sw_kschedule, blk, blksz); 574 575 /* Finalize MAC */ 576 exf->final(tag, swe->sw_kschedule); 577 578 /* Validate tag */ 579 error = 0; 580 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 581 u_char tag2[GMAC_DIGEST_LEN]; 582 583 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 584 585 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 586 explicit_bzero(tag2, sizeof(tag2)); 587 if (r != 0) { 588 error = EBADMSG; 589 goto out; 590 } 591 592 /* tag matches, decrypt data */ 593 crypto_cursor_init(&cc_in, &crp->crp_buf); 594 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 595 for (resid = crp->crp_payload_length; resid > blksz; 596 resid -= blksz) { 597 inblk = crypto_cursor_segment(&cc_in, &len); 598 if (len < blksz) { 599 crypto_cursor_copydata(&cc_in, blksz, blk); 600 inblk = blk; 601 } else 602 crypto_cursor_advance(&cc_in, blksz); 603 outblk = crypto_cursor_segment(&cc_out, &len); 604 if (len < blksz) 605 outblk = blk; 606 exf->decrypt(swe->sw_kschedule, inblk, outblk); 607 if (outblk == blk) 608 crypto_cursor_copyback(&cc_out, blksz, blk); 609 else 610 crypto_cursor_advance(&cc_out, blksz); 611 } 612 if (resid > 0) { 613 crypto_cursor_copydata(&cc_in, resid, blk); 614 exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 615 crypto_cursor_copyback(&cc_out, resid, blk); 616 } 617 } else { 618 /* Inject the authentication data */ 619 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 620 } 621 622 out: 623 explicit_bzero(blkbuf, sizeof(blkbuf)); 624 explicit_bzero(tag, sizeof(tag)); 625 626 return (error); 627 } 628 629 static void 630 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 631 u_int data_length, u_int tag_length, uint8_t *b0) 632 { 633 uint8_t *bp; 634 uint8_t flags, L; 635 636 KASSERT(nonce_length >= 7 && nonce_length <= 13, 637 ("nonce_length must be between 7 and 13 bytes")); 638 639 /* 640 * Need to determine the L field value. This is the number of 641 * bytes needed to specify the length of the message; the length 642 * is whatever is left in the 16 bytes after specifying flags and 643 * the nonce. 644 */ 645 L = 15 - nonce_length; 646 647 flags = ((aad_length > 0) << 6) + 648 (((tag_length - 2) / 2) << 3) + 649 L - 1; 650 651 /* 652 * Now we need to set up the first block, which has flags, nonce, 653 * and the message length. 654 */ 655 b0[0] = flags; 656 memcpy(b0 + 1, nonce, nonce_length); 657 bp = b0 + 1 + nonce_length; 658 659 /* Need to copy L' [aka L-1] bytes of data_length */ 660 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 661 *dst = data_length; 662 data_length >>= 8; 663 } 664 } 665 666 /* NB: OCF only supports AAD lengths < 2^32. */ 667 static int 668 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 669 { 670 if (aad_length < ((1 << 16) - (1 << 8))) { 671 be16enc(blk, aad_length); 672 return (sizeof(uint16_t)); 673 } else { 674 blk[0] = 0xff; 675 blk[1] = 0xfe; 676 be32enc(blk + 2, aad_length); 677 return (2 + sizeof(uint32_t)); 678 } 679 } 680 681 static int 682 swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp) 683 { 684 u_char iv[AES_BLOCK_LEN]; 685 u_char blk[CCM_CBC_BLOCK_LEN]; 686 u_char tag[AES_CBC_MAC_HASH_LEN]; 687 union authctx ctx; 688 const struct crypto_session_params *csp; 689 struct swcr_auth *swa; 690 const struct auth_hash *axf; 691 int error, ivlen, len; 692 693 csp = crypto_get_params(crp->crp_session); 694 swa = &ses->swcr_auth; 695 axf = swa->sw_axf; 696 697 bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 698 699 /* Initialize the IV */ 700 ivlen = csp->csp_ivlen; 701 crypto_read_iv(crp, iv); 702 703 /* Supply MAC with IV */ 704 axf->Reinit(&ctx, crp->crp_iv, ivlen); 705 706 /* Supply MAC with b0. */ 707 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 708 swa->sw_mlen, blk); 709 axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 710 711 len = build_ccm_aad_length(crp->crp_payload_length, blk); 712 axf->Update(&ctx, blk, len); 713 714 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 715 axf->Update, &ctx); 716 717 /* Finalize MAC */ 718 axf->Final(tag, &ctx); 719 720 error = 0; 721 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 722 u_char tag2[AES_CBC_MAC_HASH_LEN]; 723 724 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 725 tag2); 726 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 727 error = EBADMSG; 728 explicit_bzero(tag2, sizeof(tag)); 729 } else { 730 /* Inject the authentication data */ 731 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 732 } 733 explicit_bzero(tag, sizeof(tag)); 734 explicit_bzero(blk, sizeof(blk)); 735 explicit_bzero(iv, sizeof(iv)); 736 return (error); 737 } 738 739 static int 740 swcr_ccm(struct swcr_session *ses, struct cryptop *crp) 741 { 742 const struct crypto_session_params *csp; 743 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 744 u_char *blk = (u_char *)blkbuf; 745 u_char tag[AES_CBC_MAC_HASH_LEN]; 746 struct crypto_buffer_cursor cc_in, cc_out; 747 const u_char *inblk; 748 u_char *outblk; 749 struct swcr_auth *swa; 750 struct swcr_encdec *swe; 751 const struct enc_xform *exf; 752 size_t len; 753 int blksz, error, ivlen, r, resid; 754 755 csp = crypto_get_params(crp->crp_session); 756 swa = &ses->swcr_auth; 757 swe = &ses->swcr_encdec; 758 exf = swe->sw_exf; 759 blksz = AES_BLOCK_LEN; 760 KASSERT(blksz == exf->native_blocksize, 761 ("%s: blocksize mismatch", __func__)); 762 763 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 764 return (EMSGSIZE); 765 766 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 767 return (EINVAL); 768 769 ivlen = csp->csp_ivlen; 770 771 if (crp->crp_cipher_key != NULL) 772 exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 773 crypto_get_params(crp->crp_session)->csp_cipher_klen); 774 exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 775 776 /* Supply MAC with b0. */ 777 _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 778 "blkbuf too small for b0"); 779 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 780 crp->crp_payload_length, swa->sw_mlen, blk); 781 exf->update(swe->sw_kschedule, blk, CCM_CBC_BLOCK_LEN); 782 783 /* Supply MAC with AAD */ 784 if (crp->crp_aad_length != 0) { 785 len = build_ccm_aad_length(crp->crp_aad_length, blk); 786 exf->update(swe->sw_kschedule, blk, len); 787 if (crp->crp_aad != NULL) 788 exf->update(swe->sw_kschedule, crp->crp_aad, 789 crp->crp_aad_length); 790 else 791 crypto_apply(crp, crp->crp_aad_start, 792 crp->crp_aad_length, exf->update, 793 swe->sw_kschedule); 794 795 /* Pad the AAD (including length field) to a full block. */ 796 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 797 if (len != 0) { 798 len = CCM_CBC_BLOCK_LEN - len; 799 memset(blk, 0, CCM_CBC_BLOCK_LEN); 800 exf->update(swe->sw_kschedule, blk, len); 801 } 802 } 803 804 /* Do encryption/decryption with MAC */ 805 crypto_cursor_init(&cc_in, &crp->crp_buf); 806 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 807 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 808 crypto_cursor_init(&cc_out, &crp->crp_obuf); 809 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 810 } else 811 cc_out = cc_in; 812 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 813 inblk = crypto_cursor_segment(&cc_in, &len); 814 if (len < blksz) { 815 crypto_cursor_copydata(&cc_in, blksz, blk); 816 inblk = blk; 817 } else 818 crypto_cursor_advance(&cc_in, blksz); 819 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 820 outblk = crypto_cursor_segment(&cc_out, &len); 821 if (len < blksz) 822 outblk = blk; 823 exf->update(swe->sw_kschedule, inblk, blksz); 824 exf->encrypt(swe->sw_kschedule, inblk, outblk); 825 if (outblk == blk) 826 crypto_cursor_copyback(&cc_out, blksz, blk); 827 else 828 crypto_cursor_advance(&cc_out, blksz); 829 } else { 830 /* 831 * One of the problems with CCM+CBC is that 832 * the authentication is done on the 833 * unencrypted data. As a result, we have to 834 * decrypt the data twice: once to generate 835 * the tag and a second time after the tag is 836 * verified. 837 */ 838 exf->decrypt(swe->sw_kschedule, inblk, blk); 839 exf->update(swe->sw_kschedule, blk, blksz); 840 } 841 } 842 if (resid > 0) { 843 crypto_cursor_copydata(&cc_in, resid, blk); 844 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 845 exf->update(swe->sw_kschedule, blk, resid); 846 exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 847 crypto_cursor_copyback(&cc_out, resid, blk); 848 } else { 849 exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 850 exf->update(swe->sw_kschedule, blk, resid); 851 } 852 } 853 854 /* Finalize MAC */ 855 exf->final(tag, swe->sw_kschedule); 856 857 /* Validate tag */ 858 error = 0; 859 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 860 u_char tag2[AES_CBC_MAC_HASH_LEN]; 861 862 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 863 tag2); 864 865 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 866 explicit_bzero(tag2, sizeof(tag2)); 867 if (r != 0) { 868 error = EBADMSG; 869 goto out; 870 } 871 872 /* tag matches, decrypt data */ 873 exf->reinit(swe->sw_kschedule, crp->crp_iv, ivlen); 874 crypto_cursor_init(&cc_in, &crp->crp_buf); 875 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 876 for (resid = crp->crp_payload_length; resid > blksz; 877 resid -= blksz) { 878 inblk = crypto_cursor_segment(&cc_in, &len); 879 if (len < blksz) { 880 crypto_cursor_copydata(&cc_in, blksz, blk); 881 inblk = blk; 882 } else 883 crypto_cursor_advance(&cc_in, blksz); 884 outblk = crypto_cursor_segment(&cc_out, &len); 885 if (len < blksz) 886 outblk = blk; 887 exf->decrypt(swe->sw_kschedule, inblk, outblk); 888 if (outblk == blk) 889 crypto_cursor_copyback(&cc_out, blksz, blk); 890 else 891 crypto_cursor_advance(&cc_out, blksz); 892 } 893 if (resid > 0) { 894 crypto_cursor_copydata(&cc_in, resid, blk); 895 exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 896 crypto_cursor_copyback(&cc_out, resid, blk); 897 } 898 } else { 899 /* Inject the authentication data */ 900 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 901 } 902 903 out: 904 explicit_bzero(blkbuf, sizeof(blkbuf)); 905 explicit_bzero(tag, sizeof(tag)); 906 return (error); 907 } 908 909 static int 910 swcr_chacha20_poly1305(struct swcr_session *ses, struct cryptop *crp) 911 { 912 const struct crypto_session_params *csp; 913 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 914 u_char *blk = (u_char *)blkbuf; 915 u_char tag[POLY1305_HASH_LEN]; 916 struct crypto_buffer_cursor cc_in, cc_out; 917 const u_char *inblk; 918 u_char *outblk; 919 uint64_t *blkp; 920 struct swcr_auth *swa; 921 struct swcr_encdec *swe; 922 const struct enc_xform *exf; 923 size_t len; 924 int blksz, error, r, resid; 925 926 swa = &ses->swcr_auth; 927 swe = &ses->swcr_encdec; 928 exf = swe->sw_exf; 929 blksz = exf->native_blocksize; 930 KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 931 932 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 933 return (EINVAL); 934 935 csp = crypto_get_params(crp->crp_session); 936 937 if (crp->crp_cipher_key != NULL) 938 exf->setkey(swe->sw_kschedule, crp->crp_cipher_key, 939 csp->csp_cipher_klen); 940 exf->reinit(swe->sw_kschedule, crp->crp_iv, csp->csp_ivlen); 941 942 /* Supply MAC with AAD */ 943 if (crp->crp_aad != NULL) 944 exf->update(swe->sw_kschedule, crp->crp_aad, 945 crp->crp_aad_length); 946 else 947 crypto_apply(crp, crp->crp_aad_start, 948 crp->crp_aad_length, exf->update, swe->sw_kschedule); 949 if (crp->crp_aad_length % 16 != 0) { 950 /* padding1 */ 951 memset(blk, 0, 16); 952 exf->update(swe->sw_kschedule, blk, 953 16 - crp->crp_aad_length % 16); 954 } 955 956 /* Do encryption with MAC */ 957 crypto_cursor_init(&cc_in, &crp->crp_buf); 958 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 959 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 960 crypto_cursor_init(&cc_out, &crp->crp_obuf); 961 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 962 } else 963 cc_out = cc_in; 964 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 965 inblk = crypto_cursor_segment(&cc_in, &len); 966 if (len < blksz) { 967 crypto_cursor_copydata(&cc_in, blksz, blk); 968 inblk = blk; 969 } else 970 crypto_cursor_advance(&cc_in, blksz); 971 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 972 outblk = crypto_cursor_segment(&cc_out, &len); 973 if (len < blksz) 974 outblk = blk; 975 exf->encrypt(swe->sw_kschedule, inblk, outblk); 976 exf->update(swe->sw_kschedule, outblk, blksz); 977 if (outblk == blk) 978 crypto_cursor_copyback(&cc_out, blksz, blk); 979 else 980 crypto_cursor_advance(&cc_out, blksz); 981 } else { 982 exf->update(swe->sw_kschedule, inblk, blksz); 983 } 984 } 985 if (resid > 0) { 986 crypto_cursor_copydata(&cc_in, resid, blk); 987 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 988 exf->encrypt_last(swe->sw_kschedule, blk, blk, resid); 989 crypto_cursor_copyback(&cc_out, resid, blk); 990 } 991 exf->update(swe->sw_kschedule, blk, resid); 992 if (resid % 16 != 0) { 993 /* padding2 */ 994 memset(blk, 0, 16); 995 exf->update(swe->sw_kschedule, blk, 16 - resid % 16); 996 } 997 } 998 999 /* lengths */ 1000 blkp = (uint64_t *)blk; 1001 blkp[0] = htole64(crp->crp_aad_length); 1002 blkp[1] = htole64(crp->crp_payload_length); 1003 exf->update(swe->sw_kschedule, blk, sizeof(uint64_t) * 2); 1004 1005 /* Finalize MAC */ 1006 exf->final(tag, swe->sw_kschedule); 1007 1008 /* Validate tag */ 1009 error = 0; 1010 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1011 u_char tag2[POLY1305_HASH_LEN]; 1012 1013 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 1014 1015 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 1016 explicit_bzero(tag2, sizeof(tag2)); 1017 if (r != 0) { 1018 error = EBADMSG; 1019 goto out; 1020 } 1021 1022 /* tag matches, decrypt data */ 1023 crypto_cursor_init(&cc_in, &crp->crp_buf); 1024 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 1025 for (resid = crp->crp_payload_length; resid > blksz; 1026 resid -= blksz) { 1027 inblk = crypto_cursor_segment(&cc_in, &len); 1028 if (len < blksz) { 1029 crypto_cursor_copydata(&cc_in, blksz, blk); 1030 inblk = blk; 1031 } else 1032 crypto_cursor_advance(&cc_in, blksz); 1033 outblk = crypto_cursor_segment(&cc_out, &len); 1034 if (len < blksz) 1035 outblk = blk; 1036 exf->decrypt(swe->sw_kschedule, inblk, outblk); 1037 if (outblk == blk) 1038 crypto_cursor_copyback(&cc_out, blksz, blk); 1039 else 1040 crypto_cursor_advance(&cc_out, blksz); 1041 } 1042 if (resid > 0) { 1043 crypto_cursor_copydata(&cc_in, resid, blk); 1044 exf->decrypt_last(swe->sw_kschedule, blk, blk, resid); 1045 crypto_cursor_copyback(&cc_out, resid, blk); 1046 } 1047 } else { 1048 /* Inject the authentication data */ 1049 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 1050 } 1051 1052 out: 1053 explicit_bzero(blkbuf, sizeof(blkbuf)); 1054 explicit_bzero(tag, sizeof(tag)); 1055 return (error); 1056 } 1057 1058 /* 1059 * Apply a cipher and a digest to perform EtA. 1060 */ 1061 static int 1062 swcr_eta(struct swcr_session *ses, struct cryptop *crp) 1063 { 1064 int error; 1065 1066 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1067 error = swcr_encdec(ses, crp); 1068 if (error == 0) 1069 error = swcr_authcompute(ses, crp); 1070 } else { 1071 error = swcr_authcompute(ses, crp); 1072 if (error == 0) 1073 error = swcr_encdec(ses, crp); 1074 } 1075 return (error); 1076 } 1077 1078 /* 1079 * Apply a compression/decompression algorithm 1080 */ 1081 static int 1082 swcr_compdec(struct swcr_session *ses, struct cryptop *crp) 1083 { 1084 const struct comp_algo *cxf; 1085 uint8_t *data, *out; 1086 int adj; 1087 uint32_t result; 1088 1089 cxf = ses->swcr_compdec.sw_cxf; 1090 1091 /* We must handle the whole buffer of data in one time 1092 * then if there is not all the data in the mbuf, we must 1093 * copy in a buffer. 1094 */ 1095 1096 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1097 if (data == NULL) 1098 return (EINVAL); 1099 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1100 data); 1101 1102 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1103 result = cxf->compress(data, crp->crp_payload_length, &out); 1104 else 1105 result = cxf->decompress(data, crp->crp_payload_length, &out); 1106 1107 free(data, M_CRYPTO_DATA); 1108 if (result == 0) 1109 return (EINVAL); 1110 crp->crp_olen = result; 1111 1112 /* Check the compressed size when doing compression */ 1113 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1114 if (result >= crp->crp_payload_length) { 1115 /* Compression was useless, we lost time */ 1116 free(out, M_CRYPTO_DATA); 1117 return (0); 1118 } 1119 } 1120 1121 /* Copy back the (de)compressed data. m_copyback is 1122 * extending the mbuf as necessary. 1123 */ 1124 crypto_copyback(crp, crp->crp_payload_start, result, out); 1125 if (result < crp->crp_payload_length) { 1126 switch (crp->crp_buf.cb_type) { 1127 case CRYPTO_BUF_MBUF: 1128 case CRYPTO_BUF_SINGLE_MBUF: 1129 adj = result - crp->crp_payload_length; 1130 m_adj(crp->crp_buf.cb_mbuf, adj); 1131 break; 1132 case CRYPTO_BUF_UIO: { 1133 struct uio *uio = crp->crp_buf.cb_uio; 1134 int ind; 1135 1136 adj = crp->crp_payload_length - result; 1137 ind = uio->uio_iovcnt - 1; 1138 1139 while (adj > 0 && ind >= 0) { 1140 if (adj < uio->uio_iov[ind].iov_len) { 1141 uio->uio_iov[ind].iov_len -= adj; 1142 break; 1143 } 1144 1145 adj -= uio->uio_iov[ind].iov_len; 1146 uio->uio_iov[ind].iov_len = 0; 1147 ind--; 1148 uio->uio_iovcnt--; 1149 } 1150 } 1151 break; 1152 case CRYPTO_BUF_VMPAGE: 1153 adj = crp->crp_payload_length - result; 1154 crp->crp_buf.cb_vm_page_len -= adj; 1155 break; 1156 default: 1157 break; 1158 } 1159 } 1160 free(out, M_CRYPTO_DATA); 1161 return 0; 1162 } 1163 1164 static int 1165 swcr_setup_cipher(struct swcr_session *ses, 1166 const struct crypto_session_params *csp) 1167 { 1168 struct swcr_encdec *swe; 1169 const struct enc_xform *txf; 1170 int error; 1171 1172 swe = &ses->swcr_encdec; 1173 txf = crypto_cipher(csp); 1174 if (txf->ctxsize != 0) { 1175 swe->sw_kschedule = malloc(txf->ctxsize, M_CRYPTO_DATA, 1176 M_NOWAIT); 1177 if (swe->sw_kschedule == NULL) 1178 return (ENOMEM); 1179 } 1180 if (csp->csp_cipher_key != NULL) { 1181 error = txf->setkey(swe->sw_kschedule, 1182 csp->csp_cipher_key, csp->csp_cipher_klen); 1183 if (error) 1184 return (error); 1185 } 1186 swe->sw_exf = txf; 1187 return (0); 1188 } 1189 1190 static int 1191 swcr_setup_auth(struct swcr_session *ses, 1192 const struct crypto_session_params *csp) 1193 { 1194 struct swcr_auth *swa; 1195 const struct auth_hash *axf; 1196 1197 swa = &ses->swcr_auth; 1198 1199 axf = crypto_auth_hash(csp); 1200 swa->sw_axf = axf; 1201 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1202 return (EINVAL); 1203 if (csp->csp_auth_mlen == 0) 1204 swa->sw_mlen = axf->hashsize; 1205 else 1206 swa->sw_mlen = csp->csp_auth_mlen; 1207 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT); 1208 if (swa->sw_ictx == NULL) 1209 return (ENOBUFS); 1210 1211 switch (csp->csp_auth_alg) { 1212 case CRYPTO_SHA1_HMAC: 1213 case CRYPTO_SHA2_224_HMAC: 1214 case CRYPTO_SHA2_256_HMAC: 1215 case CRYPTO_SHA2_384_HMAC: 1216 case CRYPTO_SHA2_512_HMAC: 1217 case CRYPTO_NULL_HMAC: 1218 case CRYPTO_RIPEMD160_HMAC: 1219 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1220 M_NOWAIT); 1221 if (swa->sw_octx == NULL) 1222 return (ENOBUFS); 1223 1224 if (csp->csp_auth_key != NULL) { 1225 swcr_authprepare(axf, swa, csp->csp_auth_key, 1226 csp->csp_auth_klen); 1227 } 1228 1229 if (csp->csp_mode == CSP_MODE_DIGEST) 1230 ses->swcr_process = swcr_authcompute; 1231 break; 1232 case CRYPTO_SHA1: 1233 case CRYPTO_SHA2_224: 1234 case CRYPTO_SHA2_256: 1235 case CRYPTO_SHA2_384: 1236 case CRYPTO_SHA2_512: 1237 axf->Init(swa->sw_ictx); 1238 if (csp->csp_mode == CSP_MODE_DIGEST) 1239 ses->swcr_process = swcr_authcompute; 1240 break; 1241 case CRYPTO_AES_NIST_GMAC: 1242 axf->Init(swa->sw_ictx); 1243 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1244 csp->csp_auth_klen); 1245 if (csp->csp_mode == CSP_MODE_DIGEST) 1246 ses->swcr_process = swcr_gmac; 1247 break; 1248 case CRYPTO_POLY1305: 1249 case CRYPTO_BLAKE2B: 1250 case CRYPTO_BLAKE2S: 1251 /* 1252 * Blake2b and Blake2s support an optional key but do 1253 * not require one. 1254 */ 1255 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) 1256 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1257 csp->csp_auth_klen); 1258 axf->Init(swa->sw_ictx); 1259 if (csp->csp_mode == CSP_MODE_DIGEST) 1260 ses->swcr_process = swcr_authcompute; 1261 break; 1262 case CRYPTO_AES_CCM_CBC_MAC: 1263 axf->Init(swa->sw_ictx); 1264 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1265 csp->csp_auth_klen); 1266 if (csp->csp_mode == CSP_MODE_DIGEST) 1267 ses->swcr_process = swcr_ccm_cbc_mac; 1268 break; 1269 } 1270 1271 return (0); 1272 } 1273 1274 static int 1275 swcr_setup_aead(struct swcr_session *ses, 1276 const struct crypto_session_params *csp) 1277 { 1278 struct swcr_auth *swa; 1279 int error; 1280 1281 error = swcr_setup_cipher(ses, csp); 1282 if (error) 1283 return (error); 1284 1285 swa = &ses->swcr_auth; 1286 if (csp->csp_auth_mlen == 0) 1287 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1288 else 1289 swa->sw_mlen = csp->csp_auth_mlen; 1290 return (0); 1291 } 1292 1293 static bool 1294 swcr_auth_supported(const struct crypto_session_params *csp) 1295 { 1296 const struct auth_hash *axf; 1297 1298 axf = crypto_auth_hash(csp); 1299 if (axf == NULL) 1300 return (false); 1301 switch (csp->csp_auth_alg) { 1302 case CRYPTO_SHA1_HMAC: 1303 case CRYPTO_SHA2_224_HMAC: 1304 case CRYPTO_SHA2_256_HMAC: 1305 case CRYPTO_SHA2_384_HMAC: 1306 case CRYPTO_SHA2_512_HMAC: 1307 case CRYPTO_NULL_HMAC: 1308 case CRYPTO_RIPEMD160_HMAC: 1309 break; 1310 case CRYPTO_AES_NIST_GMAC: 1311 switch (csp->csp_auth_klen * 8) { 1312 case 128: 1313 case 192: 1314 case 256: 1315 break; 1316 default: 1317 return (false); 1318 } 1319 if (csp->csp_auth_key == NULL) 1320 return (false); 1321 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1322 return (false); 1323 break; 1324 case CRYPTO_POLY1305: 1325 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1326 return (false); 1327 break; 1328 case CRYPTO_AES_CCM_CBC_MAC: 1329 switch (csp->csp_auth_klen * 8) { 1330 case 128: 1331 case 192: 1332 case 256: 1333 break; 1334 default: 1335 return (false); 1336 } 1337 if (csp->csp_auth_key == NULL) 1338 return (false); 1339 break; 1340 } 1341 return (true); 1342 } 1343 1344 static bool 1345 swcr_cipher_supported(const struct crypto_session_params *csp) 1346 { 1347 const struct enc_xform *txf; 1348 1349 txf = crypto_cipher(csp); 1350 if (txf == NULL) 1351 return (false); 1352 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1353 txf->ivsize != csp->csp_ivlen) 1354 return (false); 1355 return (true); 1356 } 1357 1358 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1359 1360 static int 1361 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1362 { 1363 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1364 return (EINVAL); 1365 switch (csp->csp_mode) { 1366 case CSP_MODE_COMPRESS: 1367 switch (csp->csp_cipher_alg) { 1368 case CRYPTO_DEFLATE_COMP: 1369 break; 1370 default: 1371 return (EINVAL); 1372 } 1373 break; 1374 case CSP_MODE_CIPHER: 1375 switch (csp->csp_cipher_alg) { 1376 case CRYPTO_AES_NIST_GCM_16: 1377 case CRYPTO_AES_CCM_16: 1378 case CRYPTO_CHACHA20_POLY1305: 1379 return (EINVAL); 1380 default: 1381 if (!swcr_cipher_supported(csp)) 1382 return (EINVAL); 1383 break; 1384 } 1385 break; 1386 case CSP_MODE_DIGEST: 1387 if (!swcr_auth_supported(csp)) 1388 return (EINVAL); 1389 break; 1390 case CSP_MODE_AEAD: 1391 switch (csp->csp_cipher_alg) { 1392 case CRYPTO_AES_NIST_GCM_16: 1393 case CRYPTO_AES_CCM_16: 1394 switch (csp->csp_cipher_klen * 8) { 1395 case 128: 1396 case 192: 1397 case 256: 1398 break; 1399 default: 1400 return (EINVAL); 1401 } 1402 break; 1403 case CRYPTO_CHACHA20_POLY1305: 1404 break; 1405 default: 1406 return (EINVAL); 1407 } 1408 break; 1409 case CSP_MODE_ETA: 1410 /* AEAD algorithms cannot be used for EtA. */ 1411 switch (csp->csp_cipher_alg) { 1412 case CRYPTO_AES_NIST_GCM_16: 1413 case CRYPTO_AES_CCM_16: 1414 case CRYPTO_CHACHA20_POLY1305: 1415 return (EINVAL); 1416 } 1417 switch (csp->csp_auth_alg) { 1418 case CRYPTO_AES_NIST_GMAC: 1419 case CRYPTO_AES_CCM_CBC_MAC: 1420 return (EINVAL); 1421 } 1422 1423 if (!swcr_cipher_supported(csp) || 1424 !swcr_auth_supported(csp)) 1425 return (EINVAL); 1426 break; 1427 default: 1428 return (EINVAL); 1429 } 1430 1431 return (CRYPTODEV_PROBE_SOFTWARE); 1432 } 1433 1434 /* 1435 * Generate a new software session. 1436 */ 1437 static int 1438 swcr_newsession(device_t dev, crypto_session_t cses, 1439 const struct crypto_session_params *csp) 1440 { 1441 struct swcr_session *ses; 1442 const struct comp_algo *cxf; 1443 int error; 1444 1445 ses = crypto_get_driver_session(cses); 1446 mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF); 1447 1448 error = 0; 1449 switch (csp->csp_mode) { 1450 case CSP_MODE_COMPRESS: 1451 switch (csp->csp_cipher_alg) { 1452 case CRYPTO_DEFLATE_COMP: 1453 cxf = &comp_algo_deflate; 1454 break; 1455 #ifdef INVARIANTS 1456 default: 1457 panic("bad compression algo"); 1458 #endif 1459 } 1460 ses->swcr_compdec.sw_cxf = cxf; 1461 ses->swcr_process = swcr_compdec; 1462 break; 1463 case CSP_MODE_CIPHER: 1464 switch (csp->csp_cipher_alg) { 1465 case CRYPTO_NULL_CBC: 1466 ses->swcr_process = swcr_null; 1467 break; 1468 #ifdef INVARIANTS 1469 case CRYPTO_AES_NIST_GCM_16: 1470 case CRYPTO_AES_CCM_16: 1471 case CRYPTO_CHACHA20_POLY1305: 1472 panic("bad cipher algo"); 1473 #endif 1474 default: 1475 error = swcr_setup_cipher(ses, csp); 1476 if (error == 0) 1477 ses->swcr_process = swcr_encdec; 1478 } 1479 break; 1480 case CSP_MODE_DIGEST: 1481 error = swcr_setup_auth(ses, csp); 1482 break; 1483 case CSP_MODE_AEAD: 1484 switch (csp->csp_cipher_alg) { 1485 case CRYPTO_AES_NIST_GCM_16: 1486 error = swcr_setup_aead(ses, csp); 1487 if (error == 0) 1488 ses->swcr_process = swcr_gcm; 1489 break; 1490 case CRYPTO_AES_CCM_16: 1491 error = swcr_setup_aead(ses, csp); 1492 if (error == 0) 1493 ses->swcr_process = swcr_ccm; 1494 break; 1495 case CRYPTO_CHACHA20_POLY1305: 1496 error = swcr_setup_aead(ses, csp); 1497 if (error == 0) 1498 ses->swcr_process = swcr_chacha20_poly1305; 1499 break; 1500 #ifdef INVARIANTS 1501 default: 1502 panic("bad aead algo"); 1503 #endif 1504 } 1505 break; 1506 case CSP_MODE_ETA: 1507 #ifdef INVARIANTS 1508 switch (csp->csp_cipher_alg) { 1509 case CRYPTO_AES_NIST_GCM_16: 1510 case CRYPTO_AES_CCM_16: 1511 case CRYPTO_CHACHA20_POLY1305: 1512 panic("bad eta cipher algo"); 1513 } 1514 switch (csp->csp_auth_alg) { 1515 case CRYPTO_AES_NIST_GMAC: 1516 case CRYPTO_AES_CCM_CBC_MAC: 1517 panic("bad eta auth algo"); 1518 } 1519 #endif 1520 1521 error = swcr_setup_auth(ses, csp); 1522 if (error) 1523 break; 1524 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1525 /* Effectively degrade to digest mode. */ 1526 ses->swcr_process = swcr_authcompute; 1527 break; 1528 } 1529 1530 error = swcr_setup_cipher(ses, csp); 1531 if (error == 0) 1532 ses->swcr_process = swcr_eta; 1533 break; 1534 default: 1535 error = EINVAL; 1536 } 1537 1538 if (error) 1539 swcr_freesession(dev, cses); 1540 return (error); 1541 } 1542 1543 static void 1544 swcr_freesession(device_t dev, crypto_session_t cses) 1545 { 1546 struct swcr_session *ses; 1547 1548 ses = crypto_get_driver_session(cses); 1549 1550 mtx_destroy(&ses->swcr_lock); 1551 1552 zfree(ses->swcr_encdec.sw_kschedule, M_CRYPTO_DATA); 1553 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1554 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1555 } 1556 1557 /* 1558 * Process a software request. 1559 */ 1560 static int 1561 swcr_process(device_t dev, struct cryptop *crp, int hint) 1562 { 1563 struct swcr_session *ses; 1564 1565 ses = crypto_get_driver_session(crp->crp_session); 1566 mtx_lock(&ses->swcr_lock); 1567 1568 crp->crp_etype = ses->swcr_process(ses, crp); 1569 1570 mtx_unlock(&ses->swcr_lock); 1571 crypto_done(crp); 1572 return (0); 1573 } 1574 1575 static void 1576 swcr_identify(driver_t *drv, device_t parent) 1577 { 1578 /* NB: order 10 is so we get attached after h/w devices */ 1579 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1580 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1581 panic("cryptosoft: could not attach"); 1582 } 1583 1584 static int 1585 swcr_probe(device_t dev) 1586 { 1587 device_set_desc(dev, "software crypto"); 1588 device_quiet(dev); 1589 return (BUS_PROBE_NOWILDCARD); 1590 } 1591 1592 static int 1593 swcr_attach(device_t dev) 1594 { 1595 1596 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1597 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1598 if (swcr_id < 0) { 1599 device_printf(dev, "cannot initialize!"); 1600 return (ENXIO); 1601 } 1602 1603 return (0); 1604 } 1605 1606 static int 1607 swcr_detach(device_t dev) 1608 { 1609 crypto_unregister_all(swcr_id); 1610 return 0; 1611 } 1612 1613 static device_method_t swcr_methods[] = { 1614 DEVMETHOD(device_identify, swcr_identify), 1615 DEVMETHOD(device_probe, swcr_probe), 1616 DEVMETHOD(device_attach, swcr_attach), 1617 DEVMETHOD(device_detach, swcr_detach), 1618 1619 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1620 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1621 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1622 DEVMETHOD(cryptodev_process, swcr_process), 1623 1624 {0, 0}, 1625 }; 1626 1627 static driver_t swcr_driver = { 1628 "cryptosoft", 1629 swcr_methods, 1630 0, /* NB: no softc */ 1631 }; 1632 static devclass_t swcr_devclass; 1633 1634 /* 1635 * NB: We explicitly reference the crypto module so we 1636 * get the necessary ordering when built as a loadable 1637 * module. This is required because we bundle the crypto 1638 * module code together with the cryptosoft driver (otherwise 1639 * normal module dependencies would handle things). 1640 */ 1641 extern int crypto_modevent(struct module *, int, void *); 1642 /* XXX where to attach */ 1643 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1644 MODULE_VERSION(cryptosoft, 1); 1645 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1646