1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen, todo; 109 int blks, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blks = exf->blocksize; 128 } else 129 blks = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 158 159 /* 160 * Loop through encrypting blocks. 'inlen' is the remaining 161 * length of the current segment in the input buffer. 162 * 'outlen' is the remaining length of current segment in the 163 * output buffer. 164 */ 165 for (resid = crp->crp_payload_length; resid >= blks; resid -= todo) { 166 /* 167 * If the current block is not contained within the 168 * current input/output segment, use 'blk' as a local 169 * buffer. 170 */ 171 if (inlen < blks) { 172 crypto_cursor_copydata(&cc_in, blks, blk); 173 inblk = blk; 174 inlen = blks; 175 } 176 if (outlen < blks) { 177 outblk = blk; 178 outlen = blks; 179 } 180 181 todo = rounddown2(MIN(resid, MIN(inlen, outlen)), blks); 182 183 if (encrypting) 184 exf->encrypt_multi(ctx, inblk, outblk, todo); 185 else 186 exf->decrypt_multi(ctx, inblk, outblk, todo); 187 188 if (inblk == blk) { 189 inblk = crypto_cursor_segment(&cc_in, &inlen); 190 } else { 191 crypto_cursor_advance(&cc_in, todo); 192 inlen -= todo; 193 inblk += todo; 194 if (inlen == 0) 195 inblk = crypto_cursor_segment(&cc_in, &inlen); 196 } 197 198 if (outblk == blk) { 199 crypto_cursor_copyback(&cc_out, blks, blk); 200 outblk = crypto_cursor_segment(&cc_out, &outlen); 201 } else { 202 crypto_cursor_advance(&cc_out, todo); 203 outlen -= todo; 204 outblk += todo; 205 if (outlen == 0) 206 outblk = crypto_cursor_segment(&cc_out, 207 &outlen); 208 } 209 } 210 211 /* Handle trailing partial block for stream ciphers. */ 212 if (resid > 0) { 213 KASSERT(exf->native_blocksize != 0, 214 ("%s: partial block of %d bytes for cipher %s", 215 __func__, resid, exf->name)); 216 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 217 218 inblk = crypto_cursor_segment(&cc_in, &inlen); 219 outblk = crypto_cursor_segment(&cc_out, &outlen); 220 if (inlen < resid) { 221 crypto_cursor_copydata(&cc_in, resid, blk); 222 inblk = blk; 223 } 224 if (outlen < resid) 225 outblk = blk; 226 if (encrypting) 227 exf->encrypt_last(ctx, inblk, outblk, 228 resid); 229 else 230 exf->decrypt_last(ctx, inblk, outblk, 231 resid); 232 if (outlen < resid) 233 crypto_cursor_copyback(&cc_out, resid, blk); 234 } 235 236 explicit_bzero(ctx, exf->ctxsize); 237 explicit_bzero(blk, sizeof(blk)); 238 return (0); 239 } 240 241 /* 242 * Compute or verify hash. 243 */ 244 static int 245 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 246 { 247 struct { 248 union authctx ctx; 249 u_char aalg[HASH_MAX_LEN]; 250 u_char uaalg[HASH_MAX_LEN]; 251 } s; 252 const struct crypto_session_params *csp; 253 const struct swcr_auth *sw; 254 const struct auth_hash *axf; 255 int err; 256 257 sw = &ses->swcr_auth; 258 259 axf = sw->sw_axf; 260 261 csp = crypto_get_params(crp->crp_session); 262 if (crp->crp_auth_key != NULL) { 263 if (sw->sw_hmac) { 264 hmac_init_ipad(axf, crp->crp_auth_key, 265 csp->csp_auth_klen, &s.ctx); 266 } else { 267 axf->Init(&s.ctx); 268 axf->Setkey(&s.ctx, crp->crp_auth_key, 269 csp->csp_auth_klen); 270 } 271 } else 272 memcpy(&s.ctx, sw->sw_ictx, axf->ctxsize); 273 274 if (crp->crp_aad != NULL) 275 err = axf->Update(&s.ctx, crp->crp_aad, crp->crp_aad_length); 276 else 277 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 278 axf->Update, &s.ctx); 279 if (err) 280 goto out; 281 282 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 283 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 284 err = crypto_apply_buf(&crp->crp_obuf, 285 crp->crp_payload_output_start, crp->crp_payload_length, 286 axf->Update, &s.ctx); 287 else 288 err = crypto_apply(crp, crp->crp_payload_start, 289 crp->crp_payload_length, axf->Update, &s.ctx); 290 if (err) 291 goto out; 292 293 if (csp->csp_flags & CSP_F_ESN) 294 axf->Update(&s.ctx, crp->crp_esn, 4); 295 296 axf->Final(s.aalg, &s.ctx); 297 if (sw->sw_hmac) { 298 if (crp->crp_auth_key != NULL) 299 hmac_init_opad(axf, crp->crp_auth_key, 300 csp->csp_auth_klen, &s.ctx); 301 else 302 memcpy(&s.ctx, sw->sw_octx, axf->ctxsize); 303 axf->Update(&s.ctx, s.aalg, axf->hashsize); 304 axf->Final(s.aalg, &s.ctx); 305 } 306 307 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 308 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, s.uaalg); 309 if (timingsafe_bcmp(s.aalg, s.uaalg, sw->sw_mlen) != 0) 310 err = EBADMSG; 311 } else { 312 /* Inject the authentication data */ 313 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, s.aalg); 314 } 315 out: 316 explicit_bzero(&s, sizeof(s)); 317 return (err); 318 } 319 320 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 321 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 322 323 static int 324 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 325 { 326 struct { 327 union authctx ctx; 328 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 329 u_char tag[GMAC_DIGEST_LEN]; 330 u_char tag2[GMAC_DIGEST_LEN]; 331 } s; 332 u_char *blk = (u_char *)s.blkbuf; 333 struct crypto_buffer_cursor cc; 334 const u_char *inblk; 335 const struct swcr_auth *swa; 336 const struct auth_hash *axf; 337 uint32_t *blkp; 338 size_t len; 339 int blksz, error, ivlen, resid; 340 341 swa = &ses->swcr_auth; 342 axf = swa->sw_axf; 343 blksz = GMAC_BLOCK_LEN; 344 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 345 __func__)); 346 347 if (crp->crp_auth_key != NULL) { 348 axf->Init(&s.ctx); 349 axf->Setkey(&s.ctx, crp->crp_auth_key, 350 crypto_get_params(crp->crp_session)->csp_auth_klen); 351 } else 352 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 353 354 /* Initialize the IV */ 355 ivlen = AES_GCM_IV_LEN; 356 crypto_read_iv(crp, blk); 357 358 axf->Reinit(&s.ctx, blk, ivlen); 359 crypto_cursor_init(&cc, &crp->crp_buf); 360 crypto_cursor_advance(&cc, crp->crp_payload_start); 361 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 362 inblk = crypto_cursor_segment(&cc, &len); 363 if (len >= blksz) { 364 len = rounddown(MIN(len, resid), blksz); 365 crypto_cursor_advance(&cc, len); 366 } else { 367 len = blksz; 368 crypto_cursor_copydata(&cc, len, blk); 369 inblk = blk; 370 } 371 axf->Update(&s.ctx, inblk, len); 372 } 373 if (resid > 0) { 374 memset(blk, 0, blksz); 375 crypto_cursor_copydata(&cc, resid, blk); 376 axf->Update(&s.ctx, blk, blksz); 377 } 378 379 /* length block */ 380 memset(blk, 0, blksz); 381 blkp = (uint32_t *)blk + 1; 382 *blkp = htobe32(crp->crp_payload_length * 8); 383 axf->Update(&s.ctx, blk, blksz); 384 385 /* Finalize MAC */ 386 axf->Final(s.tag, &s.ctx); 387 388 error = 0; 389 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 390 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 391 s.tag2); 392 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 393 error = EBADMSG; 394 } else { 395 /* Inject the authentication data */ 396 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, s.tag); 397 } 398 explicit_bzero(&s, sizeof(s)); 399 return (error); 400 } 401 402 static int 403 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 404 { 405 struct { 406 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 407 u_char tag[GMAC_DIGEST_LEN]; 408 u_char tag2[GMAC_DIGEST_LEN]; 409 } s; 410 u_char *blk = (u_char *)s.blkbuf; 411 struct crypto_buffer_cursor cc_in, cc_out; 412 const u_char *inblk; 413 u_char *outblk; 414 const struct swcr_auth *swa; 415 const struct swcr_encdec *swe; 416 const struct enc_xform *exf; 417 void *ctx; 418 uint32_t *blkp; 419 size_t len; 420 int blksz, error, ivlen, r, resid; 421 422 swa = &ses->swcr_auth; 423 swe = &ses->swcr_encdec; 424 exf = swe->sw_exf; 425 blksz = GMAC_BLOCK_LEN; 426 KASSERT(blksz == exf->native_blocksize, 427 ("%s: blocksize mismatch", __func__)); 428 429 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 430 return (EINVAL); 431 432 ivlen = AES_GCM_IV_LEN; 433 434 ctx = __builtin_alloca(exf->ctxsize); 435 if (crp->crp_cipher_key != NULL) 436 exf->setkey(ctx, crp->crp_cipher_key, 437 crypto_get_params(crp->crp_session)->csp_cipher_klen); 438 else 439 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 440 exf->reinit(ctx, crp->crp_iv, ivlen); 441 442 /* Supply MAC with AAD */ 443 if (crp->crp_aad != NULL) { 444 len = rounddown(crp->crp_aad_length, blksz); 445 if (len != 0) 446 exf->update(ctx, crp->crp_aad, len); 447 if (crp->crp_aad_length != len) { 448 memset(blk, 0, blksz); 449 memcpy(blk, (char *)crp->crp_aad + len, 450 crp->crp_aad_length - len); 451 exf->update(ctx, blk, blksz); 452 } 453 } else { 454 crypto_cursor_init(&cc_in, &crp->crp_buf); 455 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 456 for (resid = crp->crp_aad_length; resid >= blksz; 457 resid -= len) { 458 inblk = crypto_cursor_segment(&cc_in, &len); 459 if (len >= blksz) { 460 len = rounddown(MIN(len, resid), blksz); 461 crypto_cursor_advance(&cc_in, len); 462 } else { 463 len = blksz; 464 crypto_cursor_copydata(&cc_in, len, blk); 465 inblk = blk; 466 } 467 exf->update(ctx, inblk, len); 468 } 469 if (resid > 0) { 470 memset(blk, 0, blksz); 471 crypto_cursor_copydata(&cc_in, resid, blk); 472 exf->update(ctx, blk, blksz); 473 } 474 } 475 476 /* Do encryption with MAC */ 477 crypto_cursor_init(&cc_in, &crp->crp_buf); 478 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 479 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 480 crypto_cursor_init(&cc_out, &crp->crp_obuf); 481 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 482 } else 483 cc_out = cc_in; 484 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 485 inblk = crypto_cursor_segment(&cc_in, &len); 486 if (len < blksz) { 487 crypto_cursor_copydata(&cc_in, blksz, blk); 488 inblk = blk; 489 } else { 490 crypto_cursor_advance(&cc_in, blksz); 491 } 492 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 493 outblk = crypto_cursor_segment(&cc_out, &len); 494 if (len < blksz) 495 outblk = blk; 496 exf->encrypt(ctx, inblk, outblk); 497 exf->update(ctx, outblk, blksz); 498 if (outblk == blk) 499 crypto_cursor_copyback(&cc_out, blksz, blk); 500 else 501 crypto_cursor_advance(&cc_out, blksz); 502 } else { 503 exf->update(ctx, inblk, blksz); 504 } 505 } 506 if (resid > 0) { 507 crypto_cursor_copydata(&cc_in, resid, blk); 508 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 509 exf->encrypt_last(ctx, blk, blk, resid); 510 crypto_cursor_copyback(&cc_out, resid, blk); 511 } 512 exf->update(ctx, blk, resid); 513 } 514 515 /* length block */ 516 memset(blk, 0, blksz); 517 blkp = (uint32_t *)blk + 1; 518 *blkp = htobe32(crp->crp_aad_length * 8); 519 blkp = (uint32_t *)blk + 3; 520 *blkp = htobe32(crp->crp_payload_length * 8); 521 exf->update(ctx, blk, blksz); 522 523 /* Finalize MAC */ 524 exf->final(s.tag, ctx); 525 526 /* Validate tag */ 527 error = 0; 528 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 529 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 530 s.tag2); 531 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 532 if (r != 0) { 533 error = EBADMSG; 534 goto out; 535 } 536 537 /* tag matches, decrypt data */ 538 crypto_cursor_init(&cc_in, &crp->crp_buf); 539 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 540 for (resid = crp->crp_payload_length; resid > blksz; 541 resid -= blksz) { 542 inblk = crypto_cursor_segment(&cc_in, &len); 543 if (len < blksz) { 544 crypto_cursor_copydata(&cc_in, blksz, blk); 545 inblk = blk; 546 } else 547 crypto_cursor_advance(&cc_in, blksz); 548 outblk = crypto_cursor_segment(&cc_out, &len); 549 if (len < blksz) 550 outblk = blk; 551 exf->decrypt(ctx, inblk, outblk); 552 if (outblk == blk) 553 crypto_cursor_copyback(&cc_out, blksz, blk); 554 else 555 crypto_cursor_advance(&cc_out, blksz); 556 } 557 if (resid > 0) { 558 crypto_cursor_copydata(&cc_in, resid, blk); 559 exf->decrypt_last(ctx, blk, blk, resid); 560 crypto_cursor_copyback(&cc_out, resid, blk); 561 } 562 } else { 563 /* Inject the authentication data */ 564 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 565 s.tag); 566 } 567 568 out: 569 explicit_bzero(ctx, exf->ctxsize); 570 explicit_bzero(&s, sizeof(s)); 571 572 return (error); 573 } 574 575 static void 576 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 577 u_int data_length, u_int tag_length, uint8_t *b0) 578 { 579 uint8_t *bp; 580 uint8_t flags, L; 581 582 KASSERT(nonce_length >= 7 && nonce_length <= 13, 583 ("nonce_length must be between 7 and 13 bytes")); 584 585 /* 586 * Need to determine the L field value. This is the number of 587 * bytes needed to specify the length of the message; the length 588 * is whatever is left in the 16 bytes after specifying flags and 589 * the nonce. 590 */ 591 L = 15 - nonce_length; 592 593 flags = ((aad_length > 0) << 6) + 594 (((tag_length - 2) / 2) << 3) + 595 L - 1; 596 597 /* 598 * Now we need to set up the first block, which has flags, nonce, 599 * and the message length. 600 */ 601 b0[0] = flags; 602 memcpy(b0 + 1, nonce, nonce_length); 603 bp = b0 + 1 + nonce_length; 604 605 /* Need to copy L' [aka L-1] bytes of data_length */ 606 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 607 *dst = data_length; 608 data_length >>= 8; 609 } 610 } 611 612 /* NB: OCF only supports AAD lengths < 2^32. */ 613 static int 614 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 615 { 616 if (aad_length < ((1 << 16) - (1 << 8))) { 617 be16enc(blk, aad_length); 618 return (sizeof(uint16_t)); 619 } else { 620 blk[0] = 0xff; 621 blk[1] = 0xfe; 622 be32enc(blk + 2, aad_length); 623 return (2 + sizeof(uint32_t)); 624 } 625 } 626 627 static int 628 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 629 { 630 struct { 631 union authctx ctx; 632 u_char blk[CCM_CBC_BLOCK_LEN]; 633 u_char tag[AES_CBC_MAC_HASH_LEN]; 634 u_char tag2[AES_CBC_MAC_HASH_LEN]; 635 } s; 636 const struct crypto_session_params *csp; 637 const struct swcr_auth *swa; 638 const struct auth_hash *axf; 639 int error, ivlen, len; 640 641 csp = crypto_get_params(crp->crp_session); 642 swa = &ses->swcr_auth; 643 axf = swa->sw_axf; 644 645 if (crp->crp_auth_key != NULL) { 646 axf->Init(&s.ctx); 647 axf->Setkey(&s.ctx, crp->crp_auth_key, csp->csp_auth_klen); 648 } else 649 memcpy(&s.ctx, swa->sw_ictx, axf->ctxsize); 650 651 /* Initialize the IV */ 652 ivlen = csp->csp_ivlen; 653 654 /* Supply MAC with IV */ 655 axf->Reinit(&s.ctx, crp->crp_iv, ivlen); 656 657 /* Supply MAC with b0. */ 658 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 659 swa->sw_mlen, s.blk); 660 axf->Update(&s.ctx, s.blk, CCM_CBC_BLOCK_LEN); 661 662 len = build_ccm_aad_length(crp->crp_payload_length, s.blk); 663 axf->Update(&s.ctx, s.blk, len); 664 665 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 666 axf->Update, &s.ctx); 667 668 /* Finalize MAC */ 669 axf->Final(s.tag, &s.ctx); 670 671 error = 0; 672 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 673 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 674 s.tag2); 675 if (timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen) != 0) 676 error = EBADMSG; 677 } else { 678 /* Inject the authentication data */ 679 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 680 s.tag); 681 } 682 explicit_bzero(&s, sizeof(s)); 683 return (error); 684 } 685 686 static int 687 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 688 { 689 const struct crypto_session_params *csp; 690 struct { 691 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 692 u_char tag[AES_CBC_MAC_HASH_LEN]; 693 u_char tag2[AES_CBC_MAC_HASH_LEN]; 694 } s; 695 u_char *blk = (u_char *)s.blkbuf; 696 struct crypto_buffer_cursor cc_in, cc_out; 697 const u_char *inblk; 698 u_char *outblk; 699 const struct swcr_auth *swa; 700 const struct swcr_encdec *swe; 701 const struct enc_xform *exf; 702 void *ctx; 703 size_t len; 704 int blksz, error, ivlen, r, resid; 705 706 csp = crypto_get_params(crp->crp_session); 707 swa = &ses->swcr_auth; 708 swe = &ses->swcr_encdec; 709 exf = swe->sw_exf; 710 blksz = AES_BLOCK_LEN; 711 KASSERT(blksz == exf->native_blocksize, 712 ("%s: blocksize mismatch", __func__)); 713 714 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 715 return (EMSGSIZE); 716 717 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 718 return (EINVAL); 719 720 ivlen = csp->csp_ivlen; 721 722 ctx = __builtin_alloca(exf->ctxsize); 723 if (crp->crp_cipher_key != NULL) 724 exf->setkey(ctx, crp->crp_cipher_key, 725 crypto_get_params(crp->crp_session)->csp_cipher_klen); 726 else 727 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 728 exf->reinit(ctx, crp->crp_iv, ivlen); 729 730 /* Supply MAC with b0. */ 731 _Static_assert(sizeof(s.blkbuf) >= CCM_CBC_BLOCK_LEN, 732 "blkbuf too small for b0"); 733 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 734 crp->crp_payload_length, swa->sw_mlen, blk); 735 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 736 737 /* Supply MAC with AAD */ 738 if (crp->crp_aad_length != 0) { 739 len = build_ccm_aad_length(crp->crp_aad_length, blk); 740 exf->update(ctx, blk, len); 741 if (crp->crp_aad != NULL) 742 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 743 else 744 crypto_apply(crp, crp->crp_aad_start, 745 crp->crp_aad_length, exf->update, ctx); 746 747 /* Pad the AAD (including length field) to a full block. */ 748 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 749 if (len != 0) { 750 len = CCM_CBC_BLOCK_LEN - len; 751 memset(blk, 0, CCM_CBC_BLOCK_LEN); 752 exf->update(ctx, blk, len); 753 } 754 } 755 756 /* Do encryption/decryption with MAC */ 757 crypto_cursor_init(&cc_in, &crp->crp_buf); 758 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 759 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 760 crypto_cursor_init(&cc_out, &crp->crp_obuf); 761 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 762 } else 763 cc_out = cc_in; 764 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 765 inblk = crypto_cursor_segment(&cc_in, &len); 766 if (len < blksz) { 767 crypto_cursor_copydata(&cc_in, blksz, blk); 768 inblk = blk; 769 } else 770 crypto_cursor_advance(&cc_in, blksz); 771 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 772 outblk = crypto_cursor_segment(&cc_out, &len); 773 if (len < blksz) 774 outblk = blk; 775 exf->update(ctx, inblk, blksz); 776 exf->encrypt(ctx, inblk, outblk); 777 if (outblk == blk) 778 crypto_cursor_copyback(&cc_out, blksz, blk); 779 else 780 crypto_cursor_advance(&cc_out, blksz); 781 } else { 782 /* 783 * One of the problems with CCM+CBC is that 784 * the authentication is done on the 785 * unencrypted data. As a result, we have to 786 * decrypt the data twice: once to generate 787 * the tag and a second time after the tag is 788 * verified. 789 */ 790 exf->decrypt(ctx, inblk, blk); 791 exf->update(ctx, blk, blksz); 792 } 793 } 794 if (resid > 0) { 795 crypto_cursor_copydata(&cc_in, resid, blk); 796 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 797 exf->update(ctx, blk, resid); 798 exf->encrypt_last(ctx, blk, blk, resid); 799 crypto_cursor_copyback(&cc_out, resid, blk); 800 } else { 801 exf->decrypt_last(ctx, blk, blk, resid); 802 exf->update(ctx, blk, resid); 803 } 804 } 805 806 /* Finalize MAC */ 807 exf->final(s.tag, ctx); 808 809 /* Validate tag */ 810 error = 0; 811 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 812 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 813 s.tag2); 814 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 815 if (r != 0) { 816 error = EBADMSG; 817 goto out; 818 } 819 820 /* tag matches, decrypt data */ 821 exf->reinit(ctx, crp->crp_iv, ivlen); 822 crypto_cursor_init(&cc_in, &crp->crp_buf); 823 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 824 for (resid = crp->crp_payload_length; resid > blksz; 825 resid -= blksz) { 826 inblk = crypto_cursor_segment(&cc_in, &len); 827 if (len < blksz) { 828 crypto_cursor_copydata(&cc_in, blksz, blk); 829 inblk = blk; 830 } else 831 crypto_cursor_advance(&cc_in, blksz); 832 outblk = crypto_cursor_segment(&cc_out, &len); 833 if (len < blksz) 834 outblk = blk; 835 exf->decrypt(ctx, inblk, outblk); 836 if (outblk == blk) 837 crypto_cursor_copyback(&cc_out, blksz, blk); 838 else 839 crypto_cursor_advance(&cc_out, blksz); 840 } 841 if (resid > 0) { 842 crypto_cursor_copydata(&cc_in, resid, blk); 843 exf->decrypt_last(ctx, blk, blk, resid); 844 crypto_cursor_copyback(&cc_out, resid, blk); 845 } 846 } else { 847 /* Inject the authentication data */ 848 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 849 s.tag); 850 } 851 852 out: 853 explicit_bzero(ctx, exf->ctxsize); 854 explicit_bzero(&s, sizeof(s)); 855 return (error); 856 } 857 858 static int 859 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 860 { 861 const struct crypto_session_params *csp; 862 struct { 863 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 864 u_char tag[POLY1305_HASH_LEN]; 865 u_char tag2[POLY1305_HASH_LEN]; 866 } s; 867 u_char *blk = (u_char *)s.blkbuf; 868 struct crypto_buffer_cursor cc_in, cc_out; 869 const u_char *inblk; 870 u_char *outblk; 871 uint64_t *blkp; 872 const struct swcr_auth *swa; 873 const struct swcr_encdec *swe; 874 const struct enc_xform *exf; 875 void *ctx; 876 size_t len; 877 int blksz, error, r, resid; 878 879 swa = &ses->swcr_auth; 880 swe = &ses->swcr_encdec; 881 exf = swe->sw_exf; 882 blksz = exf->native_blocksize; 883 KASSERT(blksz <= sizeof(s.blkbuf), ("%s: blocksize mismatch", __func__)); 884 885 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 886 return (EINVAL); 887 888 csp = crypto_get_params(crp->crp_session); 889 890 ctx = __builtin_alloca(exf->ctxsize); 891 if (crp->crp_cipher_key != NULL) 892 exf->setkey(ctx, crp->crp_cipher_key, 893 csp->csp_cipher_klen); 894 else 895 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 896 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 897 898 /* Supply MAC with AAD */ 899 if (crp->crp_aad != NULL) 900 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 901 else 902 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 903 exf->update, ctx); 904 if (crp->crp_aad_length % POLY1305_BLOCK_LEN != 0) { 905 /* padding1 */ 906 memset(blk, 0, POLY1305_BLOCK_LEN); 907 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 908 crp->crp_aad_length % POLY1305_BLOCK_LEN); 909 } 910 911 /* Do encryption with MAC */ 912 crypto_cursor_init(&cc_in, &crp->crp_buf); 913 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 914 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 915 crypto_cursor_init(&cc_out, &crp->crp_obuf); 916 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 917 } else 918 cc_out = cc_in; 919 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 920 inblk = crypto_cursor_segment(&cc_in, &len); 921 if (len < blksz) { 922 crypto_cursor_copydata(&cc_in, blksz, blk); 923 inblk = blk; 924 } else 925 crypto_cursor_advance(&cc_in, blksz); 926 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 927 outblk = crypto_cursor_segment(&cc_out, &len); 928 if (len < blksz) 929 outblk = blk; 930 exf->encrypt(ctx, inblk, outblk); 931 exf->update(ctx, outblk, blksz); 932 if (outblk == blk) 933 crypto_cursor_copyback(&cc_out, blksz, blk); 934 else 935 crypto_cursor_advance(&cc_out, blksz); 936 } else { 937 exf->update(ctx, inblk, blksz); 938 } 939 } 940 if (resid > 0) { 941 crypto_cursor_copydata(&cc_in, resid, blk); 942 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 943 exf->encrypt_last(ctx, blk, blk, resid); 944 crypto_cursor_copyback(&cc_out, resid, blk); 945 } 946 exf->update(ctx, blk, resid); 947 if (resid % POLY1305_BLOCK_LEN != 0) { 948 /* padding2 */ 949 memset(blk, 0, POLY1305_BLOCK_LEN); 950 exf->update(ctx, blk, POLY1305_BLOCK_LEN - 951 resid % POLY1305_BLOCK_LEN); 952 } 953 } 954 955 /* lengths */ 956 blkp = (uint64_t *)blk; 957 blkp[0] = htole64(crp->crp_aad_length); 958 blkp[1] = htole64(crp->crp_payload_length); 959 exf->update(ctx, blk, sizeof(uint64_t) * 2); 960 961 /* Finalize MAC */ 962 exf->final(s.tag, ctx); 963 964 /* Validate tag */ 965 error = 0; 966 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 967 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 968 s.tag2); 969 r = timingsafe_bcmp(s.tag, s.tag2, swa->sw_mlen); 970 if (r != 0) { 971 error = EBADMSG; 972 goto out; 973 } 974 975 /* tag matches, decrypt data */ 976 crypto_cursor_init(&cc_in, &crp->crp_buf); 977 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 978 for (resid = crp->crp_payload_length; resid > blksz; 979 resid -= blksz) { 980 inblk = crypto_cursor_segment(&cc_in, &len); 981 if (len < blksz) { 982 crypto_cursor_copydata(&cc_in, blksz, blk); 983 inblk = blk; 984 } else 985 crypto_cursor_advance(&cc_in, blksz); 986 outblk = crypto_cursor_segment(&cc_out, &len); 987 if (len < blksz) 988 outblk = blk; 989 exf->decrypt(ctx, inblk, outblk); 990 if (outblk == blk) 991 crypto_cursor_copyback(&cc_out, blksz, blk); 992 else 993 crypto_cursor_advance(&cc_out, blksz); 994 } 995 if (resid > 0) { 996 crypto_cursor_copydata(&cc_in, resid, blk); 997 exf->decrypt_last(ctx, blk, blk, resid); 998 crypto_cursor_copyback(&cc_out, resid, blk); 999 } 1000 } else { 1001 /* Inject the authentication data */ 1002 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, 1003 s.tag); 1004 } 1005 1006 out: 1007 explicit_bzero(ctx, exf->ctxsize); 1008 explicit_bzero(&s, sizeof(s)); 1009 return (error); 1010 } 1011 1012 /* 1013 * Apply a cipher and a digest to perform EtA. 1014 */ 1015 static int 1016 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1017 { 1018 int error; 1019 1020 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1021 error = swcr_encdec(ses, crp); 1022 if (error == 0) 1023 error = swcr_authcompute(ses, crp); 1024 } else { 1025 error = swcr_authcompute(ses, crp); 1026 if (error == 0) 1027 error = swcr_encdec(ses, crp); 1028 } 1029 return (error); 1030 } 1031 1032 /* 1033 * Apply a compression/decompression algorithm 1034 */ 1035 static int 1036 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1037 { 1038 const struct comp_algo *cxf; 1039 uint8_t *data, *out; 1040 int adj; 1041 uint32_t result; 1042 1043 cxf = ses->swcr_compdec.sw_cxf; 1044 1045 /* We must handle the whole buffer of data in one time 1046 * then if there is not all the data in the mbuf, we must 1047 * copy in a buffer. 1048 */ 1049 1050 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1051 if (data == NULL) 1052 return (EINVAL); 1053 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1054 data); 1055 1056 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1057 result = cxf->compress(data, crp->crp_payload_length, &out); 1058 else 1059 result = cxf->decompress(data, crp->crp_payload_length, &out); 1060 1061 free(data, M_CRYPTO_DATA); 1062 if (result == 0) 1063 return (EINVAL); 1064 crp->crp_olen = result; 1065 1066 /* Check the compressed size when doing compression */ 1067 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1068 if (result >= crp->crp_payload_length) { 1069 /* Compression was useless, we lost time */ 1070 free(out, M_CRYPTO_DATA); 1071 return (0); 1072 } 1073 } 1074 1075 /* Copy back the (de)compressed data. m_copyback is 1076 * extending the mbuf as necessary. 1077 */ 1078 crypto_copyback(crp, crp->crp_payload_start, result, out); 1079 if (result < crp->crp_payload_length) { 1080 switch (crp->crp_buf.cb_type) { 1081 case CRYPTO_BUF_MBUF: 1082 case CRYPTO_BUF_SINGLE_MBUF: 1083 adj = result - crp->crp_payload_length; 1084 m_adj(crp->crp_buf.cb_mbuf, adj); 1085 break; 1086 case CRYPTO_BUF_UIO: { 1087 struct uio *uio = crp->crp_buf.cb_uio; 1088 int ind; 1089 1090 adj = crp->crp_payload_length - result; 1091 ind = uio->uio_iovcnt - 1; 1092 1093 while (adj > 0 && ind >= 0) { 1094 if (adj < uio->uio_iov[ind].iov_len) { 1095 uio->uio_iov[ind].iov_len -= adj; 1096 break; 1097 } 1098 1099 adj -= uio->uio_iov[ind].iov_len; 1100 uio->uio_iov[ind].iov_len = 0; 1101 ind--; 1102 uio->uio_iovcnt--; 1103 } 1104 } 1105 break; 1106 case CRYPTO_BUF_VMPAGE: 1107 adj = crp->crp_payload_length - result; 1108 crp->crp_buf.cb_vm_page_len -= adj; 1109 break; 1110 default: 1111 break; 1112 } 1113 } 1114 free(out, M_CRYPTO_DATA); 1115 return 0; 1116 } 1117 1118 static int 1119 swcr_setup_cipher(struct swcr_session *ses, 1120 const struct crypto_session_params *csp) 1121 { 1122 struct swcr_encdec *swe; 1123 const struct enc_xform *txf; 1124 int error; 1125 1126 swe = &ses->swcr_encdec; 1127 txf = crypto_cipher(csp); 1128 if (csp->csp_cipher_key != NULL) { 1129 if (txf->ctxsize != 0) { 1130 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1131 M_NOWAIT); 1132 if (swe->sw_ctx == NULL) 1133 return (ENOMEM); 1134 } 1135 error = txf->setkey(swe->sw_ctx, 1136 csp->csp_cipher_key, csp->csp_cipher_klen); 1137 if (error) 1138 return (error); 1139 } 1140 swe->sw_exf = txf; 1141 return (0); 1142 } 1143 1144 static int 1145 swcr_setup_auth(struct swcr_session *ses, 1146 const struct crypto_session_params *csp) 1147 { 1148 struct swcr_auth *swa; 1149 const struct auth_hash *axf; 1150 1151 swa = &ses->swcr_auth; 1152 1153 axf = crypto_auth_hash(csp); 1154 swa->sw_axf = axf; 1155 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1156 return (EINVAL); 1157 if (csp->csp_auth_mlen == 0) 1158 swa->sw_mlen = axf->hashsize; 1159 else 1160 swa->sw_mlen = csp->csp_auth_mlen; 1161 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1162 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1163 M_NOWAIT); 1164 if (swa->sw_ictx == NULL) 1165 return (ENOBUFS); 1166 } 1167 1168 switch (csp->csp_auth_alg) { 1169 case CRYPTO_SHA1_HMAC: 1170 case CRYPTO_SHA2_224_HMAC: 1171 case CRYPTO_SHA2_256_HMAC: 1172 case CRYPTO_SHA2_384_HMAC: 1173 case CRYPTO_SHA2_512_HMAC: 1174 case CRYPTO_RIPEMD160_HMAC: 1175 swa->sw_hmac = true; 1176 if (csp->csp_auth_key != NULL) { 1177 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1178 M_NOWAIT); 1179 if (swa->sw_octx == NULL) 1180 return (ENOBUFS); 1181 hmac_init_ipad(axf, csp->csp_auth_key, 1182 csp->csp_auth_klen, swa->sw_ictx); 1183 hmac_init_opad(axf, csp->csp_auth_key, 1184 csp->csp_auth_klen, swa->sw_octx); 1185 } 1186 break; 1187 case CRYPTO_RIPEMD160: 1188 case CRYPTO_SHA1: 1189 case CRYPTO_SHA2_224: 1190 case CRYPTO_SHA2_256: 1191 case CRYPTO_SHA2_384: 1192 case CRYPTO_SHA2_512: 1193 case CRYPTO_NULL_HMAC: 1194 axf->Init(swa->sw_ictx); 1195 break; 1196 case CRYPTO_AES_NIST_GMAC: 1197 case CRYPTO_AES_CCM_CBC_MAC: 1198 case CRYPTO_POLY1305: 1199 if (csp->csp_auth_key != NULL) { 1200 axf->Init(swa->sw_ictx); 1201 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1202 csp->csp_auth_klen); 1203 } 1204 break; 1205 case CRYPTO_BLAKE2B: 1206 case CRYPTO_BLAKE2S: 1207 /* 1208 * Blake2b and Blake2s support an optional key but do 1209 * not require one. 1210 */ 1211 if (csp->csp_auth_klen == 0) 1212 axf->Init(swa->sw_ictx); 1213 else if (csp->csp_auth_key != NULL) 1214 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1215 csp->csp_auth_klen); 1216 break; 1217 } 1218 1219 if (csp->csp_mode == CSP_MODE_DIGEST) { 1220 switch (csp->csp_auth_alg) { 1221 case CRYPTO_AES_NIST_GMAC: 1222 ses->swcr_process = swcr_gmac; 1223 break; 1224 case CRYPTO_AES_CCM_CBC_MAC: 1225 ses->swcr_process = swcr_ccm_cbc_mac; 1226 break; 1227 default: 1228 ses->swcr_process = swcr_authcompute; 1229 } 1230 } 1231 1232 return (0); 1233 } 1234 1235 static int 1236 swcr_setup_aead(struct swcr_session *ses, 1237 const struct crypto_session_params *csp) 1238 { 1239 struct swcr_auth *swa; 1240 int error; 1241 1242 error = swcr_setup_cipher(ses, csp); 1243 if (error) 1244 return (error); 1245 1246 swa = &ses->swcr_auth; 1247 if (csp->csp_auth_mlen == 0) 1248 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1249 else 1250 swa->sw_mlen = csp->csp_auth_mlen; 1251 return (0); 1252 } 1253 1254 static bool 1255 swcr_auth_supported(const struct crypto_session_params *csp) 1256 { 1257 const struct auth_hash *axf; 1258 1259 axf = crypto_auth_hash(csp); 1260 if (axf == NULL) 1261 return (false); 1262 switch (csp->csp_auth_alg) { 1263 case CRYPTO_SHA1_HMAC: 1264 case CRYPTO_SHA2_224_HMAC: 1265 case CRYPTO_SHA2_256_HMAC: 1266 case CRYPTO_SHA2_384_HMAC: 1267 case CRYPTO_SHA2_512_HMAC: 1268 case CRYPTO_NULL_HMAC: 1269 case CRYPTO_RIPEMD160_HMAC: 1270 break; 1271 case CRYPTO_AES_NIST_GMAC: 1272 switch (csp->csp_auth_klen * 8) { 1273 case 128: 1274 case 192: 1275 case 256: 1276 break; 1277 default: 1278 return (false); 1279 } 1280 if (csp->csp_auth_key == NULL) 1281 return (false); 1282 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1283 return (false); 1284 break; 1285 case CRYPTO_POLY1305: 1286 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1287 return (false); 1288 break; 1289 case CRYPTO_AES_CCM_CBC_MAC: 1290 switch (csp->csp_auth_klen * 8) { 1291 case 128: 1292 case 192: 1293 case 256: 1294 break; 1295 default: 1296 return (false); 1297 } 1298 if (csp->csp_auth_key == NULL) 1299 return (false); 1300 break; 1301 } 1302 return (true); 1303 } 1304 1305 static bool 1306 swcr_cipher_supported(const struct crypto_session_params *csp) 1307 { 1308 const struct enc_xform *txf; 1309 1310 txf = crypto_cipher(csp); 1311 if (txf == NULL) 1312 return (false); 1313 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1314 txf->ivsize != csp->csp_ivlen) 1315 return (false); 1316 return (true); 1317 } 1318 1319 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1320 1321 static int 1322 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1323 { 1324 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1325 return (EINVAL); 1326 switch (csp->csp_mode) { 1327 case CSP_MODE_COMPRESS: 1328 switch (csp->csp_cipher_alg) { 1329 case CRYPTO_DEFLATE_COMP: 1330 break; 1331 default: 1332 return (EINVAL); 1333 } 1334 break; 1335 case CSP_MODE_CIPHER: 1336 switch (csp->csp_cipher_alg) { 1337 case CRYPTO_AES_NIST_GCM_16: 1338 case CRYPTO_AES_CCM_16: 1339 case CRYPTO_CHACHA20_POLY1305: 1340 case CRYPTO_XCHACHA20_POLY1305: 1341 return (EINVAL); 1342 default: 1343 if (!swcr_cipher_supported(csp)) 1344 return (EINVAL); 1345 break; 1346 } 1347 break; 1348 case CSP_MODE_DIGEST: 1349 if (!swcr_auth_supported(csp)) 1350 return (EINVAL); 1351 break; 1352 case CSP_MODE_AEAD: 1353 switch (csp->csp_cipher_alg) { 1354 case CRYPTO_AES_NIST_GCM_16: 1355 case CRYPTO_AES_CCM_16: 1356 switch (csp->csp_cipher_klen * 8) { 1357 case 128: 1358 case 192: 1359 case 256: 1360 break; 1361 default: 1362 return (EINVAL); 1363 } 1364 break; 1365 case CRYPTO_CHACHA20_POLY1305: 1366 case CRYPTO_XCHACHA20_POLY1305: 1367 break; 1368 default: 1369 return (EINVAL); 1370 } 1371 break; 1372 case CSP_MODE_ETA: 1373 /* AEAD algorithms cannot be used for EtA. */ 1374 switch (csp->csp_cipher_alg) { 1375 case CRYPTO_AES_NIST_GCM_16: 1376 case CRYPTO_AES_CCM_16: 1377 case CRYPTO_CHACHA20_POLY1305: 1378 case CRYPTO_XCHACHA20_POLY1305: 1379 return (EINVAL); 1380 } 1381 switch (csp->csp_auth_alg) { 1382 case CRYPTO_AES_NIST_GMAC: 1383 case CRYPTO_AES_CCM_CBC_MAC: 1384 return (EINVAL); 1385 } 1386 1387 if (!swcr_cipher_supported(csp) || 1388 !swcr_auth_supported(csp)) 1389 return (EINVAL); 1390 break; 1391 default: 1392 return (EINVAL); 1393 } 1394 1395 return (CRYPTODEV_PROBE_SOFTWARE); 1396 } 1397 1398 /* 1399 * Generate a new software session. 1400 */ 1401 static int 1402 swcr_newsession(device_t dev, crypto_session_t cses, 1403 const struct crypto_session_params *csp) 1404 { 1405 struct swcr_session *ses; 1406 const struct comp_algo *cxf; 1407 int error; 1408 1409 ses = crypto_get_driver_session(cses); 1410 1411 error = 0; 1412 switch (csp->csp_mode) { 1413 case CSP_MODE_COMPRESS: 1414 switch (csp->csp_cipher_alg) { 1415 case CRYPTO_DEFLATE_COMP: 1416 cxf = &comp_algo_deflate; 1417 break; 1418 #ifdef INVARIANTS 1419 default: 1420 panic("bad compression algo"); 1421 #endif 1422 } 1423 ses->swcr_compdec.sw_cxf = cxf; 1424 ses->swcr_process = swcr_compdec; 1425 break; 1426 case CSP_MODE_CIPHER: 1427 switch (csp->csp_cipher_alg) { 1428 case CRYPTO_NULL_CBC: 1429 ses->swcr_process = swcr_null; 1430 break; 1431 #ifdef INVARIANTS 1432 case CRYPTO_AES_NIST_GCM_16: 1433 case CRYPTO_AES_CCM_16: 1434 case CRYPTO_CHACHA20_POLY1305: 1435 case CRYPTO_XCHACHA20_POLY1305: 1436 panic("bad cipher algo"); 1437 #endif 1438 default: 1439 error = swcr_setup_cipher(ses, csp); 1440 if (error == 0) 1441 ses->swcr_process = swcr_encdec; 1442 } 1443 break; 1444 case CSP_MODE_DIGEST: 1445 error = swcr_setup_auth(ses, csp); 1446 break; 1447 case CSP_MODE_AEAD: 1448 switch (csp->csp_cipher_alg) { 1449 case CRYPTO_AES_NIST_GCM_16: 1450 error = swcr_setup_aead(ses, csp); 1451 if (error == 0) 1452 ses->swcr_process = swcr_gcm; 1453 break; 1454 case CRYPTO_AES_CCM_16: 1455 error = swcr_setup_aead(ses, csp); 1456 if (error == 0) 1457 ses->swcr_process = swcr_ccm; 1458 break; 1459 case CRYPTO_CHACHA20_POLY1305: 1460 case CRYPTO_XCHACHA20_POLY1305: 1461 error = swcr_setup_aead(ses, csp); 1462 if (error == 0) 1463 ses->swcr_process = swcr_chacha20_poly1305; 1464 break; 1465 #ifdef INVARIANTS 1466 default: 1467 panic("bad aead algo"); 1468 #endif 1469 } 1470 break; 1471 case CSP_MODE_ETA: 1472 #ifdef INVARIANTS 1473 switch (csp->csp_cipher_alg) { 1474 case CRYPTO_AES_NIST_GCM_16: 1475 case CRYPTO_AES_CCM_16: 1476 case CRYPTO_CHACHA20_POLY1305: 1477 case CRYPTO_XCHACHA20_POLY1305: 1478 panic("bad eta cipher algo"); 1479 } 1480 switch (csp->csp_auth_alg) { 1481 case CRYPTO_AES_NIST_GMAC: 1482 case CRYPTO_AES_CCM_CBC_MAC: 1483 panic("bad eta auth algo"); 1484 } 1485 #endif 1486 1487 error = swcr_setup_auth(ses, csp); 1488 if (error) 1489 break; 1490 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1491 /* Effectively degrade to digest mode. */ 1492 ses->swcr_process = swcr_authcompute; 1493 break; 1494 } 1495 1496 error = swcr_setup_cipher(ses, csp); 1497 if (error == 0) 1498 ses->swcr_process = swcr_eta; 1499 break; 1500 default: 1501 error = EINVAL; 1502 } 1503 1504 if (error) 1505 swcr_freesession(dev, cses); 1506 return (error); 1507 } 1508 1509 static void 1510 swcr_freesession(device_t dev, crypto_session_t cses) 1511 { 1512 struct swcr_session *ses; 1513 1514 ses = crypto_get_driver_session(cses); 1515 1516 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1517 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1518 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1519 } 1520 1521 /* 1522 * Process a software request. 1523 */ 1524 static int 1525 swcr_process(device_t dev, struct cryptop *crp, int hint) 1526 { 1527 struct swcr_session *ses; 1528 1529 ses = crypto_get_driver_session(crp->crp_session); 1530 1531 crp->crp_etype = ses->swcr_process(ses, crp); 1532 1533 crypto_done(crp); 1534 return (0); 1535 } 1536 1537 static void 1538 swcr_identify(driver_t *drv, device_t parent) 1539 { 1540 /* NB: order 10 is so we get attached after h/w devices */ 1541 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1542 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1543 panic("cryptosoft: could not attach"); 1544 } 1545 1546 static int 1547 swcr_probe(device_t dev) 1548 { 1549 device_set_desc(dev, "software crypto"); 1550 device_quiet(dev); 1551 return (BUS_PROBE_NOWILDCARD); 1552 } 1553 1554 static int 1555 swcr_attach(device_t dev) 1556 { 1557 1558 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1559 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1560 if (swcr_id < 0) { 1561 device_printf(dev, "cannot initialize!"); 1562 return (ENXIO); 1563 } 1564 1565 return (0); 1566 } 1567 1568 static int 1569 swcr_detach(device_t dev) 1570 { 1571 crypto_unregister_all(swcr_id); 1572 return 0; 1573 } 1574 1575 static device_method_t swcr_methods[] = { 1576 DEVMETHOD(device_identify, swcr_identify), 1577 DEVMETHOD(device_probe, swcr_probe), 1578 DEVMETHOD(device_attach, swcr_attach), 1579 DEVMETHOD(device_detach, swcr_detach), 1580 1581 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1582 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1583 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1584 DEVMETHOD(cryptodev_process, swcr_process), 1585 1586 {0, 0}, 1587 }; 1588 1589 static driver_t swcr_driver = { 1590 "cryptosoft", 1591 swcr_methods, 1592 0, /* NB: no softc */ 1593 }; 1594 static devclass_t swcr_devclass; 1595 1596 /* 1597 * NB: We explicitly reference the crypto module so we 1598 * get the necessary ordering when built as a loadable 1599 * module. This is required because we bundle the crypto 1600 * module code together with the cryptosoft driver (otherwise 1601 * normal module dependencies would handle things). 1602 */ 1603 extern int crypto_modevent(struct module *, int, void *); 1604 /* XXX where to attach */ 1605 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1606 MODULE_VERSION(cryptosoft, 1); 1607 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1608