1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen; 109 int blks, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blks = exf->blocksize; 128 } else 129 blks = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 resid = crp->crp_payload_length; 158 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 159 160 /* 161 * Loop through encrypting blocks. 'inlen' is the remaining 162 * length of the current segment in the input buffer. 163 * 'outlen' is the remaining length of current segment in the 164 * output buffer. 165 */ 166 while (resid >= blks) { 167 /* 168 * If the current block is not contained within the 169 * current input/output segment, use 'blk' as a local 170 * buffer. 171 */ 172 if (inlen < blks) { 173 crypto_cursor_copydata(&cc_in, blks, blk); 174 inblk = blk; 175 } 176 if (outlen < blks) 177 outblk = blk; 178 179 if (encrypting) 180 exf->encrypt(ctx, inblk, outblk); 181 else 182 exf->decrypt(ctx, inblk, outblk); 183 184 if (inlen < blks) { 185 inblk = crypto_cursor_segment(&cc_in, &inlen); 186 } else { 187 crypto_cursor_advance(&cc_in, blks); 188 inlen -= blks; 189 inblk += blks; 190 } 191 192 if (outlen < blks) { 193 crypto_cursor_copyback(&cc_out, blks, blk); 194 outblk = crypto_cursor_segment(&cc_out, &outlen); 195 } else { 196 crypto_cursor_advance(&cc_out, blks); 197 outlen -= blks; 198 outblk += blks; 199 } 200 201 resid -= blks; 202 } 203 204 /* Handle trailing partial block for stream ciphers. */ 205 if (resid > 0) { 206 KASSERT(exf->native_blocksize != 0, 207 ("%s: partial block of %d bytes for cipher %s", 208 __func__, resid, exf->name)); 209 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 210 211 inblk = crypto_cursor_segment(&cc_in, &inlen); 212 outblk = crypto_cursor_segment(&cc_out, &outlen); 213 if (inlen < resid) { 214 crypto_cursor_copydata(&cc_in, resid, blk); 215 inblk = blk; 216 } 217 if (outlen < resid) 218 outblk = blk; 219 if (encrypting) 220 exf->encrypt_last(ctx, inblk, outblk, 221 resid); 222 else 223 exf->decrypt_last(ctx, inblk, outblk, 224 resid); 225 if (outlen < resid) 226 crypto_cursor_copyback(&cc_out, resid, blk); 227 } 228 229 explicit_bzero(ctx, exf->ctxsize); 230 explicit_bzero(blk, sizeof(blk)); 231 return (0); 232 } 233 234 /* 235 * Compute or verify hash. 236 */ 237 static int 238 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 239 { 240 u_char aalg[HASH_MAX_LEN]; 241 const struct crypto_session_params *csp; 242 const struct swcr_auth *sw; 243 const struct auth_hash *axf; 244 union authctx ctx; 245 int err; 246 247 sw = &ses->swcr_auth; 248 249 axf = sw->sw_axf; 250 251 csp = crypto_get_params(crp->crp_session); 252 if (crp->crp_auth_key != NULL) { 253 if (sw->sw_hmac) { 254 hmac_init_ipad(axf, crp->crp_auth_key, 255 csp->csp_auth_klen, &ctx); 256 } else { 257 axf->Init(&ctx); 258 axf->Setkey(&ctx, crp->crp_auth_key, 259 csp->csp_auth_klen); 260 } 261 } else 262 memcpy(&ctx, sw->sw_ictx, axf->ctxsize); 263 264 if (crp->crp_aad != NULL) 265 err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 266 else 267 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 268 axf->Update, &ctx); 269 if (err) 270 goto out; 271 272 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 273 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 274 err = crypto_apply_buf(&crp->crp_obuf, 275 crp->crp_payload_output_start, crp->crp_payload_length, 276 axf->Update, &ctx); 277 else 278 err = crypto_apply(crp, crp->crp_payload_start, 279 crp->crp_payload_length, axf->Update, &ctx); 280 if (err) 281 goto out; 282 283 if (csp->csp_flags & CSP_F_ESN) 284 axf->Update(&ctx, crp->crp_esn, 4); 285 286 axf->Final(aalg, &ctx); 287 if (sw->sw_hmac) { 288 if (crp->crp_auth_key != NULL) 289 hmac_init_opad(axf, crp->crp_auth_key, 290 csp->csp_auth_klen, &ctx); 291 else 292 memcpy(&ctx, sw->sw_octx, axf->ctxsize); 293 axf->Update(&ctx, aalg, axf->hashsize); 294 axf->Final(aalg, &ctx); 295 } 296 297 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 298 u_char uaalg[HASH_MAX_LEN]; 299 300 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 301 if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 302 err = EBADMSG; 303 explicit_bzero(uaalg, sizeof(uaalg)); 304 } else { 305 /* Inject the authentication data */ 306 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 307 } 308 explicit_bzero(aalg, sizeof(aalg)); 309 out: 310 explicit_bzero(&ctx, sizeof(ctx)); 311 return (err); 312 } 313 314 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 315 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 316 317 static int 318 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 319 { 320 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 321 u_char *blk = (u_char *)blkbuf; 322 u_char tag[GMAC_DIGEST_LEN]; 323 struct crypto_buffer_cursor cc; 324 const u_char *inblk; 325 union authctx ctx; 326 const struct swcr_auth *swa; 327 const struct auth_hash *axf; 328 uint32_t *blkp; 329 size_t len; 330 int blksz, error, ivlen, resid; 331 332 swa = &ses->swcr_auth; 333 axf = swa->sw_axf; 334 blksz = GMAC_BLOCK_LEN; 335 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 336 __func__)); 337 338 if (crp->crp_auth_key != NULL) { 339 axf->Init(&ctx); 340 axf->Setkey(&ctx, crp->crp_auth_key, 341 crypto_get_params(crp->crp_session)->csp_auth_klen); 342 } else 343 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 344 345 /* Initialize the IV */ 346 ivlen = AES_GCM_IV_LEN; 347 crypto_read_iv(crp, blk); 348 349 axf->Reinit(&ctx, blk, ivlen); 350 crypto_cursor_init(&cc, &crp->crp_buf); 351 crypto_cursor_advance(&cc, crp->crp_payload_start); 352 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 353 inblk = crypto_cursor_segment(&cc, &len); 354 if (len >= blksz) { 355 len = rounddown(MIN(len, resid), blksz); 356 crypto_cursor_advance(&cc, len); 357 } else { 358 len = blksz; 359 crypto_cursor_copydata(&cc, len, blk); 360 inblk = blk; 361 } 362 axf->Update(&ctx, inblk, len); 363 } 364 if (resid > 0) { 365 memset(blk, 0, blksz); 366 crypto_cursor_copydata(&cc, resid, blk); 367 axf->Update(&ctx, blk, blksz); 368 } 369 370 /* length block */ 371 memset(blk, 0, blksz); 372 blkp = (uint32_t *)blk + 1; 373 *blkp = htobe32(crp->crp_payload_length * 8); 374 axf->Update(&ctx, blk, blksz); 375 376 /* Finalize MAC */ 377 axf->Final(tag, &ctx); 378 379 error = 0; 380 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 381 u_char tag2[GMAC_DIGEST_LEN]; 382 383 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 384 tag2); 385 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 386 error = EBADMSG; 387 explicit_bzero(tag2, sizeof(tag2)); 388 } else { 389 /* Inject the authentication data */ 390 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 391 } 392 explicit_bzero(blkbuf, sizeof(blkbuf)); 393 explicit_bzero(tag, sizeof(tag)); 394 return (error); 395 } 396 397 static int 398 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 399 { 400 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 401 u_char *blk = (u_char *)blkbuf; 402 u_char tag[GMAC_DIGEST_LEN]; 403 struct crypto_buffer_cursor cc_in, cc_out; 404 const u_char *inblk; 405 u_char *outblk; 406 const struct swcr_auth *swa; 407 const struct swcr_encdec *swe; 408 const struct enc_xform *exf; 409 void *ctx; 410 uint32_t *blkp; 411 size_t len; 412 int blksz, error, ivlen, r, resid; 413 414 swa = &ses->swcr_auth; 415 swe = &ses->swcr_encdec; 416 exf = swe->sw_exf; 417 blksz = GMAC_BLOCK_LEN; 418 KASSERT(blksz == exf->native_blocksize, 419 ("%s: blocksize mismatch", __func__)); 420 421 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 422 return (EINVAL); 423 424 ivlen = AES_GCM_IV_LEN; 425 426 ctx = __builtin_alloca(exf->ctxsize); 427 if (crp->crp_cipher_key != NULL) 428 exf->setkey(ctx, crp->crp_cipher_key, 429 crypto_get_params(crp->crp_session)->csp_cipher_klen); 430 else 431 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 432 exf->reinit(ctx, crp->crp_iv, ivlen); 433 434 /* Supply MAC with AAD */ 435 if (crp->crp_aad != NULL) { 436 len = rounddown(crp->crp_aad_length, blksz); 437 if (len != 0) 438 exf->update(ctx, crp->crp_aad, len); 439 if (crp->crp_aad_length != len) { 440 memset(blk, 0, blksz); 441 memcpy(blk, (char *)crp->crp_aad + len, 442 crp->crp_aad_length - len); 443 exf->update(ctx, blk, blksz); 444 } 445 } else { 446 crypto_cursor_init(&cc_in, &crp->crp_buf); 447 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 448 for (resid = crp->crp_aad_length; resid >= blksz; 449 resid -= len) { 450 inblk = crypto_cursor_segment(&cc_in, &len); 451 if (len >= blksz) { 452 len = rounddown(MIN(len, resid), blksz); 453 crypto_cursor_advance(&cc_in, len); 454 } else { 455 len = blksz; 456 crypto_cursor_copydata(&cc_in, len, blk); 457 inblk = blk; 458 } 459 exf->update(ctx, inblk, len); 460 } 461 if (resid > 0) { 462 memset(blk, 0, blksz); 463 crypto_cursor_copydata(&cc_in, resid, blk); 464 exf->update(ctx, blk, blksz); 465 } 466 } 467 468 /* Do encryption with MAC */ 469 crypto_cursor_init(&cc_in, &crp->crp_buf); 470 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 471 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 472 crypto_cursor_init(&cc_out, &crp->crp_obuf); 473 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 474 } else 475 cc_out = cc_in; 476 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 477 inblk = crypto_cursor_segment(&cc_in, &len); 478 if (len < blksz) { 479 crypto_cursor_copydata(&cc_in, blksz, blk); 480 inblk = blk; 481 } else { 482 crypto_cursor_advance(&cc_in, blksz); 483 } 484 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 485 outblk = crypto_cursor_segment(&cc_out, &len); 486 if (len < blksz) 487 outblk = blk; 488 exf->encrypt(ctx, inblk, outblk); 489 exf->update(ctx, outblk, blksz); 490 if (outblk == blk) 491 crypto_cursor_copyback(&cc_out, blksz, blk); 492 else 493 crypto_cursor_advance(&cc_out, blksz); 494 } else { 495 exf->update(ctx, inblk, blksz); 496 } 497 } 498 if (resid > 0) { 499 crypto_cursor_copydata(&cc_in, resid, blk); 500 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 501 exf->encrypt_last(ctx, blk, blk, resid); 502 crypto_cursor_copyback(&cc_out, resid, blk); 503 } 504 exf->update(ctx, blk, resid); 505 } 506 507 /* length block */ 508 memset(blk, 0, blksz); 509 blkp = (uint32_t *)blk + 1; 510 *blkp = htobe32(crp->crp_aad_length * 8); 511 blkp = (uint32_t *)blk + 3; 512 *blkp = htobe32(crp->crp_payload_length * 8); 513 exf->update(ctx, blk, blksz); 514 515 /* Finalize MAC */ 516 exf->final(tag, ctx); 517 518 /* Validate tag */ 519 error = 0; 520 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 521 u_char tag2[GMAC_DIGEST_LEN]; 522 523 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 524 525 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 526 explicit_bzero(tag2, sizeof(tag2)); 527 if (r != 0) { 528 error = EBADMSG; 529 goto out; 530 } 531 532 /* tag matches, decrypt data */ 533 crypto_cursor_init(&cc_in, &crp->crp_buf); 534 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 535 for (resid = crp->crp_payload_length; resid > blksz; 536 resid -= blksz) { 537 inblk = crypto_cursor_segment(&cc_in, &len); 538 if (len < blksz) { 539 crypto_cursor_copydata(&cc_in, blksz, blk); 540 inblk = blk; 541 } else 542 crypto_cursor_advance(&cc_in, blksz); 543 outblk = crypto_cursor_segment(&cc_out, &len); 544 if (len < blksz) 545 outblk = blk; 546 exf->decrypt(ctx, inblk, outblk); 547 if (outblk == blk) 548 crypto_cursor_copyback(&cc_out, blksz, blk); 549 else 550 crypto_cursor_advance(&cc_out, blksz); 551 } 552 if (resid > 0) { 553 crypto_cursor_copydata(&cc_in, resid, blk); 554 exf->decrypt_last(ctx, blk, blk, resid); 555 crypto_cursor_copyback(&cc_out, resid, blk); 556 } 557 } else { 558 /* Inject the authentication data */ 559 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 560 } 561 562 out: 563 explicit_bzero(ctx, exf->ctxsize); 564 explicit_bzero(blkbuf, sizeof(blkbuf)); 565 explicit_bzero(tag, sizeof(tag)); 566 567 return (error); 568 } 569 570 static void 571 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 572 u_int data_length, u_int tag_length, uint8_t *b0) 573 { 574 uint8_t *bp; 575 uint8_t flags, L; 576 577 KASSERT(nonce_length >= 7 && nonce_length <= 13, 578 ("nonce_length must be between 7 and 13 bytes")); 579 580 /* 581 * Need to determine the L field value. This is the number of 582 * bytes needed to specify the length of the message; the length 583 * is whatever is left in the 16 bytes after specifying flags and 584 * the nonce. 585 */ 586 L = 15 - nonce_length; 587 588 flags = ((aad_length > 0) << 6) + 589 (((tag_length - 2) / 2) << 3) + 590 L - 1; 591 592 /* 593 * Now we need to set up the first block, which has flags, nonce, 594 * and the message length. 595 */ 596 b0[0] = flags; 597 memcpy(b0 + 1, nonce, nonce_length); 598 bp = b0 + 1 + nonce_length; 599 600 /* Need to copy L' [aka L-1] bytes of data_length */ 601 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 602 *dst = data_length; 603 data_length >>= 8; 604 } 605 } 606 607 /* NB: OCF only supports AAD lengths < 2^32. */ 608 static int 609 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 610 { 611 if (aad_length < ((1 << 16) - (1 << 8))) { 612 be16enc(blk, aad_length); 613 return (sizeof(uint16_t)); 614 } else { 615 blk[0] = 0xff; 616 blk[1] = 0xfe; 617 be32enc(blk + 2, aad_length); 618 return (2 + sizeof(uint32_t)); 619 } 620 } 621 622 static int 623 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 624 { 625 u_char blk[CCM_CBC_BLOCK_LEN]; 626 u_char tag[AES_CBC_MAC_HASH_LEN]; 627 union authctx ctx; 628 const struct crypto_session_params *csp; 629 const struct swcr_auth *swa; 630 const struct auth_hash *axf; 631 int error, ivlen, len; 632 633 csp = crypto_get_params(crp->crp_session); 634 swa = &ses->swcr_auth; 635 axf = swa->sw_axf; 636 637 if (crp->crp_auth_key != NULL) { 638 axf->Init(&ctx); 639 axf->Setkey(&ctx, crp->crp_auth_key, csp->csp_auth_klen); 640 } else 641 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 642 643 /* Initialize the IV */ 644 ivlen = csp->csp_ivlen; 645 646 /* Supply MAC with IV */ 647 axf->Reinit(&ctx, crp->crp_iv, ivlen); 648 649 /* Supply MAC with b0. */ 650 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 651 swa->sw_mlen, blk); 652 axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 653 654 len = build_ccm_aad_length(crp->crp_payload_length, blk); 655 axf->Update(&ctx, blk, len); 656 657 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 658 axf->Update, &ctx); 659 660 /* Finalize MAC */ 661 axf->Final(tag, &ctx); 662 663 error = 0; 664 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 665 u_char tag2[AES_CBC_MAC_HASH_LEN]; 666 667 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 668 tag2); 669 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 670 error = EBADMSG; 671 explicit_bzero(tag2, sizeof(tag)); 672 } else { 673 /* Inject the authentication data */ 674 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 675 } 676 explicit_bzero(tag, sizeof(tag)); 677 explicit_bzero(blk, sizeof(blk)); 678 return (error); 679 } 680 681 static int 682 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 683 { 684 const struct crypto_session_params *csp; 685 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 686 u_char *blk = (u_char *)blkbuf; 687 u_char tag[AES_CBC_MAC_HASH_LEN]; 688 struct crypto_buffer_cursor cc_in, cc_out; 689 const u_char *inblk; 690 u_char *outblk; 691 const struct swcr_auth *swa; 692 const struct swcr_encdec *swe; 693 const struct enc_xform *exf; 694 void *ctx; 695 size_t len; 696 int blksz, error, ivlen, r, resid; 697 698 csp = crypto_get_params(crp->crp_session); 699 swa = &ses->swcr_auth; 700 swe = &ses->swcr_encdec; 701 exf = swe->sw_exf; 702 blksz = AES_BLOCK_LEN; 703 KASSERT(blksz == exf->native_blocksize, 704 ("%s: blocksize mismatch", __func__)); 705 706 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 707 return (EMSGSIZE); 708 709 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 710 return (EINVAL); 711 712 ivlen = csp->csp_ivlen; 713 714 ctx = __builtin_alloca(exf->ctxsize); 715 if (crp->crp_cipher_key != NULL) 716 exf->setkey(ctx, crp->crp_cipher_key, 717 crypto_get_params(crp->crp_session)->csp_cipher_klen); 718 else 719 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 720 exf->reinit(ctx, crp->crp_iv, ivlen); 721 722 /* Supply MAC with b0. */ 723 _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 724 "blkbuf too small for b0"); 725 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 726 crp->crp_payload_length, swa->sw_mlen, blk); 727 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 728 729 /* Supply MAC with AAD */ 730 if (crp->crp_aad_length != 0) { 731 len = build_ccm_aad_length(crp->crp_aad_length, blk); 732 exf->update(ctx, blk, len); 733 if (crp->crp_aad != NULL) 734 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 735 else 736 crypto_apply(crp, crp->crp_aad_start, 737 crp->crp_aad_length, exf->update, ctx); 738 739 /* Pad the AAD (including length field) to a full block. */ 740 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 741 if (len != 0) { 742 len = CCM_CBC_BLOCK_LEN - len; 743 memset(blk, 0, CCM_CBC_BLOCK_LEN); 744 exf->update(ctx, blk, len); 745 } 746 } 747 748 /* Do encryption/decryption with MAC */ 749 crypto_cursor_init(&cc_in, &crp->crp_buf); 750 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 751 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 752 crypto_cursor_init(&cc_out, &crp->crp_obuf); 753 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 754 } else 755 cc_out = cc_in; 756 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 757 inblk = crypto_cursor_segment(&cc_in, &len); 758 if (len < blksz) { 759 crypto_cursor_copydata(&cc_in, blksz, blk); 760 inblk = blk; 761 } else 762 crypto_cursor_advance(&cc_in, blksz); 763 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 764 outblk = crypto_cursor_segment(&cc_out, &len); 765 if (len < blksz) 766 outblk = blk; 767 exf->update(ctx, inblk, blksz); 768 exf->encrypt(ctx, inblk, outblk); 769 if (outblk == blk) 770 crypto_cursor_copyback(&cc_out, blksz, blk); 771 else 772 crypto_cursor_advance(&cc_out, blksz); 773 } else { 774 /* 775 * One of the problems with CCM+CBC is that 776 * the authentication is done on the 777 * unencrypted data. As a result, we have to 778 * decrypt the data twice: once to generate 779 * the tag and a second time after the tag is 780 * verified. 781 */ 782 exf->decrypt(ctx, inblk, blk); 783 exf->update(ctx, blk, blksz); 784 } 785 } 786 if (resid > 0) { 787 crypto_cursor_copydata(&cc_in, resid, blk); 788 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 789 exf->update(ctx, blk, resid); 790 exf->encrypt_last(ctx, blk, blk, resid); 791 crypto_cursor_copyback(&cc_out, resid, blk); 792 } else { 793 exf->decrypt_last(ctx, blk, blk, resid); 794 exf->update(ctx, blk, resid); 795 } 796 } 797 798 /* Finalize MAC */ 799 exf->final(tag, ctx); 800 801 /* Validate tag */ 802 error = 0; 803 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 804 u_char tag2[AES_CBC_MAC_HASH_LEN]; 805 806 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 807 tag2); 808 809 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 810 explicit_bzero(tag2, sizeof(tag2)); 811 if (r != 0) { 812 error = EBADMSG; 813 goto out; 814 } 815 816 /* tag matches, decrypt data */ 817 exf->reinit(ctx, crp->crp_iv, ivlen); 818 crypto_cursor_init(&cc_in, &crp->crp_buf); 819 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 820 for (resid = crp->crp_payload_length; resid > blksz; 821 resid -= blksz) { 822 inblk = crypto_cursor_segment(&cc_in, &len); 823 if (len < blksz) { 824 crypto_cursor_copydata(&cc_in, blksz, blk); 825 inblk = blk; 826 } else 827 crypto_cursor_advance(&cc_in, blksz); 828 outblk = crypto_cursor_segment(&cc_out, &len); 829 if (len < blksz) 830 outblk = blk; 831 exf->decrypt(ctx, inblk, outblk); 832 if (outblk == blk) 833 crypto_cursor_copyback(&cc_out, blksz, blk); 834 else 835 crypto_cursor_advance(&cc_out, blksz); 836 } 837 if (resid > 0) { 838 crypto_cursor_copydata(&cc_in, resid, blk); 839 exf->decrypt_last(ctx, blk, blk, resid); 840 crypto_cursor_copyback(&cc_out, resid, blk); 841 } 842 } else { 843 /* Inject the authentication data */ 844 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 845 } 846 847 out: 848 explicit_bzero(ctx, exf->ctxsize); 849 explicit_bzero(blkbuf, sizeof(blkbuf)); 850 explicit_bzero(tag, sizeof(tag)); 851 return (error); 852 } 853 854 static int 855 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 856 { 857 const struct crypto_session_params *csp; 858 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 859 u_char *blk = (u_char *)blkbuf; 860 u_char tag[POLY1305_HASH_LEN]; 861 struct crypto_buffer_cursor cc_in, cc_out; 862 const u_char *inblk; 863 u_char *outblk; 864 uint64_t *blkp; 865 const struct swcr_auth *swa; 866 const struct swcr_encdec *swe; 867 const struct enc_xform *exf; 868 void *ctx; 869 size_t len; 870 int blksz, error, r, resid; 871 872 swa = &ses->swcr_auth; 873 swe = &ses->swcr_encdec; 874 exf = swe->sw_exf; 875 blksz = exf->native_blocksize; 876 KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 877 878 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 879 return (EINVAL); 880 881 csp = crypto_get_params(crp->crp_session); 882 883 ctx = __builtin_alloca(exf->ctxsize); 884 if (crp->crp_cipher_key != NULL) 885 exf->setkey(ctx, crp->crp_cipher_key, 886 csp->csp_cipher_klen); 887 else 888 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 889 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 890 891 /* Supply MAC with AAD */ 892 if (crp->crp_aad != NULL) 893 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 894 else 895 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 896 exf->update, ctx); 897 if (crp->crp_aad_length % 16 != 0) { 898 /* padding1 */ 899 memset(blk, 0, 16); 900 exf->update(ctx, blk, 16 - crp->crp_aad_length % 16); 901 } 902 903 /* Do encryption with MAC */ 904 crypto_cursor_init(&cc_in, &crp->crp_buf); 905 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 906 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 907 crypto_cursor_init(&cc_out, &crp->crp_obuf); 908 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 909 } else 910 cc_out = cc_in; 911 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 912 inblk = crypto_cursor_segment(&cc_in, &len); 913 if (len < blksz) { 914 crypto_cursor_copydata(&cc_in, blksz, blk); 915 inblk = blk; 916 } else 917 crypto_cursor_advance(&cc_in, blksz); 918 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 919 outblk = crypto_cursor_segment(&cc_out, &len); 920 if (len < blksz) 921 outblk = blk; 922 exf->encrypt(ctx, inblk, outblk); 923 exf->update(ctx, outblk, blksz); 924 if (outblk == blk) 925 crypto_cursor_copyback(&cc_out, blksz, blk); 926 else 927 crypto_cursor_advance(&cc_out, blksz); 928 } else { 929 exf->update(ctx, inblk, blksz); 930 } 931 } 932 if (resid > 0) { 933 crypto_cursor_copydata(&cc_in, resid, blk); 934 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 935 exf->encrypt_last(ctx, blk, blk, resid); 936 crypto_cursor_copyback(&cc_out, resid, blk); 937 } 938 exf->update(ctx, blk, resid); 939 if (resid % 16 != 0) { 940 /* padding2 */ 941 memset(blk, 0, 16); 942 exf->update(ctx, blk, 16 - resid % 16); 943 } 944 } 945 946 /* lengths */ 947 blkp = (uint64_t *)blk; 948 blkp[0] = htole64(crp->crp_aad_length); 949 blkp[1] = htole64(crp->crp_payload_length); 950 exf->update(ctx, blk, sizeof(uint64_t) * 2); 951 952 /* Finalize MAC */ 953 exf->final(tag, ctx); 954 955 /* Validate tag */ 956 error = 0; 957 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 958 u_char tag2[POLY1305_HASH_LEN]; 959 960 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 961 962 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 963 explicit_bzero(tag2, sizeof(tag2)); 964 if (r != 0) { 965 error = EBADMSG; 966 goto out; 967 } 968 969 /* tag matches, decrypt data */ 970 crypto_cursor_init(&cc_in, &crp->crp_buf); 971 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 972 for (resid = crp->crp_payload_length; resid > blksz; 973 resid -= blksz) { 974 inblk = crypto_cursor_segment(&cc_in, &len); 975 if (len < blksz) { 976 crypto_cursor_copydata(&cc_in, blksz, blk); 977 inblk = blk; 978 } else 979 crypto_cursor_advance(&cc_in, blksz); 980 outblk = crypto_cursor_segment(&cc_out, &len); 981 if (len < blksz) 982 outblk = blk; 983 exf->decrypt(ctx, inblk, outblk); 984 if (outblk == blk) 985 crypto_cursor_copyback(&cc_out, blksz, blk); 986 else 987 crypto_cursor_advance(&cc_out, blksz); 988 } 989 if (resid > 0) { 990 crypto_cursor_copydata(&cc_in, resid, blk); 991 exf->decrypt_last(ctx, blk, blk, resid); 992 crypto_cursor_copyback(&cc_out, resid, blk); 993 } 994 } else { 995 /* Inject the authentication data */ 996 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 997 } 998 999 out: 1000 explicit_bzero(ctx, exf->ctxsize); 1001 explicit_bzero(blkbuf, sizeof(blkbuf)); 1002 explicit_bzero(tag, sizeof(tag)); 1003 return (error); 1004 } 1005 1006 /* 1007 * Apply a cipher and a digest to perform EtA. 1008 */ 1009 static int 1010 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1011 { 1012 int error; 1013 1014 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1015 error = swcr_encdec(ses, crp); 1016 if (error == 0) 1017 error = swcr_authcompute(ses, crp); 1018 } else { 1019 error = swcr_authcompute(ses, crp); 1020 if (error == 0) 1021 error = swcr_encdec(ses, crp); 1022 } 1023 return (error); 1024 } 1025 1026 /* 1027 * Apply a compression/decompression algorithm 1028 */ 1029 static int 1030 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1031 { 1032 const struct comp_algo *cxf; 1033 uint8_t *data, *out; 1034 int adj; 1035 uint32_t result; 1036 1037 cxf = ses->swcr_compdec.sw_cxf; 1038 1039 /* We must handle the whole buffer of data in one time 1040 * then if there is not all the data in the mbuf, we must 1041 * copy in a buffer. 1042 */ 1043 1044 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1045 if (data == NULL) 1046 return (EINVAL); 1047 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1048 data); 1049 1050 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1051 result = cxf->compress(data, crp->crp_payload_length, &out); 1052 else 1053 result = cxf->decompress(data, crp->crp_payload_length, &out); 1054 1055 free(data, M_CRYPTO_DATA); 1056 if (result == 0) 1057 return (EINVAL); 1058 crp->crp_olen = result; 1059 1060 /* Check the compressed size when doing compression */ 1061 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1062 if (result >= crp->crp_payload_length) { 1063 /* Compression was useless, we lost time */ 1064 free(out, M_CRYPTO_DATA); 1065 return (0); 1066 } 1067 } 1068 1069 /* Copy back the (de)compressed data. m_copyback is 1070 * extending the mbuf as necessary. 1071 */ 1072 crypto_copyback(crp, crp->crp_payload_start, result, out); 1073 if (result < crp->crp_payload_length) { 1074 switch (crp->crp_buf.cb_type) { 1075 case CRYPTO_BUF_MBUF: 1076 case CRYPTO_BUF_SINGLE_MBUF: 1077 adj = result - crp->crp_payload_length; 1078 m_adj(crp->crp_buf.cb_mbuf, adj); 1079 break; 1080 case CRYPTO_BUF_UIO: { 1081 struct uio *uio = crp->crp_buf.cb_uio; 1082 int ind; 1083 1084 adj = crp->crp_payload_length - result; 1085 ind = uio->uio_iovcnt - 1; 1086 1087 while (adj > 0 && ind >= 0) { 1088 if (adj < uio->uio_iov[ind].iov_len) { 1089 uio->uio_iov[ind].iov_len -= adj; 1090 break; 1091 } 1092 1093 adj -= uio->uio_iov[ind].iov_len; 1094 uio->uio_iov[ind].iov_len = 0; 1095 ind--; 1096 uio->uio_iovcnt--; 1097 } 1098 } 1099 break; 1100 case CRYPTO_BUF_VMPAGE: 1101 adj = crp->crp_payload_length - result; 1102 crp->crp_buf.cb_vm_page_len -= adj; 1103 break; 1104 default: 1105 break; 1106 } 1107 } 1108 free(out, M_CRYPTO_DATA); 1109 return 0; 1110 } 1111 1112 static int 1113 swcr_setup_cipher(struct swcr_session *ses, 1114 const struct crypto_session_params *csp) 1115 { 1116 struct swcr_encdec *swe; 1117 const struct enc_xform *txf; 1118 int error; 1119 1120 swe = &ses->swcr_encdec; 1121 txf = crypto_cipher(csp); 1122 if (csp->csp_cipher_key != NULL) { 1123 if (txf->ctxsize != 0) { 1124 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1125 M_NOWAIT); 1126 if (swe->sw_ctx == NULL) 1127 return (ENOMEM); 1128 } 1129 error = txf->setkey(swe->sw_ctx, 1130 csp->csp_cipher_key, csp->csp_cipher_klen); 1131 if (error) 1132 return (error); 1133 } 1134 swe->sw_exf = txf; 1135 return (0); 1136 } 1137 1138 static int 1139 swcr_setup_auth(struct swcr_session *ses, 1140 const struct crypto_session_params *csp) 1141 { 1142 struct swcr_auth *swa; 1143 const struct auth_hash *axf; 1144 1145 swa = &ses->swcr_auth; 1146 1147 axf = crypto_auth_hash(csp); 1148 swa->sw_axf = axf; 1149 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1150 return (EINVAL); 1151 if (csp->csp_auth_mlen == 0) 1152 swa->sw_mlen = axf->hashsize; 1153 else 1154 swa->sw_mlen = csp->csp_auth_mlen; 1155 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1156 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1157 M_NOWAIT); 1158 if (swa->sw_ictx == NULL) 1159 return (ENOBUFS); 1160 } 1161 1162 switch (csp->csp_auth_alg) { 1163 case CRYPTO_SHA1_HMAC: 1164 case CRYPTO_SHA2_224_HMAC: 1165 case CRYPTO_SHA2_256_HMAC: 1166 case CRYPTO_SHA2_384_HMAC: 1167 case CRYPTO_SHA2_512_HMAC: 1168 case CRYPTO_RIPEMD160_HMAC: 1169 swa->sw_hmac = true; 1170 if (csp->csp_auth_key != NULL) { 1171 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1172 M_NOWAIT); 1173 if (swa->sw_octx == NULL) 1174 return (ENOBUFS); 1175 hmac_init_ipad(axf, csp->csp_auth_key, 1176 csp->csp_auth_klen, swa->sw_ictx); 1177 hmac_init_opad(axf, csp->csp_auth_key, 1178 csp->csp_auth_klen, swa->sw_octx); 1179 } 1180 break; 1181 case CRYPTO_SHA1: 1182 case CRYPTO_SHA2_224: 1183 case CRYPTO_SHA2_256: 1184 case CRYPTO_SHA2_384: 1185 case CRYPTO_SHA2_512: 1186 case CRYPTO_NULL_HMAC: 1187 axf->Init(swa->sw_ictx); 1188 break; 1189 case CRYPTO_AES_NIST_GMAC: 1190 case CRYPTO_AES_CCM_CBC_MAC: 1191 case CRYPTO_POLY1305: 1192 if (csp->csp_auth_key != NULL) { 1193 axf->Init(swa->sw_ictx); 1194 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1195 csp->csp_auth_klen); 1196 } 1197 break; 1198 case CRYPTO_BLAKE2B: 1199 case CRYPTO_BLAKE2S: 1200 /* 1201 * Blake2b and Blake2s support an optional key but do 1202 * not require one. 1203 */ 1204 if (csp->csp_auth_klen == 0) 1205 axf->Init(swa->sw_ictx); 1206 else if (csp->csp_auth_key != NULL) 1207 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1208 csp->csp_auth_klen); 1209 break; 1210 } 1211 1212 if (csp->csp_mode == CSP_MODE_DIGEST) { 1213 switch (csp->csp_auth_alg) { 1214 case CRYPTO_AES_NIST_GMAC: 1215 ses->swcr_process = swcr_gmac; 1216 break; 1217 case CRYPTO_AES_CCM_CBC_MAC: 1218 ses->swcr_process = swcr_ccm_cbc_mac; 1219 break; 1220 default: 1221 ses->swcr_process = swcr_authcompute; 1222 } 1223 } 1224 1225 return (0); 1226 } 1227 1228 static int 1229 swcr_setup_aead(struct swcr_session *ses, 1230 const struct crypto_session_params *csp) 1231 { 1232 struct swcr_auth *swa; 1233 int error; 1234 1235 error = swcr_setup_cipher(ses, csp); 1236 if (error) 1237 return (error); 1238 1239 swa = &ses->swcr_auth; 1240 if (csp->csp_auth_mlen == 0) 1241 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1242 else 1243 swa->sw_mlen = csp->csp_auth_mlen; 1244 return (0); 1245 } 1246 1247 static bool 1248 swcr_auth_supported(const struct crypto_session_params *csp) 1249 { 1250 const struct auth_hash *axf; 1251 1252 axf = crypto_auth_hash(csp); 1253 if (axf == NULL) 1254 return (false); 1255 switch (csp->csp_auth_alg) { 1256 case CRYPTO_SHA1_HMAC: 1257 case CRYPTO_SHA2_224_HMAC: 1258 case CRYPTO_SHA2_256_HMAC: 1259 case CRYPTO_SHA2_384_HMAC: 1260 case CRYPTO_SHA2_512_HMAC: 1261 case CRYPTO_NULL_HMAC: 1262 case CRYPTO_RIPEMD160_HMAC: 1263 break; 1264 case CRYPTO_AES_NIST_GMAC: 1265 switch (csp->csp_auth_klen * 8) { 1266 case 128: 1267 case 192: 1268 case 256: 1269 break; 1270 default: 1271 return (false); 1272 } 1273 if (csp->csp_auth_key == NULL) 1274 return (false); 1275 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1276 return (false); 1277 break; 1278 case CRYPTO_POLY1305: 1279 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1280 return (false); 1281 break; 1282 case CRYPTO_AES_CCM_CBC_MAC: 1283 switch (csp->csp_auth_klen * 8) { 1284 case 128: 1285 case 192: 1286 case 256: 1287 break; 1288 default: 1289 return (false); 1290 } 1291 if (csp->csp_auth_key == NULL) 1292 return (false); 1293 break; 1294 } 1295 return (true); 1296 } 1297 1298 static bool 1299 swcr_cipher_supported(const struct crypto_session_params *csp) 1300 { 1301 const struct enc_xform *txf; 1302 1303 txf = crypto_cipher(csp); 1304 if (txf == NULL) 1305 return (false); 1306 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1307 txf->ivsize != csp->csp_ivlen) 1308 return (false); 1309 return (true); 1310 } 1311 1312 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1313 1314 static int 1315 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1316 { 1317 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1318 return (EINVAL); 1319 switch (csp->csp_mode) { 1320 case CSP_MODE_COMPRESS: 1321 switch (csp->csp_cipher_alg) { 1322 case CRYPTO_DEFLATE_COMP: 1323 break; 1324 default: 1325 return (EINVAL); 1326 } 1327 break; 1328 case CSP_MODE_CIPHER: 1329 switch (csp->csp_cipher_alg) { 1330 case CRYPTO_AES_NIST_GCM_16: 1331 case CRYPTO_AES_CCM_16: 1332 case CRYPTO_CHACHA20_POLY1305: 1333 return (EINVAL); 1334 default: 1335 if (!swcr_cipher_supported(csp)) 1336 return (EINVAL); 1337 break; 1338 } 1339 break; 1340 case CSP_MODE_DIGEST: 1341 if (!swcr_auth_supported(csp)) 1342 return (EINVAL); 1343 break; 1344 case CSP_MODE_AEAD: 1345 switch (csp->csp_cipher_alg) { 1346 case CRYPTO_AES_NIST_GCM_16: 1347 case CRYPTO_AES_CCM_16: 1348 switch (csp->csp_cipher_klen * 8) { 1349 case 128: 1350 case 192: 1351 case 256: 1352 break; 1353 default: 1354 return (EINVAL); 1355 } 1356 break; 1357 case CRYPTO_CHACHA20_POLY1305: 1358 break; 1359 default: 1360 return (EINVAL); 1361 } 1362 break; 1363 case CSP_MODE_ETA: 1364 /* AEAD algorithms cannot be used for EtA. */ 1365 switch (csp->csp_cipher_alg) { 1366 case CRYPTO_AES_NIST_GCM_16: 1367 case CRYPTO_AES_CCM_16: 1368 case CRYPTO_CHACHA20_POLY1305: 1369 return (EINVAL); 1370 } 1371 switch (csp->csp_auth_alg) { 1372 case CRYPTO_AES_NIST_GMAC: 1373 case CRYPTO_AES_CCM_CBC_MAC: 1374 return (EINVAL); 1375 } 1376 1377 if (!swcr_cipher_supported(csp) || 1378 !swcr_auth_supported(csp)) 1379 return (EINVAL); 1380 break; 1381 default: 1382 return (EINVAL); 1383 } 1384 1385 return (CRYPTODEV_PROBE_SOFTWARE); 1386 } 1387 1388 /* 1389 * Generate a new software session. 1390 */ 1391 static int 1392 swcr_newsession(device_t dev, crypto_session_t cses, 1393 const struct crypto_session_params *csp) 1394 { 1395 struct swcr_session *ses; 1396 const struct comp_algo *cxf; 1397 int error; 1398 1399 ses = crypto_get_driver_session(cses); 1400 1401 error = 0; 1402 switch (csp->csp_mode) { 1403 case CSP_MODE_COMPRESS: 1404 switch (csp->csp_cipher_alg) { 1405 case CRYPTO_DEFLATE_COMP: 1406 cxf = &comp_algo_deflate; 1407 break; 1408 #ifdef INVARIANTS 1409 default: 1410 panic("bad compression algo"); 1411 #endif 1412 } 1413 ses->swcr_compdec.sw_cxf = cxf; 1414 ses->swcr_process = swcr_compdec; 1415 break; 1416 case CSP_MODE_CIPHER: 1417 switch (csp->csp_cipher_alg) { 1418 case CRYPTO_NULL_CBC: 1419 ses->swcr_process = swcr_null; 1420 break; 1421 #ifdef INVARIANTS 1422 case CRYPTO_AES_NIST_GCM_16: 1423 case CRYPTO_AES_CCM_16: 1424 case CRYPTO_CHACHA20_POLY1305: 1425 panic("bad cipher algo"); 1426 #endif 1427 default: 1428 error = swcr_setup_cipher(ses, csp); 1429 if (error == 0) 1430 ses->swcr_process = swcr_encdec; 1431 } 1432 break; 1433 case CSP_MODE_DIGEST: 1434 error = swcr_setup_auth(ses, csp); 1435 break; 1436 case CSP_MODE_AEAD: 1437 switch (csp->csp_cipher_alg) { 1438 case CRYPTO_AES_NIST_GCM_16: 1439 error = swcr_setup_aead(ses, csp); 1440 if (error == 0) 1441 ses->swcr_process = swcr_gcm; 1442 break; 1443 case CRYPTO_AES_CCM_16: 1444 error = swcr_setup_aead(ses, csp); 1445 if (error == 0) 1446 ses->swcr_process = swcr_ccm; 1447 break; 1448 case CRYPTO_CHACHA20_POLY1305: 1449 error = swcr_setup_aead(ses, csp); 1450 if (error == 0) 1451 ses->swcr_process = swcr_chacha20_poly1305; 1452 break; 1453 #ifdef INVARIANTS 1454 default: 1455 panic("bad aead algo"); 1456 #endif 1457 } 1458 break; 1459 case CSP_MODE_ETA: 1460 #ifdef INVARIANTS 1461 switch (csp->csp_cipher_alg) { 1462 case CRYPTO_AES_NIST_GCM_16: 1463 case CRYPTO_AES_CCM_16: 1464 case CRYPTO_CHACHA20_POLY1305: 1465 panic("bad eta cipher algo"); 1466 } 1467 switch (csp->csp_auth_alg) { 1468 case CRYPTO_AES_NIST_GMAC: 1469 case CRYPTO_AES_CCM_CBC_MAC: 1470 panic("bad eta auth algo"); 1471 } 1472 #endif 1473 1474 error = swcr_setup_auth(ses, csp); 1475 if (error) 1476 break; 1477 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1478 /* Effectively degrade to digest mode. */ 1479 ses->swcr_process = swcr_authcompute; 1480 break; 1481 } 1482 1483 error = swcr_setup_cipher(ses, csp); 1484 if (error == 0) 1485 ses->swcr_process = swcr_eta; 1486 break; 1487 default: 1488 error = EINVAL; 1489 } 1490 1491 if (error) 1492 swcr_freesession(dev, cses); 1493 return (error); 1494 } 1495 1496 static void 1497 swcr_freesession(device_t dev, crypto_session_t cses) 1498 { 1499 struct swcr_session *ses; 1500 1501 ses = crypto_get_driver_session(cses); 1502 1503 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1504 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1505 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1506 } 1507 1508 /* 1509 * Process a software request. 1510 */ 1511 static int 1512 swcr_process(device_t dev, struct cryptop *crp, int hint) 1513 { 1514 struct swcr_session *ses; 1515 1516 ses = crypto_get_driver_session(crp->crp_session); 1517 1518 crp->crp_etype = ses->swcr_process(ses, crp); 1519 1520 crypto_done(crp); 1521 return (0); 1522 } 1523 1524 static void 1525 swcr_identify(driver_t *drv, device_t parent) 1526 { 1527 /* NB: order 10 is so we get attached after h/w devices */ 1528 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1529 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1530 panic("cryptosoft: could not attach"); 1531 } 1532 1533 static int 1534 swcr_probe(device_t dev) 1535 { 1536 device_set_desc(dev, "software crypto"); 1537 device_quiet(dev); 1538 return (BUS_PROBE_NOWILDCARD); 1539 } 1540 1541 static int 1542 swcr_attach(device_t dev) 1543 { 1544 1545 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1546 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1547 if (swcr_id < 0) { 1548 device_printf(dev, "cannot initialize!"); 1549 return (ENXIO); 1550 } 1551 1552 return (0); 1553 } 1554 1555 static int 1556 swcr_detach(device_t dev) 1557 { 1558 crypto_unregister_all(swcr_id); 1559 return 0; 1560 } 1561 1562 static device_method_t swcr_methods[] = { 1563 DEVMETHOD(device_identify, swcr_identify), 1564 DEVMETHOD(device_probe, swcr_probe), 1565 DEVMETHOD(device_attach, swcr_attach), 1566 DEVMETHOD(device_detach, swcr_detach), 1567 1568 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1569 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1570 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1571 DEVMETHOD(cryptodev_process, swcr_process), 1572 1573 {0, 0}, 1574 }; 1575 1576 static driver_t swcr_driver = { 1577 "cryptosoft", 1578 swcr_methods, 1579 0, /* NB: no softc */ 1580 }; 1581 static devclass_t swcr_devclass; 1582 1583 /* 1584 * NB: We explicitly reference the crypto module so we 1585 * get the necessary ordering when built as a loadable 1586 * module. This is required because we bundle the crypto 1587 * module code together with the cryptosoft driver (otherwise 1588 * normal module dependencies would handle things). 1589 */ 1590 extern int crypto_modevent(struct module *, int, void *); 1591 /* XXX where to attach */ 1592 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1593 MODULE_VERSION(cryptosoft, 1); 1594 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1595