1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014-2021 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Portions of this software were developed by Ararat River 20 * Consulting, LLC under sponsorship of the FreeBSD Foundation. 21 * 22 * Permission to use, copy, and modify this software with or without fee 23 * is hereby granted, provided that this entire notice is included in 24 * all source code copies of any software which is or includes a copy or 25 * modification of this software. 26 * 27 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 28 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 29 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 30 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 31 * PURPOSE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/module.h> 42 #include <sys/sysctl.h> 43 #include <sys/errno.h> 44 #include <sys/random.h> 45 #include <sys/kernel.h> 46 #include <sys/uio.h> 47 #include <sys/endian.h> 48 #include <sys/limits.h> 49 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 53 #include <opencrypto/cryptodev.h> 54 #include <opencrypto/xform.h> 55 56 #include <sys/kobj.h> 57 #include <sys/bus.h> 58 #include "cryptodev_if.h" 59 60 struct swcr_auth { 61 void *sw_ictx; 62 void *sw_octx; 63 const struct auth_hash *sw_axf; 64 uint16_t sw_mlen; 65 bool sw_hmac; 66 }; 67 68 struct swcr_encdec { 69 void *sw_ctx; 70 const struct enc_xform *sw_exf; 71 }; 72 73 struct swcr_compdec { 74 const struct comp_algo *sw_cxf; 75 }; 76 77 struct swcr_session { 78 int (*swcr_process)(const struct swcr_session *, struct cryptop *); 79 80 struct swcr_auth swcr_auth; 81 struct swcr_encdec swcr_encdec; 82 struct swcr_compdec swcr_compdec; 83 }; 84 85 static int32_t swcr_id; 86 87 static void swcr_freesession(device_t dev, crypto_session_t cses); 88 89 /* Used for CRYPTO_NULL_CBC. */ 90 static int 91 swcr_null(const struct swcr_session *ses, struct cryptop *crp) 92 { 93 94 return (0); 95 } 96 97 /* 98 * Apply a symmetric encryption/decryption algorithm. 99 */ 100 static int 101 swcr_encdec(const struct swcr_session *ses, struct cryptop *crp) 102 { 103 unsigned char blk[EALG_MAX_BLOCK_LEN]; 104 const struct crypto_session_params *csp; 105 const struct enc_xform *exf; 106 const struct swcr_encdec *sw; 107 void *ctx; 108 size_t inlen, outlen; 109 int blks, resid; 110 struct crypto_buffer_cursor cc_in, cc_out; 111 const unsigned char *inblk; 112 unsigned char *outblk; 113 int error; 114 bool encrypting; 115 116 error = 0; 117 118 sw = &ses->swcr_encdec; 119 exf = sw->sw_exf; 120 csp = crypto_get_params(crp->crp_session); 121 122 if (exf->native_blocksize == 0) { 123 /* Check for non-padded data */ 124 if ((crp->crp_payload_length % exf->blocksize) != 0) 125 return (EINVAL); 126 127 blks = exf->blocksize; 128 } else 129 blks = exf->native_blocksize; 130 131 if (exf == &enc_xform_aes_icm && 132 (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 133 return (EINVAL); 134 135 ctx = __builtin_alloca(exf->ctxsize); 136 if (crp->crp_cipher_key != NULL) { 137 error = exf->setkey(ctx, crp->crp_cipher_key, 138 csp->csp_cipher_klen); 139 if (error) 140 return (error); 141 } else 142 memcpy(ctx, sw->sw_ctx, exf->ctxsize); 143 144 crypto_read_iv(crp, blk); 145 exf->reinit(ctx, blk, csp->csp_ivlen); 146 147 crypto_cursor_init(&cc_in, &crp->crp_buf); 148 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 149 inblk = crypto_cursor_segment(&cc_in, &inlen); 150 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 151 crypto_cursor_init(&cc_out, &crp->crp_obuf); 152 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 153 } else 154 cc_out = cc_in; 155 outblk = crypto_cursor_segment(&cc_out, &outlen); 156 157 resid = crp->crp_payload_length; 158 encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); 159 160 /* 161 * Loop through encrypting blocks. 'inlen' is the remaining 162 * length of the current segment in the input buffer. 163 * 'outlen' is the remaining length of current segment in the 164 * output buffer. 165 */ 166 while (resid >= blks) { 167 /* 168 * If the current block is not contained within the 169 * current input/output segment, use 'blk' as a local 170 * buffer. 171 */ 172 if (inlen < blks) { 173 crypto_cursor_copydata(&cc_in, blks, blk); 174 inblk = blk; 175 } 176 if (outlen < blks) 177 outblk = blk; 178 179 if (encrypting) 180 exf->encrypt(ctx, inblk, outblk); 181 else 182 exf->decrypt(ctx, inblk, outblk); 183 184 if (inlen < blks) { 185 inblk = crypto_cursor_segment(&cc_in, &inlen); 186 } else { 187 crypto_cursor_advance(&cc_in, blks); 188 inlen -= blks; 189 inblk += blks; 190 } 191 192 if (outlen < blks) { 193 crypto_cursor_copyback(&cc_out, blks, blk); 194 outblk = crypto_cursor_segment(&cc_out, &outlen); 195 } else { 196 crypto_cursor_advance(&cc_out, blks); 197 outlen -= blks; 198 outblk += blks; 199 } 200 201 resid -= blks; 202 } 203 204 /* Handle trailing partial block for stream ciphers. */ 205 if (resid > 0) { 206 KASSERT(exf->native_blocksize != 0, 207 ("%s: partial block of %d bytes for cipher %s", 208 __func__, resid, exf->name)); 209 KASSERT(resid < blks, ("%s: partial block too big", __func__)); 210 211 inblk = crypto_cursor_segment(&cc_in, &inlen); 212 outblk = crypto_cursor_segment(&cc_out, &outlen); 213 if (inlen < resid) { 214 crypto_cursor_copydata(&cc_in, resid, blk); 215 inblk = blk; 216 } 217 if (outlen < resid) 218 outblk = blk; 219 if (encrypting) 220 exf->encrypt_last(ctx, inblk, outblk, 221 resid); 222 else 223 exf->decrypt_last(ctx, inblk, outblk, 224 resid); 225 if (outlen < resid) 226 crypto_cursor_copyback(&cc_out, resid, blk); 227 } 228 229 explicit_bzero(ctx, exf->ctxsize); 230 explicit_bzero(blk, sizeof(blk)); 231 return (0); 232 } 233 234 /* 235 * Compute or verify hash. 236 */ 237 static int 238 swcr_authcompute(const struct swcr_session *ses, struct cryptop *crp) 239 { 240 u_char aalg[HASH_MAX_LEN]; 241 const struct crypto_session_params *csp; 242 const struct swcr_auth *sw; 243 const struct auth_hash *axf; 244 union authctx ctx; 245 int err; 246 247 sw = &ses->swcr_auth; 248 249 axf = sw->sw_axf; 250 251 csp = crypto_get_params(crp->crp_session); 252 if (crp->crp_auth_key != NULL) { 253 if (sw->sw_hmac) { 254 hmac_init_ipad(axf, crp->crp_auth_key, 255 csp->csp_auth_klen, &ctx); 256 } else { 257 axf->Init(&ctx); 258 axf->Setkey(&ctx, crp->crp_auth_key, 259 csp->csp_auth_klen); 260 } 261 } else 262 memcpy(&ctx, sw->sw_ictx, axf->ctxsize); 263 264 if (crp->crp_aad != NULL) 265 err = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length); 266 else 267 err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 268 axf->Update, &ctx); 269 if (err) 270 goto out; 271 272 if (CRYPTO_HAS_OUTPUT_BUFFER(crp) && 273 CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 274 err = crypto_apply_buf(&crp->crp_obuf, 275 crp->crp_payload_output_start, crp->crp_payload_length, 276 axf->Update, &ctx); 277 else 278 err = crypto_apply(crp, crp->crp_payload_start, 279 crp->crp_payload_length, axf->Update, &ctx); 280 if (err) 281 goto out; 282 283 if (csp->csp_flags & CSP_F_ESN) 284 axf->Update(&ctx, crp->crp_esn, 4); 285 286 axf->Final(aalg, &ctx); 287 if (sw->sw_hmac) { 288 if (crp->crp_auth_key != NULL) 289 hmac_init_opad(axf, crp->crp_auth_key, 290 csp->csp_auth_klen, &ctx); 291 else 292 memcpy(&ctx, sw->sw_octx, axf->ctxsize); 293 axf->Update(&ctx, aalg, axf->hashsize); 294 axf->Final(aalg, &ctx); 295 } 296 297 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 298 u_char uaalg[HASH_MAX_LEN]; 299 300 crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg); 301 if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0) 302 err = EBADMSG; 303 explicit_bzero(uaalg, sizeof(uaalg)); 304 } else { 305 /* Inject the authentication data */ 306 crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg); 307 } 308 explicit_bzero(aalg, sizeof(aalg)); 309 out: 310 explicit_bzero(&ctx, sizeof(ctx)); 311 return (err); 312 } 313 314 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 315 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 316 317 static int 318 swcr_gmac(const struct swcr_session *ses, struct cryptop *crp) 319 { 320 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 321 u_char *blk = (u_char *)blkbuf; 322 u_char tag[GMAC_DIGEST_LEN]; 323 u_char iv[AES_BLOCK_LEN]; 324 struct crypto_buffer_cursor cc; 325 const u_char *inblk; 326 union authctx ctx; 327 const struct swcr_auth *swa; 328 const struct auth_hash *axf; 329 uint32_t *blkp; 330 size_t len; 331 int blksz, error, ivlen, resid; 332 333 swa = &ses->swcr_auth; 334 axf = swa->sw_axf; 335 blksz = GMAC_BLOCK_LEN; 336 KASSERT(axf->blocksize == blksz, ("%s: axf block size mismatch", 337 __func__)); 338 339 if (crp->crp_auth_key != NULL) { 340 axf->Init(&ctx); 341 axf->Setkey(&ctx, crp->crp_auth_key, 342 crypto_get_params(crp->crp_session)->csp_auth_klen); 343 } else 344 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 345 346 /* Initialize the IV */ 347 ivlen = AES_GCM_IV_LEN; 348 crypto_read_iv(crp, iv); 349 350 axf->Reinit(&ctx, iv, ivlen); 351 crypto_cursor_init(&cc, &crp->crp_buf); 352 crypto_cursor_advance(&cc, crp->crp_payload_start); 353 for (resid = crp->crp_payload_length; resid >= blksz; resid -= len) { 354 inblk = crypto_cursor_segment(&cc, &len); 355 if (len >= blksz) { 356 len = rounddown(MIN(len, resid), blksz); 357 crypto_cursor_advance(&cc, len); 358 } else { 359 len = blksz; 360 crypto_cursor_copydata(&cc, len, blk); 361 inblk = blk; 362 } 363 axf->Update(&ctx, inblk, len); 364 } 365 if (resid > 0) { 366 memset(blk, 0, blksz); 367 crypto_cursor_copydata(&cc, resid, blk); 368 axf->Update(&ctx, blk, blksz); 369 } 370 371 /* length block */ 372 memset(blk, 0, blksz); 373 blkp = (uint32_t *)blk + 1; 374 *blkp = htobe32(crp->crp_payload_length * 8); 375 axf->Update(&ctx, blk, blksz); 376 377 /* Finalize MAC */ 378 axf->Final(tag, &ctx); 379 380 error = 0; 381 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 382 u_char tag2[GMAC_DIGEST_LEN]; 383 384 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 385 tag2); 386 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 387 error = EBADMSG; 388 explicit_bzero(tag2, sizeof(tag2)); 389 } else { 390 /* Inject the authentication data */ 391 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 392 } 393 explicit_bzero(blkbuf, sizeof(blkbuf)); 394 explicit_bzero(tag, sizeof(tag)); 395 explicit_bzero(iv, sizeof(iv)); 396 return (error); 397 } 398 399 static int 400 swcr_gcm(const struct swcr_session *ses, struct cryptop *crp) 401 { 402 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 403 u_char *blk = (u_char *)blkbuf; 404 u_char tag[GMAC_DIGEST_LEN]; 405 struct crypto_buffer_cursor cc_in, cc_out; 406 const u_char *inblk; 407 u_char *outblk; 408 const struct swcr_auth *swa; 409 const struct swcr_encdec *swe; 410 const struct enc_xform *exf; 411 void *ctx; 412 uint32_t *blkp; 413 size_t len; 414 int blksz, error, ivlen, r, resid; 415 416 swa = &ses->swcr_auth; 417 swe = &ses->swcr_encdec; 418 exf = swe->sw_exf; 419 blksz = GMAC_BLOCK_LEN; 420 KASSERT(blksz == exf->native_blocksize, 421 ("%s: blocksize mismatch", __func__)); 422 423 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 424 return (EINVAL); 425 426 ivlen = AES_GCM_IV_LEN; 427 428 ctx = __builtin_alloca(exf->ctxsize); 429 if (crp->crp_cipher_key != NULL) 430 exf->setkey(ctx, crp->crp_cipher_key, 431 crypto_get_params(crp->crp_session)->csp_cipher_klen); 432 else 433 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 434 exf->reinit(ctx, crp->crp_iv, ivlen); 435 436 /* Supply MAC with AAD */ 437 if (crp->crp_aad != NULL) { 438 len = rounddown(crp->crp_aad_length, blksz); 439 if (len != 0) 440 exf->update(ctx, crp->crp_aad, len); 441 if (crp->crp_aad_length != len) { 442 memset(blk, 0, blksz); 443 memcpy(blk, (char *)crp->crp_aad + len, 444 crp->crp_aad_length - len); 445 exf->update(ctx, blk, blksz); 446 } 447 } else { 448 crypto_cursor_init(&cc_in, &crp->crp_buf); 449 crypto_cursor_advance(&cc_in, crp->crp_aad_start); 450 for (resid = crp->crp_aad_length; resid >= blksz; 451 resid -= len) { 452 inblk = crypto_cursor_segment(&cc_in, &len); 453 if (len >= blksz) { 454 len = rounddown(MIN(len, resid), blksz); 455 crypto_cursor_advance(&cc_in, len); 456 } else { 457 len = blksz; 458 crypto_cursor_copydata(&cc_in, len, blk); 459 inblk = blk; 460 } 461 exf->update(ctx, inblk, len); 462 } 463 if (resid > 0) { 464 memset(blk, 0, blksz); 465 crypto_cursor_copydata(&cc_in, resid, blk); 466 exf->update(ctx, blk, blksz); 467 } 468 } 469 470 /* Do encryption with MAC */ 471 crypto_cursor_init(&cc_in, &crp->crp_buf); 472 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 473 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 474 crypto_cursor_init(&cc_out, &crp->crp_obuf); 475 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 476 } else 477 cc_out = cc_in; 478 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 479 inblk = crypto_cursor_segment(&cc_in, &len); 480 if (len < blksz) { 481 crypto_cursor_copydata(&cc_in, blksz, blk); 482 inblk = blk; 483 } else { 484 crypto_cursor_advance(&cc_in, blksz); 485 } 486 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 487 outblk = crypto_cursor_segment(&cc_out, &len); 488 if (len < blksz) 489 outblk = blk; 490 exf->encrypt(ctx, inblk, outblk); 491 exf->update(ctx, outblk, blksz); 492 if (outblk == blk) 493 crypto_cursor_copyback(&cc_out, blksz, blk); 494 else 495 crypto_cursor_advance(&cc_out, blksz); 496 } else { 497 exf->update(ctx, inblk, blksz); 498 } 499 } 500 if (resid > 0) { 501 crypto_cursor_copydata(&cc_in, resid, blk); 502 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 503 exf->encrypt_last(ctx, blk, blk, resid); 504 crypto_cursor_copyback(&cc_out, resid, blk); 505 } 506 exf->update(ctx, blk, resid); 507 } 508 509 /* length block */ 510 memset(blk, 0, blksz); 511 blkp = (uint32_t *)blk + 1; 512 *blkp = htobe32(crp->crp_aad_length * 8); 513 blkp = (uint32_t *)blk + 3; 514 *blkp = htobe32(crp->crp_payload_length * 8); 515 exf->update(ctx, blk, blksz); 516 517 /* Finalize MAC */ 518 exf->final(tag, ctx); 519 520 /* Validate tag */ 521 error = 0; 522 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 523 u_char tag2[GMAC_DIGEST_LEN]; 524 525 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 526 527 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 528 explicit_bzero(tag2, sizeof(tag2)); 529 if (r != 0) { 530 error = EBADMSG; 531 goto out; 532 } 533 534 /* tag matches, decrypt data */ 535 crypto_cursor_init(&cc_in, &crp->crp_buf); 536 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 537 for (resid = crp->crp_payload_length; resid > blksz; 538 resid -= blksz) { 539 inblk = crypto_cursor_segment(&cc_in, &len); 540 if (len < blksz) { 541 crypto_cursor_copydata(&cc_in, blksz, blk); 542 inblk = blk; 543 } else 544 crypto_cursor_advance(&cc_in, blksz); 545 outblk = crypto_cursor_segment(&cc_out, &len); 546 if (len < blksz) 547 outblk = blk; 548 exf->decrypt(ctx, inblk, outblk); 549 if (outblk == blk) 550 crypto_cursor_copyback(&cc_out, blksz, blk); 551 else 552 crypto_cursor_advance(&cc_out, blksz); 553 } 554 if (resid > 0) { 555 crypto_cursor_copydata(&cc_in, resid, blk); 556 exf->decrypt_last(ctx, blk, blk, resid); 557 crypto_cursor_copyback(&cc_out, resid, blk); 558 } 559 } else { 560 /* Inject the authentication data */ 561 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 562 } 563 564 out: 565 explicit_bzero(ctx, exf->ctxsize); 566 explicit_bzero(blkbuf, sizeof(blkbuf)); 567 explicit_bzero(tag, sizeof(tag)); 568 569 return (error); 570 } 571 572 static void 573 build_ccm_b0(const char *nonce, u_int nonce_length, u_int aad_length, 574 u_int data_length, u_int tag_length, uint8_t *b0) 575 { 576 uint8_t *bp; 577 uint8_t flags, L; 578 579 KASSERT(nonce_length >= 7 && nonce_length <= 13, 580 ("nonce_length must be between 7 and 13 bytes")); 581 582 /* 583 * Need to determine the L field value. This is the number of 584 * bytes needed to specify the length of the message; the length 585 * is whatever is left in the 16 bytes after specifying flags and 586 * the nonce. 587 */ 588 L = 15 - nonce_length; 589 590 flags = ((aad_length > 0) << 6) + 591 (((tag_length - 2) / 2) << 3) + 592 L - 1; 593 594 /* 595 * Now we need to set up the first block, which has flags, nonce, 596 * and the message length. 597 */ 598 b0[0] = flags; 599 memcpy(b0 + 1, nonce, nonce_length); 600 bp = b0 + 1 + nonce_length; 601 602 /* Need to copy L' [aka L-1] bytes of data_length */ 603 for (uint8_t *dst = b0 + CCM_CBC_BLOCK_LEN - 1; dst >= bp; dst--) { 604 *dst = data_length; 605 data_length >>= 8; 606 } 607 } 608 609 /* NB: OCF only supports AAD lengths < 2^32. */ 610 static int 611 build_ccm_aad_length(u_int aad_length, uint8_t *blk) 612 { 613 if (aad_length < ((1 << 16) - (1 << 8))) { 614 be16enc(blk, aad_length); 615 return (sizeof(uint16_t)); 616 } else { 617 blk[0] = 0xff; 618 blk[1] = 0xfe; 619 be32enc(blk + 2, aad_length); 620 return (2 + sizeof(uint32_t)); 621 } 622 } 623 624 static int 625 swcr_ccm_cbc_mac(const struct swcr_session *ses, struct cryptop *crp) 626 { 627 u_char blk[CCM_CBC_BLOCK_LEN]; 628 u_char tag[AES_CBC_MAC_HASH_LEN]; 629 union authctx ctx; 630 const struct crypto_session_params *csp; 631 const struct swcr_auth *swa; 632 const struct auth_hash *axf; 633 int error, ivlen, len; 634 635 csp = crypto_get_params(crp->crp_session); 636 swa = &ses->swcr_auth; 637 axf = swa->sw_axf; 638 639 if (crp->crp_auth_key != NULL) { 640 axf->Init(&ctx); 641 axf->Setkey(&ctx, crp->crp_auth_key, csp->csp_auth_klen); 642 } else 643 memcpy(&ctx, swa->sw_ictx, axf->ctxsize); 644 645 /* Initialize the IV */ 646 ivlen = csp->csp_ivlen; 647 648 /* Supply MAC with IV */ 649 axf->Reinit(&ctx, crp->crp_iv, ivlen); 650 651 /* Supply MAC with b0. */ 652 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_payload_length, 0, 653 swa->sw_mlen, blk); 654 axf->Update(&ctx, blk, CCM_CBC_BLOCK_LEN); 655 656 len = build_ccm_aad_length(crp->crp_payload_length, blk); 657 axf->Update(&ctx, blk, len); 658 659 crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length, 660 axf->Update, &ctx); 661 662 /* Finalize MAC */ 663 axf->Final(tag, &ctx); 664 665 error = 0; 666 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) { 667 u_char tag2[AES_CBC_MAC_HASH_LEN]; 668 669 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 670 tag2); 671 if (timingsafe_bcmp(tag, tag2, swa->sw_mlen) != 0) 672 error = EBADMSG; 673 explicit_bzero(tag2, sizeof(tag)); 674 } else { 675 /* Inject the authentication data */ 676 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 677 } 678 explicit_bzero(tag, sizeof(tag)); 679 explicit_bzero(blk, sizeof(blk)); 680 return (error); 681 } 682 683 static int 684 swcr_ccm(const struct swcr_session *ses, struct cryptop *crp) 685 { 686 const struct crypto_session_params *csp; 687 uint32_t blkbuf[howmany(AES_BLOCK_LEN, sizeof(uint32_t))]; 688 u_char *blk = (u_char *)blkbuf; 689 u_char tag[AES_CBC_MAC_HASH_LEN]; 690 struct crypto_buffer_cursor cc_in, cc_out; 691 const u_char *inblk; 692 u_char *outblk; 693 const struct swcr_auth *swa; 694 const struct swcr_encdec *swe; 695 const struct enc_xform *exf; 696 void *ctx; 697 size_t len; 698 int blksz, error, ivlen, r, resid; 699 700 csp = crypto_get_params(crp->crp_session); 701 swa = &ses->swcr_auth; 702 swe = &ses->swcr_encdec; 703 exf = swe->sw_exf; 704 blksz = AES_BLOCK_LEN; 705 KASSERT(blksz == exf->native_blocksize, 706 ("%s: blocksize mismatch", __func__)); 707 708 if (crp->crp_payload_length > ccm_max_payload_length(csp)) 709 return (EMSGSIZE); 710 711 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 712 return (EINVAL); 713 714 ivlen = csp->csp_ivlen; 715 716 ctx = __builtin_alloca(exf->ctxsize); 717 if (crp->crp_cipher_key != NULL) 718 exf->setkey(ctx, crp->crp_cipher_key, 719 crypto_get_params(crp->crp_session)->csp_cipher_klen); 720 else 721 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 722 exf->reinit(ctx, crp->crp_iv, ivlen); 723 724 /* Supply MAC with b0. */ 725 _Static_assert(sizeof(blkbuf) >= CCM_CBC_BLOCK_LEN, 726 "blkbuf too small for b0"); 727 build_ccm_b0(crp->crp_iv, ivlen, crp->crp_aad_length, 728 crp->crp_payload_length, swa->sw_mlen, blk); 729 exf->update(ctx, blk, CCM_CBC_BLOCK_LEN); 730 731 /* Supply MAC with AAD */ 732 if (crp->crp_aad_length != 0) { 733 len = build_ccm_aad_length(crp->crp_aad_length, blk); 734 exf->update(ctx, blk, len); 735 if (crp->crp_aad != NULL) 736 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 737 else 738 crypto_apply(crp, crp->crp_aad_start, 739 crp->crp_aad_length, exf->update, ctx); 740 741 /* Pad the AAD (including length field) to a full block. */ 742 len = (len + crp->crp_aad_length) % CCM_CBC_BLOCK_LEN; 743 if (len != 0) { 744 len = CCM_CBC_BLOCK_LEN - len; 745 memset(blk, 0, CCM_CBC_BLOCK_LEN); 746 exf->update(ctx, blk, len); 747 } 748 } 749 750 /* Do encryption/decryption with MAC */ 751 crypto_cursor_init(&cc_in, &crp->crp_buf); 752 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 753 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 754 crypto_cursor_init(&cc_out, &crp->crp_obuf); 755 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 756 } else 757 cc_out = cc_in; 758 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 759 inblk = crypto_cursor_segment(&cc_in, &len); 760 if (len < blksz) { 761 crypto_cursor_copydata(&cc_in, blksz, blk); 762 inblk = blk; 763 } else 764 crypto_cursor_advance(&cc_in, blksz); 765 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 766 outblk = crypto_cursor_segment(&cc_out, &len); 767 if (len < blksz) 768 outblk = blk; 769 exf->update(ctx, inblk, blksz); 770 exf->encrypt(ctx, inblk, outblk); 771 if (outblk == blk) 772 crypto_cursor_copyback(&cc_out, blksz, blk); 773 else 774 crypto_cursor_advance(&cc_out, blksz); 775 } else { 776 /* 777 * One of the problems with CCM+CBC is that 778 * the authentication is done on the 779 * unencrypted data. As a result, we have to 780 * decrypt the data twice: once to generate 781 * the tag and a second time after the tag is 782 * verified. 783 */ 784 exf->decrypt(ctx, inblk, blk); 785 exf->update(ctx, blk, blksz); 786 } 787 } 788 if (resid > 0) { 789 crypto_cursor_copydata(&cc_in, resid, blk); 790 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 791 exf->update(ctx, blk, resid); 792 exf->encrypt_last(ctx, blk, blk, resid); 793 crypto_cursor_copyback(&cc_out, resid, blk); 794 } else { 795 exf->decrypt_last(ctx, blk, blk, resid); 796 exf->update(ctx, blk, resid); 797 } 798 } 799 800 /* Finalize MAC */ 801 exf->final(tag, ctx); 802 803 /* Validate tag */ 804 error = 0; 805 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 806 u_char tag2[AES_CBC_MAC_HASH_LEN]; 807 808 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, 809 tag2); 810 811 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 812 explicit_bzero(tag2, sizeof(tag2)); 813 if (r != 0) { 814 error = EBADMSG; 815 goto out; 816 } 817 818 /* tag matches, decrypt data */ 819 exf->reinit(ctx, crp->crp_iv, ivlen); 820 crypto_cursor_init(&cc_in, &crp->crp_buf); 821 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 822 for (resid = crp->crp_payload_length; resid > blksz; 823 resid -= blksz) { 824 inblk = crypto_cursor_segment(&cc_in, &len); 825 if (len < blksz) { 826 crypto_cursor_copydata(&cc_in, blksz, blk); 827 inblk = blk; 828 } else 829 crypto_cursor_advance(&cc_in, blksz); 830 outblk = crypto_cursor_segment(&cc_out, &len); 831 if (len < blksz) 832 outblk = blk; 833 exf->decrypt(ctx, inblk, outblk); 834 if (outblk == blk) 835 crypto_cursor_copyback(&cc_out, blksz, blk); 836 else 837 crypto_cursor_advance(&cc_out, blksz); 838 } 839 if (resid > 0) { 840 crypto_cursor_copydata(&cc_in, resid, blk); 841 exf->decrypt_last(ctx, blk, blk, resid); 842 crypto_cursor_copyback(&cc_out, resid, blk); 843 } 844 } else { 845 /* Inject the authentication data */ 846 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 847 } 848 849 out: 850 explicit_bzero(ctx, exf->ctxsize); 851 explicit_bzero(blkbuf, sizeof(blkbuf)); 852 explicit_bzero(tag, sizeof(tag)); 853 return (error); 854 } 855 856 static int 857 swcr_chacha20_poly1305(const struct swcr_session *ses, struct cryptop *crp) 858 { 859 const struct crypto_session_params *csp; 860 uint64_t blkbuf[howmany(CHACHA20_NATIVE_BLOCK_LEN, sizeof(uint64_t))]; 861 u_char *blk = (u_char *)blkbuf; 862 u_char tag[POLY1305_HASH_LEN]; 863 struct crypto_buffer_cursor cc_in, cc_out; 864 const u_char *inblk; 865 u_char *outblk; 866 uint64_t *blkp; 867 const struct swcr_auth *swa; 868 const struct swcr_encdec *swe; 869 const struct enc_xform *exf; 870 void *ctx; 871 size_t len; 872 int blksz, error, r, resid; 873 874 swa = &ses->swcr_auth; 875 swe = &ses->swcr_encdec; 876 exf = swe->sw_exf; 877 blksz = exf->native_blocksize; 878 KASSERT(blksz <= sizeof(blkbuf), ("%s: blocksize mismatch", __func__)); 879 880 if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) 881 return (EINVAL); 882 883 csp = crypto_get_params(crp->crp_session); 884 885 ctx = __builtin_alloca(exf->ctxsize); 886 if (crp->crp_cipher_key != NULL) 887 exf->setkey(ctx, crp->crp_cipher_key, 888 csp->csp_cipher_klen); 889 else 890 memcpy(ctx, swe->sw_ctx, exf->ctxsize); 891 exf->reinit(ctx, crp->crp_iv, csp->csp_ivlen); 892 893 /* Supply MAC with AAD */ 894 if (crp->crp_aad != NULL) 895 exf->update(ctx, crp->crp_aad, crp->crp_aad_length); 896 else 897 crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length, 898 exf->update, ctx); 899 if (crp->crp_aad_length % 16 != 0) { 900 /* padding1 */ 901 memset(blk, 0, 16); 902 exf->update(ctx, blk, 16 - crp->crp_aad_length % 16); 903 } 904 905 /* Do encryption with MAC */ 906 crypto_cursor_init(&cc_in, &crp->crp_buf); 907 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 908 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 909 crypto_cursor_init(&cc_out, &crp->crp_obuf); 910 crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); 911 } else 912 cc_out = cc_in; 913 for (resid = crp->crp_payload_length; resid >= blksz; resid -= blksz) { 914 inblk = crypto_cursor_segment(&cc_in, &len); 915 if (len < blksz) { 916 crypto_cursor_copydata(&cc_in, blksz, blk); 917 inblk = blk; 918 } else 919 crypto_cursor_advance(&cc_in, blksz); 920 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 921 outblk = crypto_cursor_segment(&cc_out, &len); 922 if (len < blksz) 923 outblk = blk; 924 exf->encrypt(ctx, inblk, outblk); 925 exf->update(ctx, outblk, blksz); 926 if (outblk == blk) 927 crypto_cursor_copyback(&cc_out, blksz, blk); 928 else 929 crypto_cursor_advance(&cc_out, blksz); 930 } else { 931 exf->update(ctx, inblk, blksz); 932 } 933 } 934 if (resid > 0) { 935 crypto_cursor_copydata(&cc_in, resid, blk); 936 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 937 exf->encrypt_last(ctx, blk, blk, resid); 938 crypto_cursor_copyback(&cc_out, resid, blk); 939 } 940 exf->update(ctx, blk, resid); 941 if (resid % 16 != 0) { 942 /* padding2 */ 943 memset(blk, 0, 16); 944 exf->update(ctx, blk, 16 - resid % 16); 945 } 946 } 947 948 /* lengths */ 949 blkp = (uint64_t *)blk; 950 blkp[0] = htole64(crp->crp_aad_length); 951 blkp[1] = htole64(crp->crp_payload_length); 952 exf->update(ctx, blk, sizeof(uint64_t) * 2); 953 954 /* Finalize MAC */ 955 exf->final(tag, ctx); 956 957 /* Validate tag */ 958 error = 0; 959 if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 960 u_char tag2[POLY1305_HASH_LEN]; 961 962 crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen, tag2); 963 964 r = timingsafe_bcmp(tag, tag2, swa->sw_mlen); 965 explicit_bzero(tag2, sizeof(tag2)); 966 if (r != 0) { 967 error = EBADMSG; 968 goto out; 969 } 970 971 /* tag matches, decrypt data */ 972 crypto_cursor_init(&cc_in, &crp->crp_buf); 973 crypto_cursor_advance(&cc_in, crp->crp_payload_start); 974 for (resid = crp->crp_payload_length; resid > blksz; 975 resid -= blksz) { 976 inblk = crypto_cursor_segment(&cc_in, &len); 977 if (len < blksz) { 978 crypto_cursor_copydata(&cc_in, blksz, blk); 979 inblk = blk; 980 } else 981 crypto_cursor_advance(&cc_in, blksz); 982 outblk = crypto_cursor_segment(&cc_out, &len); 983 if (len < blksz) 984 outblk = blk; 985 exf->decrypt(ctx, inblk, outblk); 986 if (outblk == blk) 987 crypto_cursor_copyback(&cc_out, blksz, blk); 988 else 989 crypto_cursor_advance(&cc_out, blksz); 990 } 991 if (resid > 0) { 992 crypto_cursor_copydata(&cc_in, resid, blk); 993 exf->decrypt_last(ctx, blk, blk, resid); 994 crypto_cursor_copyback(&cc_out, resid, blk); 995 } 996 } else { 997 /* Inject the authentication data */ 998 crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, tag); 999 } 1000 1001 out: 1002 explicit_bzero(ctx, exf->ctxsize); 1003 explicit_bzero(blkbuf, sizeof(blkbuf)); 1004 explicit_bzero(tag, sizeof(tag)); 1005 return (error); 1006 } 1007 1008 /* 1009 * Apply a cipher and a digest to perform EtA. 1010 */ 1011 static int 1012 swcr_eta(const struct swcr_session *ses, struct cryptop *crp) 1013 { 1014 int error; 1015 1016 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 1017 error = swcr_encdec(ses, crp); 1018 if (error == 0) 1019 error = swcr_authcompute(ses, crp); 1020 } else { 1021 error = swcr_authcompute(ses, crp); 1022 if (error == 0) 1023 error = swcr_encdec(ses, crp); 1024 } 1025 return (error); 1026 } 1027 1028 /* 1029 * Apply a compression/decompression algorithm 1030 */ 1031 static int 1032 swcr_compdec(const struct swcr_session *ses, struct cryptop *crp) 1033 { 1034 const struct comp_algo *cxf; 1035 uint8_t *data, *out; 1036 int adj; 1037 uint32_t result; 1038 1039 cxf = ses->swcr_compdec.sw_cxf; 1040 1041 /* We must handle the whole buffer of data in one time 1042 * then if there is not all the data in the mbuf, we must 1043 * copy in a buffer. 1044 */ 1045 1046 data = malloc(crp->crp_payload_length, M_CRYPTO_DATA, M_NOWAIT); 1047 if (data == NULL) 1048 return (EINVAL); 1049 crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length, 1050 data); 1051 1052 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) 1053 result = cxf->compress(data, crp->crp_payload_length, &out); 1054 else 1055 result = cxf->decompress(data, crp->crp_payload_length, &out); 1056 1057 free(data, M_CRYPTO_DATA); 1058 if (result == 0) 1059 return (EINVAL); 1060 crp->crp_olen = result; 1061 1062 /* Check the compressed size when doing compression */ 1063 if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) { 1064 if (result >= crp->crp_payload_length) { 1065 /* Compression was useless, we lost time */ 1066 free(out, M_CRYPTO_DATA); 1067 return (0); 1068 } 1069 } 1070 1071 /* Copy back the (de)compressed data. m_copyback is 1072 * extending the mbuf as necessary. 1073 */ 1074 crypto_copyback(crp, crp->crp_payload_start, result, out); 1075 if (result < crp->crp_payload_length) { 1076 switch (crp->crp_buf.cb_type) { 1077 case CRYPTO_BUF_MBUF: 1078 case CRYPTO_BUF_SINGLE_MBUF: 1079 adj = result - crp->crp_payload_length; 1080 m_adj(crp->crp_buf.cb_mbuf, adj); 1081 break; 1082 case CRYPTO_BUF_UIO: { 1083 struct uio *uio = crp->crp_buf.cb_uio; 1084 int ind; 1085 1086 adj = crp->crp_payload_length - result; 1087 ind = uio->uio_iovcnt - 1; 1088 1089 while (adj > 0 && ind >= 0) { 1090 if (adj < uio->uio_iov[ind].iov_len) { 1091 uio->uio_iov[ind].iov_len -= adj; 1092 break; 1093 } 1094 1095 adj -= uio->uio_iov[ind].iov_len; 1096 uio->uio_iov[ind].iov_len = 0; 1097 ind--; 1098 uio->uio_iovcnt--; 1099 } 1100 } 1101 break; 1102 case CRYPTO_BUF_VMPAGE: 1103 adj = crp->crp_payload_length - result; 1104 crp->crp_buf.cb_vm_page_len -= adj; 1105 break; 1106 default: 1107 break; 1108 } 1109 } 1110 free(out, M_CRYPTO_DATA); 1111 return 0; 1112 } 1113 1114 static int 1115 swcr_setup_cipher(struct swcr_session *ses, 1116 const struct crypto_session_params *csp) 1117 { 1118 struct swcr_encdec *swe; 1119 const struct enc_xform *txf; 1120 int error; 1121 1122 swe = &ses->swcr_encdec; 1123 txf = crypto_cipher(csp); 1124 if (csp->csp_cipher_key != NULL) { 1125 if (txf->ctxsize != 0) { 1126 swe->sw_ctx = malloc(txf->ctxsize, M_CRYPTO_DATA, 1127 M_NOWAIT); 1128 if (swe->sw_ctx == NULL) 1129 return (ENOMEM); 1130 } 1131 error = txf->setkey(swe->sw_ctx, 1132 csp->csp_cipher_key, csp->csp_cipher_klen); 1133 if (error) 1134 return (error); 1135 } 1136 swe->sw_exf = txf; 1137 return (0); 1138 } 1139 1140 static int 1141 swcr_setup_auth(struct swcr_session *ses, 1142 const struct crypto_session_params *csp) 1143 { 1144 struct swcr_auth *swa; 1145 const struct auth_hash *axf; 1146 1147 swa = &ses->swcr_auth; 1148 1149 axf = crypto_auth_hash(csp); 1150 swa->sw_axf = axf; 1151 if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize) 1152 return (EINVAL); 1153 if (csp->csp_auth_mlen == 0) 1154 swa->sw_mlen = axf->hashsize; 1155 else 1156 swa->sw_mlen = csp->csp_auth_mlen; 1157 if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL) { 1158 swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1159 M_NOWAIT); 1160 if (swa->sw_ictx == NULL) 1161 return (ENOBUFS); 1162 } 1163 1164 switch (csp->csp_auth_alg) { 1165 case CRYPTO_SHA1_HMAC: 1166 case CRYPTO_SHA2_224_HMAC: 1167 case CRYPTO_SHA2_256_HMAC: 1168 case CRYPTO_SHA2_384_HMAC: 1169 case CRYPTO_SHA2_512_HMAC: 1170 case CRYPTO_RIPEMD160_HMAC: 1171 swa->sw_hmac = true; 1172 if (csp->csp_auth_key != NULL) { 1173 swa->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 1174 M_NOWAIT); 1175 if (swa->sw_octx == NULL) 1176 return (ENOBUFS); 1177 hmac_init_ipad(axf, csp->csp_auth_key, 1178 csp->csp_auth_klen, swa->sw_ictx); 1179 hmac_init_opad(axf, csp->csp_auth_key, 1180 csp->csp_auth_klen, swa->sw_octx); 1181 } 1182 break; 1183 case CRYPTO_SHA1: 1184 case CRYPTO_SHA2_224: 1185 case CRYPTO_SHA2_256: 1186 case CRYPTO_SHA2_384: 1187 case CRYPTO_SHA2_512: 1188 case CRYPTO_NULL_HMAC: 1189 axf->Init(swa->sw_ictx); 1190 break; 1191 case CRYPTO_AES_NIST_GMAC: 1192 case CRYPTO_AES_CCM_CBC_MAC: 1193 case CRYPTO_POLY1305: 1194 if (csp->csp_auth_key != NULL) { 1195 axf->Init(swa->sw_ictx); 1196 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1197 csp->csp_auth_klen); 1198 } 1199 break; 1200 case CRYPTO_BLAKE2B: 1201 case CRYPTO_BLAKE2S: 1202 /* 1203 * Blake2b and Blake2s support an optional key but do 1204 * not require one. 1205 */ 1206 if (csp->csp_auth_klen == 0) 1207 axf->Init(swa->sw_ictx); 1208 else if (csp->csp_auth_key != NULL) 1209 axf->Setkey(swa->sw_ictx, csp->csp_auth_key, 1210 csp->csp_auth_klen); 1211 break; 1212 } 1213 1214 if (csp->csp_mode == CSP_MODE_DIGEST) { 1215 switch (csp->csp_auth_alg) { 1216 case CRYPTO_AES_NIST_GMAC: 1217 ses->swcr_process = swcr_gmac; 1218 break; 1219 case CRYPTO_AES_CCM_CBC_MAC: 1220 ses->swcr_process = swcr_ccm_cbc_mac; 1221 break; 1222 default: 1223 ses->swcr_process = swcr_authcompute; 1224 } 1225 } 1226 1227 return (0); 1228 } 1229 1230 static int 1231 swcr_setup_aead(struct swcr_session *ses, 1232 const struct crypto_session_params *csp) 1233 { 1234 struct swcr_auth *swa; 1235 int error; 1236 1237 error = swcr_setup_cipher(ses, csp); 1238 if (error) 1239 return (error); 1240 1241 swa = &ses->swcr_auth; 1242 if (csp->csp_auth_mlen == 0) 1243 swa->sw_mlen = ses->swcr_encdec.sw_exf->macsize; 1244 else 1245 swa->sw_mlen = csp->csp_auth_mlen; 1246 return (0); 1247 } 1248 1249 static bool 1250 swcr_auth_supported(const struct crypto_session_params *csp) 1251 { 1252 const struct auth_hash *axf; 1253 1254 axf = crypto_auth_hash(csp); 1255 if (axf == NULL) 1256 return (false); 1257 switch (csp->csp_auth_alg) { 1258 case CRYPTO_SHA1_HMAC: 1259 case CRYPTO_SHA2_224_HMAC: 1260 case CRYPTO_SHA2_256_HMAC: 1261 case CRYPTO_SHA2_384_HMAC: 1262 case CRYPTO_SHA2_512_HMAC: 1263 case CRYPTO_NULL_HMAC: 1264 case CRYPTO_RIPEMD160_HMAC: 1265 break; 1266 case CRYPTO_AES_NIST_GMAC: 1267 switch (csp->csp_auth_klen * 8) { 1268 case 128: 1269 case 192: 1270 case 256: 1271 break; 1272 default: 1273 return (false); 1274 } 1275 if (csp->csp_auth_key == NULL) 1276 return (false); 1277 if (csp->csp_ivlen != AES_GCM_IV_LEN) 1278 return (false); 1279 break; 1280 case CRYPTO_POLY1305: 1281 if (csp->csp_auth_klen != POLY1305_KEY_LEN) 1282 return (false); 1283 break; 1284 case CRYPTO_AES_CCM_CBC_MAC: 1285 switch (csp->csp_auth_klen * 8) { 1286 case 128: 1287 case 192: 1288 case 256: 1289 break; 1290 default: 1291 return (false); 1292 } 1293 if (csp->csp_auth_key == NULL) 1294 return (false); 1295 break; 1296 } 1297 return (true); 1298 } 1299 1300 static bool 1301 swcr_cipher_supported(const struct crypto_session_params *csp) 1302 { 1303 const struct enc_xform *txf; 1304 1305 txf = crypto_cipher(csp); 1306 if (txf == NULL) 1307 return (false); 1308 if (csp->csp_cipher_alg != CRYPTO_NULL_CBC && 1309 txf->ivsize != csp->csp_ivlen) 1310 return (false); 1311 return (true); 1312 } 1313 1314 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD | CSP_F_ESN) 1315 1316 static int 1317 swcr_probesession(device_t dev, const struct crypto_session_params *csp) 1318 { 1319 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0) 1320 return (EINVAL); 1321 switch (csp->csp_mode) { 1322 case CSP_MODE_COMPRESS: 1323 switch (csp->csp_cipher_alg) { 1324 case CRYPTO_DEFLATE_COMP: 1325 break; 1326 default: 1327 return (EINVAL); 1328 } 1329 break; 1330 case CSP_MODE_CIPHER: 1331 switch (csp->csp_cipher_alg) { 1332 case CRYPTO_AES_NIST_GCM_16: 1333 case CRYPTO_AES_CCM_16: 1334 case CRYPTO_CHACHA20_POLY1305: 1335 return (EINVAL); 1336 default: 1337 if (!swcr_cipher_supported(csp)) 1338 return (EINVAL); 1339 break; 1340 } 1341 break; 1342 case CSP_MODE_DIGEST: 1343 if (!swcr_auth_supported(csp)) 1344 return (EINVAL); 1345 break; 1346 case CSP_MODE_AEAD: 1347 switch (csp->csp_cipher_alg) { 1348 case CRYPTO_AES_NIST_GCM_16: 1349 case CRYPTO_AES_CCM_16: 1350 switch (csp->csp_cipher_klen * 8) { 1351 case 128: 1352 case 192: 1353 case 256: 1354 break; 1355 default: 1356 return (EINVAL); 1357 } 1358 break; 1359 case CRYPTO_CHACHA20_POLY1305: 1360 break; 1361 default: 1362 return (EINVAL); 1363 } 1364 break; 1365 case CSP_MODE_ETA: 1366 /* AEAD algorithms cannot be used for EtA. */ 1367 switch (csp->csp_cipher_alg) { 1368 case CRYPTO_AES_NIST_GCM_16: 1369 case CRYPTO_AES_CCM_16: 1370 case CRYPTO_CHACHA20_POLY1305: 1371 return (EINVAL); 1372 } 1373 switch (csp->csp_auth_alg) { 1374 case CRYPTO_AES_NIST_GMAC: 1375 case CRYPTO_AES_CCM_CBC_MAC: 1376 return (EINVAL); 1377 } 1378 1379 if (!swcr_cipher_supported(csp) || 1380 !swcr_auth_supported(csp)) 1381 return (EINVAL); 1382 break; 1383 default: 1384 return (EINVAL); 1385 } 1386 1387 return (CRYPTODEV_PROBE_SOFTWARE); 1388 } 1389 1390 /* 1391 * Generate a new software session. 1392 */ 1393 static int 1394 swcr_newsession(device_t dev, crypto_session_t cses, 1395 const struct crypto_session_params *csp) 1396 { 1397 struct swcr_session *ses; 1398 const struct comp_algo *cxf; 1399 int error; 1400 1401 ses = crypto_get_driver_session(cses); 1402 1403 error = 0; 1404 switch (csp->csp_mode) { 1405 case CSP_MODE_COMPRESS: 1406 switch (csp->csp_cipher_alg) { 1407 case CRYPTO_DEFLATE_COMP: 1408 cxf = &comp_algo_deflate; 1409 break; 1410 #ifdef INVARIANTS 1411 default: 1412 panic("bad compression algo"); 1413 #endif 1414 } 1415 ses->swcr_compdec.sw_cxf = cxf; 1416 ses->swcr_process = swcr_compdec; 1417 break; 1418 case CSP_MODE_CIPHER: 1419 switch (csp->csp_cipher_alg) { 1420 case CRYPTO_NULL_CBC: 1421 ses->swcr_process = swcr_null; 1422 break; 1423 #ifdef INVARIANTS 1424 case CRYPTO_AES_NIST_GCM_16: 1425 case CRYPTO_AES_CCM_16: 1426 case CRYPTO_CHACHA20_POLY1305: 1427 panic("bad cipher algo"); 1428 #endif 1429 default: 1430 error = swcr_setup_cipher(ses, csp); 1431 if (error == 0) 1432 ses->swcr_process = swcr_encdec; 1433 } 1434 break; 1435 case CSP_MODE_DIGEST: 1436 error = swcr_setup_auth(ses, csp); 1437 break; 1438 case CSP_MODE_AEAD: 1439 switch (csp->csp_cipher_alg) { 1440 case CRYPTO_AES_NIST_GCM_16: 1441 error = swcr_setup_aead(ses, csp); 1442 if (error == 0) 1443 ses->swcr_process = swcr_gcm; 1444 break; 1445 case CRYPTO_AES_CCM_16: 1446 error = swcr_setup_aead(ses, csp); 1447 if (error == 0) 1448 ses->swcr_process = swcr_ccm; 1449 break; 1450 case CRYPTO_CHACHA20_POLY1305: 1451 error = swcr_setup_aead(ses, csp); 1452 if (error == 0) 1453 ses->swcr_process = swcr_chacha20_poly1305; 1454 break; 1455 #ifdef INVARIANTS 1456 default: 1457 panic("bad aead algo"); 1458 #endif 1459 } 1460 break; 1461 case CSP_MODE_ETA: 1462 #ifdef INVARIANTS 1463 switch (csp->csp_cipher_alg) { 1464 case CRYPTO_AES_NIST_GCM_16: 1465 case CRYPTO_AES_CCM_16: 1466 case CRYPTO_CHACHA20_POLY1305: 1467 panic("bad eta cipher algo"); 1468 } 1469 switch (csp->csp_auth_alg) { 1470 case CRYPTO_AES_NIST_GMAC: 1471 case CRYPTO_AES_CCM_CBC_MAC: 1472 panic("bad eta auth algo"); 1473 } 1474 #endif 1475 1476 error = swcr_setup_auth(ses, csp); 1477 if (error) 1478 break; 1479 if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) { 1480 /* Effectively degrade to digest mode. */ 1481 ses->swcr_process = swcr_authcompute; 1482 break; 1483 } 1484 1485 error = swcr_setup_cipher(ses, csp); 1486 if (error == 0) 1487 ses->swcr_process = swcr_eta; 1488 break; 1489 default: 1490 error = EINVAL; 1491 } 1492 1493 if (error) 1494 swcr_freesession(dev, cses); 1495 return (error); 1496 } 1497 1498 static void 1499 swcr_freesession(device_t dev, crypto_session_t cses) 1500 { 1501 struct swcr_session *ses; 1502 1503 ses = crypto_get_driver_session(cses); 1504 1505 zfree(ses->swcr_encdec.sw_ctx, M_CRYPTO_DATA); 1506 zfree(ses->swcr_auth.sw_ictx, M_CRYPTO_DATA); 1507 zfree(ses->swcr_auth.sw_octx, M_CRYPTO_DATA); 1508 } 1509 1510 /* 1511 * Process a software request. 1512 */ 1513 static int 1514 swcr_process(device_t dev, struct cryptop *crp, int hint) 1515 { 1516 struct swcr_session *ses; 1517 1518 ses = crypto_get_driver_session(crp->crp_session); 1519 1520 crp->crp_etype = ses->swcr_process(ses, crp); 1521 1522 crypto_done(crp); 1523 return (0); 1524 } 1525 1526 static void 1527 swcr_identify(driver_t *drv, device_t parent) 1528 { 1529 /* NB: order 10 is so we get attached after h/w devices */ 1530 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1531 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1532 panic("cryptosoft: could not attach"); 1533 } 1534 1535 static int 1536 swcr_probe(device_t dev) 1537 { 1538 device_set_desc(dev, "software crypto"); 1539 device_quiet(dev); 1540 return (BUS_PROBE_NOWILDCARD); 1541 } 1542 1543 static int 1544 swcr_attach(device_t dev) 1545 { 1546 1547 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session), 1548 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1549 if (swcr_id < 0) { 1550 device_printf(dev, "cannot initialize!"); 1551 return (ENXIO); 1552 } 1553 1554 return (0); 1555 } 1556 1557 static int 1558 swcr_detach(device_t dev) 1559 { 1560 crypto_unregister_all(swcr_id); 1561 return 0; 1562 } 1563 1564 static device_method_t swcr_methods[] = { 1565 DEVMETHOD(device_identify, swcr_identify), 1566 DEVMETHOD(device_probe, swcr_probe), 1567 DEVMETHOD(device_attach, swcr_attach), 1568 DEVMETHOD(device_detach, swcr_detach), 1569 1570 DEVMETHOD(cryptodev_probesession, swcr_probesession), 1571 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1572 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1573 DEVMETHOD(cryptodev_process, swcr_process), 1574 1575 {0, 0}, 1576 }; 1577 1578 static driver_t swcr_driver = { 1579 "cryptosoft", 1580 swcr_methods, 1581 0, /* NB: no softc */ 1582 }; 1583 static devclass_t swcr_devclass; 1584 1585 /* 1586 * NB: We explicitly reference the crypto module so we 1587 * get the necessary ordering when built as a loadable 1588 * module. This is required because we bundle the crypto 1589 * module code together with the cryptosoft driver (otherwise 1590 * normal module dependencies would handle things). 1591 */ 1592 extern int crypto_modevent(struct module *, int, void *); 1593 /* XXX where to attach */ 1594 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1595 MODULE_VERSION(cryptosoft, 1); 1596 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1597