1 /* $OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $ */ 2 3 /*- 4 * The author of this code is Angelos D. Keromytis ([email protected]) 5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting 6 * 7 * This code was written by Angelos D. Keromytis in Athens, Greece, in 8 * February 2000. Network Security Technologies Inc. (NSTI) kindly 9 * supported the development of this code. 10 * 11 * Copyright (c) 2000, 2001 Angelos D. Keromytis 12 * Copyright (c) 2014 The FreeBSD Foundation 13 * All rights reserved. 14 * 15 * Portions of this software were developed by John-Mark Gurney 16 * under sponsorship of the FreeBSD Foundation and 17 * Rubicon Communications, LLC (Netgate). 18 * 19 * Permission to use, copy, and modify this software with or without fee 20 * is hereby granted, provided that this entire notice is included in 21 * all source code copies of any software which is or includes a copy or 22 * modification of this software. 23 * 24 * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR 25 * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY 26 * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE 27 * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR 28 * PURPOSE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/malloc.h> 37 #include <sys/mbuf.h> 38 #include <sys/module.h> 39 #include <sys/sysctl.h> 40 #include <sys/errno.h> 41 #include <sys/random.h> 42 #include <sys/kernel.h> 43 #include <sys/uio.h> 44 #include <sys/lock.h> 45 #include <sys/rwlock.h> 46 #include <sys/endian.h> 47 #include <sys/limits.h> 48 49 #include <crypto/blowfish/blowfish.h> 50 #include <crypto/sha1.h> 51 #include <opencrypto/rmd160.h> 52 #include <opencrypto/cast.h> 53 #include <opencrypto/skipjack.h> 54 #include <sys/md5.h> 55 56 #include <opencrypto/cryptodev.h> 57 #include <opencrypto/cryptosoft.h> 58 #include <opencrypto/xform.h> 59 60 #include <sys/kobj.h> 61 #include <sys/bus.h> 62 #include "cryptodev_if.h" 63 64 static int32_t swcr_id; 65 66 u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN]; 67 u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN]; 68 69 static int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); 70 static int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int); 71 static int swcr_authenc(struct cryptop *crp); 72 static int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int); 73 static void swcr_freesession(device_t dev, crypto_session_t cses); 74 75 /* 76 * Apply a symmetric encryption/decryption algorithm. 77 */ 78 static int 79 swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, 80 int flags) 81 { 82 unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN]; 83 unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN]; 84 struct enc_xform *exf; 85 int i, j, k, blks, ind, count, ivlen; 86 struct uio *uio, uiolcl; 87 struct iovec iovlcl[4]; 88 struct iovec *iov; 89 int iovcnt, iovalloc; 90 int error; 91 92 error = 0; 93 94 exf = sw->sw_exf; 95 blks = exf->blocksize; 96 ivlen = exf->ivsize; 97 98 /* Check for non-padded data */ 99 if (crd->crd_len % blks) 100 return EINVAL; 101 102 if (crd->crd_alg == CRYPTO_AES_ICM && 103 (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0) 104 return (EINVAL); 105 106 /* Initialize the IV */ 107 if (crd->crd_flags & CRD_F_ENCRYPT) { 108 /* IV explicitly provided ? */ 109 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 110 bcopy(crd->crd_iv, iv, ivlen); 111 else 112 arc4rand(iv, ivlen, 0); 113 114 /* Do we need to write the IV */ 115 if (!(crd->crd_flags & CRD_F_IV_PRESENT)) 116 crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv); 117 118 } else { /* Decryption */ 119 /* IV explicitly provided ? */ 120 if (crd->crd_flags & CRD_F_IV_EXPLICIT) 121 bcopy(crd->crd_iv, iv, ivlen); 122 else { 123 /* Get IV off buf */ 124 crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv); 125 } 126 } 127 128 if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { 129 int error; 130 131 if (sw->sw_kschedule) 132 exf->zerokey(&(sw->sw_kschedule)); 133 134 error = exf->setkey(&sw->sw_kschedule, 135 crd->crd_key, crd->crd_klen / 8); 136 if (error) 137 return (error); 138 } 139 140 iov = iovlcl; 141 iovcnt = nitems(iovlcl); 142 iovalloc = 0; 143 uio = &uiolcl; 144 if ((flags & CRYPTO_F_IMBUF) != 0) { 145 error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt, 146 &iovalloc); 147 if (error) 148 return (error); 149 uio->uio_iov = iov; 150 uio->uio_iovcnt = iovcnt; 151 } else if ((flags & CRYPTO_F_IOV) != 0) 152 uio = (struct uio *)buf; 153 else { 154 iov[0].iov_base = buf; 155 iov[0].iov_len = crd->crd_skip + crd->crd_len; 156 uio->uio_iov = iov; 157 uio->uio_iovcnt = 1; 158 } 159 160 ivp = iv; 161 162 if (exf->reinit) { 163 /* 164 * xforms that provide a reinit method perform all IV 165 * handling themselves. 166 */ 167 exf->reinit(sw->sw_kschedule, iv); 168 } 169 170 count = crd->crd_skip; 171 ind = cuio_getptr(uio, count, &k); 172 if (ind == -1) { 173 error = EINVAL; 174 goto out; 175 } 176 177 i = crd->crd_len; 178 179 while (i > 0) { 180 /* 181 * If there's insufficient data at the end of 182 * an iovec, we have to do some copying. 183 */ 184 if (uio->uio_iov[ind].iov_len < k + blks && 185 uio->uio_iov[ind].iov_len != k) { 186 cuio_copydata(uio, count, blks, blk); 187 188 /* Actual encryption/decryption */ 189 if (exf->reinit) { 190 if (crd->crd_flags & CRD_F_ENCRYPT) { 191 exf->encrypt(sw->sw_kschedule, 192 blk); 193 } else { 194 exf->decrypt(sw->sw_kschedule, 195 blk); 196 } 197 } else if (crd->crd_flags & CRD_F_ENCRYPT) { 198 /* XOR with previous block */ 199 for (j = 0; j < blks; j++) 200 blk[j] ^= ivp[j]; 201 202 exf->encrypt(sw->sw_kschedule, blk); 203 204 /* 205 * Keep encrypted block for XOR'ing 206 * with next block 207 */ 208 bcopy(blk, iv, blks); 209 ivp = iv; 210 } else { /* decrypt */ 211 /* 212 * Keep encrypted block for XOR'ing 213 * with next block 214 */ 215 nivp = (ivp == iv) ? iv2 : iv; 216 bcopy(blk, nivp, blks); 217 218 exf->decrypt(sw->sw_kschedule, blk); 219 220 /* XOR with previous block */ 221 for (j = 0; j < blks; j++) 222 blk[j] ^= ivp[j]; 223 224 ivp = nivp; 225 } 226 227 /* Copy back decrypted block */ 228 cuio_copyback(uio, count, blks, blk); 229 230 count += blks; 231 232 /* Advance pointer */ 233 ind = cuio_getptr(uio, count, &k); 234 if (ind == -1) { 235 error = EINVAL; 236 goto out; 237 } 238 239 i -= blks; 240 241 /* Could be done... */ 242 if (i == 0) 243 break; 244 } 245 246 while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) { 247 uint8_t *idat; 248 size_t nb, rem; 249 250 nb = blks; 251 rem = MIN((size_t)i, 252 uio->uio_iov[ind].iov_len - (size_t)k); 253 idat = (uint8_t *)uio->uio_iov[ind].iov_base + k; 254 255 if (exf->reinit) { 256 if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 && 257 exf->encrypt_multi == NULL) 258 exf->encrypt(sw->sw_kschedule, 259 idat); 260 else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) { 261 nb = rounddown(rem, blks); 262 exf->encrypt_multi(sw->sw_kschedule, 263 idat, nb); 264 } else if (exf->decrypt_multi == NULL) 265 exf->decrypt(sw->sw_kschedule, 266 idat); 267 else { 268 nb = rounddown(rem, blks); 269 exf->decrypt_multi(sw->sw_kschedule, 270 idat, nb); 271 } 272 } else if (crd->crd_flags & CRD_F_ENCRYPT) { 273 /* XOR with previous block/IV */ 274 for (j = 0; j < blks; j++) 275 idat[j] ^= ivp[j]; 276 277 exf->encrypt(sw->sw_kschedule, idat); 278 ivp = idat; 279 } else { /* decrypt */ 280 /* 281 * Keep encrypted block to be used 282 * in next block's processing. 283 */ 284 nivp = (ivp == iv) ? iv2 : iv; 285 bcopy(idat, nivp, blks); 286 287 exf->decrypt(sw->sw_kschedule, idat); 288 289 /* XOR with previous block/IV */ 290 for (j = 0; j < blks; j++) 291 idat[j] ^= ivp[j]; 292 293 ivp = nivp; 294 } 295 296 count += nb; 297 k += nb; 298 i -= nb; 299 } 300 301 /* 302 * Advance to the next iov if the end of the current iov 303 * is aligned with the end of a cipher block. 304 * Note that the code is equivalent to calling: 305 * ind = cuio_getptr(uio, count, &k); 306 */ 307 if (i > 0 && k == uio->uio_iov[ind].iov_len) { 308 k = 0; 309 ind++; 310 if (ind >= uio->uio_iovcnt) { 311 error = EINVAL; 312 goto out; 313 } 314 } 315 } 316 317 out: 318 if (iovalloc) 319 free(iov, M_CRYPTO_DATA); 320 321 return (error); 322 } 323 324 static int __result_use_check 325 swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key, 326 int klen) 327 { 328 int k; 329 330 klen /= 8; 331 332 switch (axf->type) { 333 case CRYPTO_MD5_HMAC: 334 case CRYPTO_SHA1_HMAC: 335 case CRYPTO_SHA2_224_HMAC: 336 case CRYPTO_SHA2_256_HMAC: 337 case CRYPTO_SHA2_384_HMAC: 338 case CRYPTO_SHA2_512_HMAC: 339 case CRYPTO_NULL_HMAC: 340 case CRYPTO_RIPEMD160_HMAC: 341 for (k = 0; k < klen; k++) 342 key[k] ^= HMAC_IPAD_VAL; 343 344 axf->Init(sw->sw_ictx); 345 axf->Update(sw->sw_ictx, key, klen); 346 axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen); 347 348 for (k = 0; k < klen; k++) 349 key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL); 350 351 axf->Init(sw->sw_octx); 352 axf->Update(sw->sw_octx, key, klen); 353 axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen); 354 355 for (k = 0; k < klen; k++) 356 key[k] ^= HMAC_OPAD_VAL; 357 break; 358 case CRYPTO_MD5_KPDK: 359 case CRYPTO_SHA1_KPDK: 360 { 361 /* 362 * We need a buffer that can hold an md5 and a sha1 result 363 * just to throw it away. 364 * What we do here is the initial part of: 365 * ALGO( key, keyfill, .. ) 366 * adding the key to sw_ictx and abusing Final() to get the 367 * "keyfill" padding. 368 * In addition we abuse the sw_octx to save the key to have 369 * it to be able to append it at the end in swcr_authcompute(). 370 */ 371 u_char buf[SHA1_RESULTLEN]; 372 373 sw->sw_klen = klen; 374 bcopy(key, sw->sw_octx, klen); 375 axf->Init(sw->sw_ictx); 376 axf->Update(sw->sw_ictx, key, klen); 377 axf->Final(buf, sw->sw_ictx); 378 break; 379 } 380 case CRYPTO_POLY1305: 381 if (klen != POLY1305_KEY_LEN) { 382 CRYPTDEB("bad poly1305 key size %d", klen); 383 return EINVAL; 384 } 385 /* FALLTHROUGH */ 386 case CRYPTO_BLAKE2B: 387 case CRYPTO_BLAKE2S: 388 axf->Setkey(sw->sw_ictx, key, klen); 389 axf->Init(sw->sw_ictx); 390 break; 391 default: 392 printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d " 393 "doesn't use keys.\n", __func__, axf->type); 394 return EINVAL; 395 } 396 return 0; 397 } 398 399 /* 400 * Compute keyed-hash authenticator. 401 */ 402 static int 403 swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf, 404 int flags) 405 { 406 unsigned char aalg[HASH_MAX_LEN]; 407 struct auth_hash *axf; 408 union authctx ctx; 409 int err; 410 411 if (sw->sw_ictx == 0) 412 return EINVAL; 413 414 axf = sw->sw_axf; 415 416 if (crd->crd_flags & CRD_F_KEY_EXPLICIT) { 417 err = swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen); 418 if (err != 0) 419 return err; 420 } 421 422 bcopy(sw->sw_ictx, &ctx, axf->ctxsize); 423 424 err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len, 425 (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx); 426 if (err) 427 return err; 428 429 switch (sw->sw_alg) { 430 case CRYPTO_SHA1: 431 case CRYPTO_SHA2_224: 432 case CRYPTO_SHA2_256: 433 case CRYPTO_SHA2_384: 434 case CRYPTO_SHA2_512: 435 axf->Final(aalg, &ctx); 436 break; 437 438 case CRYPTO_MD5_HMAC: 439 case CRYPTO_SHA1_HMAC: 440 case CRYPTO_SHA2_224_HMAC: 441 case CRYPTO_SHA2_256_HMAC: 442 case CRYPTO_SHA2_384_HMAC: 443 case CRYPTO_SHA2_512_HMAC: 444 case CRYPTO_RIPEMD160_HMAC: 445 if (sw->sw_octx == NULL) 446 return EINVAL; 447 448 axf->Final(aalg, &ctx); 449 bcopy(sw->sw_octx, &ctx, axf->ctxsize); 450 axf->Update(&ctx, aalg, axf->hashsize); 451 axf->Final(aalg, &ctx); 452 break; 453 454 case CRYPTO_MD5_KPDK: 455 case CRYPTO_SHA1_KPDK: 456 /* If we have no key saved, return error. */ 457 if (sw->sw_octx == NULL) 458 return EINVAL; 459 460 /* 461 * Add the trailing copy of the key (see comment in 462 * swcr_authprepare()) after the data: 463 * ALGO( .., key, algofill ) 464 * and let Final() do the proper, natural "algofill" 465 * padding. 466 */ 467 axf->Update(&ctx, sw->sw_octx, sw->sw_klen); 468 axf->Final(aalg, &ctx); 469 break; 470 471 case CRYPTO_BLAKE2B: 472 case CRYPTO_BLAKE2S: 473 case CRYPTO_NULL_HMAC: 474 case CRYPTO_POLY1305: 475 axf->Final(aalg, &ctx); 476 break; 477 } 478 479 /* Inject the authentication data */ 480 crypto_copyback(flags, buf, crd->crd_inject, 481 sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg); 482 return 0; 483 } 484 485 CTASSERT(INT_MAX <= (1ll<<39) - 256); /* GCM: plain text < 2^39-256 */ 486 CTASSERT(INT_MAX <= (uint64_t)-1); /* GCM: associated data <= 2^64-1 */ 487 488 /* 489 * Apply a combined encryption-authentication transformation 490 */ 491 static int 492 swcr_authenc(struct cryptop *crp) 493 { 494 uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))]; 495 u_char *blk = (u_char *)blkbuf; 496 u_char aalg[AALG_MAX_RESULT_LEN]; 497 u_char uaalg[AALG_MAX_RESULT_LEN]; 498 u_char iv[EALG_MAX_BLOCK_LEN]; 499 union authctx ctx; 500 struct cryptodesc *crd, *crda = NULL, *crde = NULL; 501 struct swcr_data *sw, *swa, *swe = NULL; 502 struct auth_hash *axf = NULL; 503 struct enc_xform *exf = NULL; 504 caddr_t buf = (caddr_t)crp->crp_buf; 505 uint32_t *blkp; 506 int aadlen, blksz, i, ivlen, len, iskip, oskip, r; 507 508 ivlen = blksz = iskip = oskip = 0; 509 510 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 511 for (sw = crypto_get_driver_session(crp->crp_session); 512 sw && sw->sw_alg != crd->crd_alg; 513 sw = sw->sw_next) 514 ; 515 if (sw == NULL) 516 return (EINVAL); 517 518 switch (sw->sw_alg) { 519 case CRYPTO_AES_NIST_GCM_16: 520 case CRYPTO_AES_NIST_GMAC: 521 swe = sw; 522 crde = crd; 523 exf = swe->sw_exf; 524 ivlen = 12; 525 break; 526 case CRYPTO_AES_128_NIST_GMAC: 527 case CRYPTO_AES_192_NIST_GMAC: 528 case CRYPTO_AES_256_NIST_GMAC: 529 swa = sw; 530 crda = crd; 531 axf = swa->sw_axf; 532 if (swa->sw_ictx == 0) 533 return (EINVAL); 534 bcopy(swa->sw_ictx, &ctx, axf->ctxsize); 535 blksz = axf->blocksize; 536 break; 537 default: 538 return (EINVAL); 539 } 540 } 541 if (crde == NULL || crda == NULL) 542 return (EINVAL); 543 544 if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16 && 545 (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0) 546 return (EINVAL); 547 548 if (crde->crd_klen != crda->crd_klen) 549 return (EINVAL); 550 551 /* Initialize the IV */ 552 if (crde->crd_flags & CRD_F_ENCRYPT) { 553 /* IV explicitly provided ? */ 554 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 555 bcopy(crde->crd_iv, iv, ivlen); 556 else 557 arc4rand(iv, ivlen, 0); 558 559 /* Do we need to write the IV */ 560 if (!(crde->crd_flags & CRD_F_IV_PRESENT)) 561 crypto_copyback(crp->crp_flags, buf, crde->crd_inject, 562 ivlen, iv); 563 564 } else { /* Decryption */ 565 /* IV explicitly provided ? */ 566 if (crde->crd_flags & CRD_F_IV_EXPLICIT) 567 bcopy(crde->crd_iv, iv, ivlen); 568 else { 569 /* Get IV off buf */ 570 crypto_copydata(crp->crp_flags, buf, crde->crd_inject, 571 ivlen, iv); 572 } 573 } 574 575 /* Supply MAC with IV */ 576 if (axf->Reinit) 577 axf->Reinit(&ctx, iv, ivlen); 578 579 /* Supply MAC with AAD */ 580 aadlen = crda->crd_len; 581 582 for (i = iskip; i < crda->crd_len; i += blksz) { 583 len = MIN(crda->crd_len - i, blksz - oskip); 584 crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len, 585 blk + oskip); 586 bzero(blk + len + oskip, blksz - len - oskip); 587 axf->Update(&ctx, blk, blksz); 588 oskip = 0; /* reset initial output offset */ 589 } 590 591 if (exf->reinit) 592 exf->reinit(swe->sw_kschedule, iv); 593 594 /* Do encryption/decryption with MAC */ 595 for (i = 0; i < crde->crd_len; i += len) { 596 if (exf->encrypt_multi != NULL) { 597 len = rounddown(crde->crd_len - i, blksz); 598 if (len == 0) 599 len = blksz; 600 else 601 len = MIN(len, sizeof(blkbuf)); 602 } else 603 len = blksz; 604 len = MIN(crde->crd_len - i, len); 605 if (len < blksz) 606 bzero(blk, blksz); 607 crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len, 608 blk); 609 if (crde->crd_flags & CRD_F_ENCRYPT) { 610 if (exf->encrypt_multi != NULL) 611 exf->encrypt_multi(swe->sw_kschedule, blk, 612 len); 613 else 614 exf->encrypt(swe->sw_kschedule, blk); 615 axf->Update(&ctx, blk, len); 616 crypto_copyback(crp->crp_flags, buf, 617 crde->crd_skip + i, len, blk); 618 } else { 619 axf->Update(&ctx, blk, len); 620 } 621 } 622 623 /* Do any required special finalization */ 624 switch (crda->crd_alg) { 625 case CRYPTO_AES_128_NIST_GMAC: 626 case CRYPTO_AES_192_NIST_GMAC: 627 case CRYPTO_AES_256_NIST_GMAC: 628 /* length block */ 629 bzero(blk, blksz); 630 blkp = (uint32_t *)blk + 1; 631 *blkp = htobe32(aadlen * 8); 632 blkp = (uint32_t *)blk + 3; 633 *blkp = htobe32(crde->crd_len * 8); 634 axf->Update(&ctx, blk, blksz); 635 break; 636 } 637 638 /* Finalize MAC */ 639 axf->Final(aalg, &ctx); 640 641 /* Validate tag */ 642 if (!(crde->crd_flags & CRD_F_ENCRYPT)) { 643 crypto_copydata(crp->crp_flags, buf, crda->crd_inject, 644 axf->hashsize, uaalg); 645 646 r = timingsafe_bcmp(aalg, uaalg, axf->hashsize); 647 if (r == 0) { 648 /* tag matches, decrypt data */ 649 for (i = 0; i < crde->crd_len; i += blksz) { 650 len = MIN(crde->crd_len - i, blksz); 651 if (len < blksz) 652 bzero(blk, blksz); 653 crypto_copydata(crp->crp_flags, buf, 654 crde->crd_skip + i, len, blk); 655 exf->decrypt(swe->sw_kschedule, blk); 656 crypto_copyback(crp->crp_flags, buf, 657 crde->crd_skip + i, len, blk); 658 } 659 } else 660 return (EBADMSG); 661 } else { 662 /* Inject the authentication data */ 663 crypto_copyback(crp->crp_flags, buf, crda->crd_inject, 664 axf->hashsize, aalg); 665 } 666 667 return (0); 668 } 669 670 /* 671 * Apply a compression/decompression algorithm 672 */ 673 static int 674 swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw, 675 caddr_t buf, int flags) 676 { 677 u_int8_t *data, *out; 678 struct comp_algo *cxf; 679 int adj; 680 u_int32_t result; 681 682 cxf = sw->sw_cxf; 683 684 /* We must handle the whole buffer of data in one time 685 * then if there is not all the data in the mbuf, we must 686 * copy in a buffer. 687 */ 688 689 data = malloc(crd->crd_len, M_CRYPTO_DATA, M_NOWAIT); 690 if (data == NULL) 691 return (EINVAL); 692 crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data); 693 694 if (crd->crd_flags & CRD_F_COMP) 695 result = cxf->compress(data, crd->crd_len, &out); 696 else 697 result = cxf->decompress(data, crd->crd_len, &out); 698 699 free(data, M_CRYPTO_DATA); 700 if (result == 0) 701 return EINVAL; 702 703 /* Copy back the (de)compressed data. m_copyback is 704 * extending the mbuf as necessary. 705 */ 706 sw->sw_size = result; 707 /* Check the compressed size when doing compression */ 708 if (crd->crd_flags & CRD_F_COMP) { 709 if (result >= crd->crd_len) { 710 /* Compression was useless, we lost time */ 711 free(out, M_CRYPTO_DATA); 712 return 0; 713 } 714 } 715 716 crypto_copyback(flags, buf, crd->crd_skip, result, out); 717 if (result < crd->crd_len) { 718 adj = result - crd->crd_len; 719 if (flags & CRYPTO_F_IMBUF) { 720 adj = result - crd->crd_len; 721 m_adj((struct mbuf *)buf, adj); 722 } else if (flags & CRYPTO_F_IOV) { 723 struct uio *uio = (struct uio *)buf; 724 int ind; 725 726 adj = crd->crd_len - result; 727 ind = uio->uio_iovcnt - 1; 728 729 while (adj > 0 && ind >= 0) { 730 if (adj < uio->uio_iov[ind].iov_len) { 731 uio->uio_iov[ind].iov_len -= adj; 732 break; 733 } 734 735 adj -= uio->uio_iov[ind].iov_len; 736 uio->uio_iov[ind].iov_len = 0; 737 ind--; 738 uio->uio_iovcnt--; 739 } 740 } 741 } 742 free(out, M_CRYPTO_DATA); 743 return 0; 744 } 745 746 /* 747 * Generate a new software session. 748 */ 749 static int 750 swcr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri) 751 { 752 struct swcr_data **swd, *ses; 753 struct auth_hash *axf; 754 struct enc_xform *txf; 755 struct comp_algo *cxf; 756 int len; 757 int error; 758 759 if (cses == NULL || cri == NULL) 760 return EINVAL; 761 762 ses = crypto_get_driver_session(cses); 763 swd = &ses; 764 765 while (cri) { 766 if (*swd == NULL) 767 *swd = malloc(sizeof(struct swcr_data), 768 M_CRYPTO_DATA, M_WAITOK | M_ZERO); 769 if (*swd == NULL) { 770 swcr_freesession(dev, cses); 771 return ENOBUFS; 772 } 773 774 switch (cri->cri_alg) { 775 case CRYPTO_DES_CBC: 776 txf = &enc_xform_des; 777 goto enccommon; 778 case CRYPTO_3DES_CBC: 779 txf = &enc_xform_3des; 780 goto enccommon; 781 case CRYPTO_BLF_CBC: 782 txf = &enc_xform_blf; 783 goto enccommon; 784 case CRYPTO_CAST_CBC: 785 txf = &enc_xform_cast5; 786 goto enccommon; 787 case CRYPTO_SKIPJACK_CBC: 788 txf = &enc_xform_skipjack; 789 goto enccommon; 790 case CRYPTO_RIJNDAEL128_CBC: 791 txf = &enc_xform_rijndael128; 792 goto enccommon; 793 case CRYPTO_AES_XTS: 794 txf = &enc_xform_aes_xts; 795 goto enccommon; 796 case CRYPTO_AES_ICM: 797 txf = &enc_xform_aes_icm; 798 goto enccommon; 799 case CRYPTO_AES_NIST_GCM_16: 800 txf = &enc_xform_aes_nist_gcm; 801 goto enccommon; 802 case CRYPTO_AES_NIST_GMAC: 803 txf = &enc_xform_aes_nist_gmac; 804 (*swd)->sw_exf = txf; 805 break; 806 case CRYPTO_CAMELLIA_CBC: 807 txf = &enc_xform_camellia; 808 goto enccommon; 809 case CRYPTO_NULL_CBC: 810 txf = &enc_xform_null; 811 goto enccommon; 812 case CRYPTO_CHACHA20: 813 txf = &enc_xform_chacha20; 814 goto enccommon; 815 enccommon: 816 if (cri->cri_key != NULL) { 817 error = txf->setkey(&((*swd)->sw_kschedule), 818 cri->cri_key, cri->cri_klen / 8); 819 if (error) { 820 swcr_freesession(dev, cses); 821 return error; 822 } 823 } 824 (*swd)->sw_exf = txf; 825 break; 826 827 case CRYPTO_MD5_HMAC: 828 axf = &auth_hash_hmac_md5; 829 goto authcommon; 830 case CRYPTO_SHA1_HMAC: 831 axf = &auth_hash_hmac_sha1; 832 goto authcommon; 833 case CRYPTO_SHA2_224_HMAC: 834 axf = &auth_hash_hmac_sha2_224; 835 goto authcommon; 836 case CRYPTO_SHA2_256_HMAC: 837 axf = &auth_hash_hmac_sha2_256; 838 goto authcommon; 839 case CRYPTO_SHA2_384_HMAC: 840 axf = &auth_hash_hmac_sha2_384; 841 goto authcommon; 842 case CRYPTO_SHA2_512_HMAC: 843 axf = &auth_hash_hmac_sha2_512; 844 goto authcommon; 845 case CRYPTO_NULL_HMAC: 846 axf = &auth_hash_null; 847 goto authcommon; 848 case CRYPTO_RIPEMD160_HMAC: 849 axf = &auth_hash_hmac_ripemd_160; 850 authcommon: 851 (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 852 M_NOWAIT); 853 if ((*swd)->sw_ictx == NULL) { 854 swcr_freesession(dev, cses); 855 return ENOBUFS; 856 } 857 858 (*swd)->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA, 859 M_NOWAIT); 860 if ((*swd)->sw_octx == NULL) { 861 swcr_freesession(dev, cses); 862 return ENOBUFS; 863 } 864 865 if (cri->cri_key != NULL) { 866 error = swcr_authprepare(axf, *swd, 867 cri->cri_key, cri->cri_klen); 868 if (error != 0) { 869 swcr_freesession(dev, cses); 870 return error; 871 } 872 } 873 874 (*swd)->sw_mlen = cri->cri_mlen; 875 (*swd)->sw_axf = axf; 876 break; 877 878 case CRYPTO_MD5_KPDK: 879 axf = &auth_hash_key_md5; 880 goto auth2common; 881 882 case CRYPTO_SHA1_KPDK: 883 axf = &auth_hash_key_sha1; 884 auth2common: 885 (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 886 M_NOWAIT); 887 if ((*swd)->sw_ictx == NULL) { 888 swcr_freesession(dev, cses); 889 return ENOBUFS; 890 } 891 892 (*swd)->sw_octx = malloc(cri->cri_klen / 8, 893 M_CRYPTO_DATA, M_NOWAIT); 894 if ((*swd)->sw_octx == NULL) { 895 swcr_freesession(dev, cses); 896 return ENOBUFS; 897 } 898 899 /* Store the key so we can "append" it to the payload */ 900 if (cri->cri_key != NULL) { 901 error = swcr_authprepare(axf, *swd, 902 cri->cri_key, cri->cri_klen); 903 if (error != 0) { 904 swcr_freesession(dev, cses); 905 return error; 906 } 907 } 908 909 (*swd)->sw_mlen = cri->cri_mlen; 910 (*swd)->sw_axf = axf; 911 break; 912 #ifdef notdef 913 case CRYPTO_MD5: 914 axf = &auth_hash_md5; 915 goto auth3common; 916 #endif 917 918 case CRYPTO_SHA1: 919 axf = &auth_hash_sha1; 920 goto auth3common; 921 case CRYPTO_SHA2_224: 922 axf = &auth_hash_sha2_224; 923 goto auth3common; 924 case CRYPTO_SHA2_256: 925 axf = &auth_hash_sha2_256; 926 goto auth3common; 927 case CRYPTO_SHA2_384: 928 axf = &auth_hash_sha2_384; 929 goto auth3common; 930 case CRYPTO_SHA2_512: 931 axf = &auth_hash_sha2_512; 932 933 auth3common: 934 (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 935 M_NOWAIT); 936 if ((*swd)->sw_ictx == NULL) { 937 swcr_freesession(dev, cses); 938 return ENOBUFS; 939 } 940 941 axf->Init((*swd)->sw_ictx); 942 (*swd)->sw_mlen = cri->cri_mlen; 943 (*swd)->sw_axf = axf; 944 break; 945 946 case CRYPTO_AES_128_NIST_GMAC: 947 axf = &auth_hash_nist_gmac_aes_128; 948 goto auth4common; 949 950 case CRYPTO_AES_192_NIST_GMAC: 951 axf = &auth_hash_nist_gmac_aes_192; 952 goto auth4common; 953 954 case CRYPTO_AES_256_NIST_GMAC: 955 axf = &auth_hash_nist_gmac_aes_256; 956 auth4common: 957 len = cri->cri_klen / 8; 958 if (len != 16 && len != 24 && len != 32) { 959 swcr_freesession(dev, cses); 960 return EINVAL; 961 } 962 963 (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 964 M_NOWAIT); 965 if ((*swd)->sw_ictx == NULL) { 966 swcr_freesession(dev, cses); 967 return ENOBUFS; 968 } 969 axf->Init((*swd)->sw_ictx); 970 axf->Setkey((*swd)->sw_ictx, cri->cri_key, len); 971 (*swd)->sw_axf = axf; 972 break; 973 974 case CRYPTO_BLAKE2B: 975 axf = &auth_hash_blake2b; 976 goto auth5common; 977 case CRYPTO_BLAKE2S: 978 axf = &auth_hash_blake2s; 979 goto auth5common; 980 case CRYPTO_POLY1305: 981 axf = &auth_hash_poly1305; 982 auth5common: 983 (*swd)->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, 984 M_NOWAIT); 985 if ((*swd)->sw_ictx == NULL) { 986 swcr_freesession(dev, cses); 987 return ENOBUFS; 988 } 989 axf->Setkey((*swd)->sw_ictx, cri->cri_key, 990 cri->cri_klen / 8); 991 axf->Init((*swd)->sw_ictx); 992 (*swd)->sw_axf = axf; 993 break; 994 995 case CRYPTO_DEFLATE_COMP: 996 cxf = &comp_algo_deflate; 997 (*swd)->sw_cxf = cxf; 998 break; 999 default: 1000 swcr_freesession(dev, cses); 1001 return EINVAL; 1002 } 1003 1004 (*swd)->sw_alg = cri->cri_alg; 1005 cri = cri->cri_next; 1006 swd = &((*swd)->sw_next); 1007 } 1008 return 0; 1009 } 1010 1011 static void 1012 swcr_freesession(device_t dev, crypto_session_t cses) 1013 { 1014 struct swcr_data *ses, *swd, *next; 1015 struct enc_xform *txf; 1016 struct auth_hash *axf; 1017 1018 ses = crypto_get_driver_session(cses); 1019 1020 for (swd = ses; swd != NULL; swd = next) { 1021 next = swd->sw_next; 1022 1023 switch (swd->sw_alg) { 1024 case CRYPTO_DES_CBC: 1025 case CRYPTO_3DES_CBC: 1026 case CRYPTO_BLF_CBC: 1027 case CRYPTO_CAST_CBC: 1028 case CRYPTO_SKIPJACK_CBC: 1029 case CRYPTO_RIJNDAEL128_CBC: 1030 case CRYPTO_AES_XTS: 1031 case CRYPTO_AES_ICM: 1032 case CRYPTO_AES_NIST_GCM_16: 1033 case CRYPTO_AES_NIST_GMAC: 1034 case CRYPTO_CAMELLIA_CBC: 1035 case CRYPTO_NULL_CBC: 1036 case CRYPTO_CHACHA20: 1037 txf = swd->sw_exf; 1038 1039 if (swd->sw_kschedule) 1040 txf->zerokey(&(swd->sw_kschedule)); 1041 break; 1042 1043 case CRYPTO_MD5_HMAC: 1044 case CRYPTO_SHA1_HMAC: 1045 case CRYPTO_SHA2_224_HMAC: 1046 case CRYPTO_SHA2_256_HMAC: 1047 case CRYPTO_SHA2_384_HMAC: 1048 case CRYPTO_SHA2_512_HMAC: 1049 case CRYPTO_RIPEMD160_HMAC: 1050 case CRYPTO_NULL_HMAC: 1051 axf = swd->sw_axf; 1052 1053 if (swd->sw_ictx) { 1054 bzero(swd->sw_ictx, axf->ctxsize); 1055 free(swd->sw_ictx, M_CRYPTO_DATA); 1056 } 1057 if (swd->sw_octx) { 1058 bzero(swd->sw_octx, axf->ctxsize); 1059 free(swd->sw_octx, M_CRYPTO_DATA); 1060 } 1061 break; 1062 1063 case CRYPTO_MD5_KPDK: 1064 case CRYPTO_SHA1_KPDK: 1065 axf = swd->sw_axf; 1066 1067 if (swd->sw_ictx) { 1068 bzero(swd->sw_ictx, axf->ctxsize); 1069 free(swd->sw_ictx, M_CRYPTO_DATA); 1070 } 1071 if (swd->sw_octx) { 1072 bzero(swd->sw_octx, swd->sw_klen); 1073 free(swd->sw_octx, M_CRYPTO_DATA); 1074 } 1075 break; 1076 1077 case CRYPTO_BLAKE2B: 1078 case CRYPTO_BLAKE2S: 1079 case CRYPTO_MD5: 1080 case CRYPTO_POLY1305: 1081 case CRYPTO_SHA1: 1082 case CRYPTO_SHA2_224: 1083 case CRYPTO_SHA2_256: 1084 case CRYPTO_SHA2_384: 1085 case CRYPTO_SHA2_512: 1086 axf = swd->sw_axf; 1087 1088 if (swd->sw_ictx) { 1089 explicit_bzero(swd->sw_ictx, axf->ctxsize); 1090 free(swd->sw_ictx, M_CRYPTO_DATA); 1091 } 1092 break; 1093 1094 case CRYPTO_DEFLATE_COMP: 1095 /* Nothing to do */ 1096 break; 1097 } 1098 1099 /* OCF owns and frees the primary session object */ 1100 if (swd != ses) 1101 free(swd, M_CRYPTO_DATA); 1102 } 1103 } 1104 1105 /* 1106 * Process a software request. 1107 */ 1108 static int 1109 swcr_process(device_t dev, struct cryptop *crp, int hint) 1110 { 1111 struct cryptodesc *crd; 1112 struct swcr_data *sw, *ses; 1113 1114 /* Sanity check */ 1115 if (crp == NULL) 1116 return EINVAL; 1117 1118 if (crp->crp_desc == NULL || crp->crp_buf == NULL) { 1119 crp->crp_etype = EINVAL; 1120 goto done; 1121 } 1122 1123 ses = crypto_get_driver_session(crp->crp_session); 1124 1125 /* Go through crypto descriptors, processing as we go */ 1126 for (crd = crp->crp_desc; crd; crd = crd->crd_next) { 1127 /* 1128 * Find the crypto context. 1129 * 1130 * XXX Note that the logic here prevents us from having 1131 * XXX the same algorithm multiple times in a session 1132 * XXX (or rather, we can but it won't give us the right 1133 * XXX results). To do that, we'd need some way of differentiating 1134 * XXX between the various instances of an algorithm (so we can 1135 * XXX locate the correct crypto context). 1136 */ 1137 for (sw = ses; sw && sw->sw_alg != crd->crd_alg; 1138 sw = sw->sw_next) 1139 ; 1140 1141 /* No such context ? */ 1142 if (sw == NULL) { 1143 crp->crp_etype = EINVAL; 1144 goto done; 1145 } 1146 switch (sw->sw_alg) { 1147 case CRYPTO_DES_CBC: 1148 case CRYPTO_3DES_CBC: 1149 case CRYPTO_BLF_CBC: 1150 case CRYPTO_CAST_CBC: 1151 case CRYPTO_SKIPJACK_CBC: 1152 case CRYPTO_RIJNDAEL128_CBC: 1153 case CRYPTO_AES_XTS: 1154 case CRYPTO_AES_ICM: 1155 case CRYPTO_CAMELLIA_CBC: 1156 case CRYPTO_CHACHA20: 1157 if ((crp->crp_etype = swcr_encdec(crd, sw, 1158 crp->crp_buf, crp->crp_flags)) != 0) 1159 goto done; 1160 break; 1161 case CRYPTO_NULL_CBC: 1162 crp->crp_etype = 0; 1163 break; 1164 case CRYPTO_MD5_HMAC: 1165 case CRYPTO_SHA1_HMAC: 1166 case CRYPTO_SHA2_224_HMAC: 1167 case CRYPTO_SHA2_256_HMAC: 1168 case CRYPTO_SHA2_384_HMAC: 1169 case CRYPTO_SHA2_512_HMAC: 1170 case CRYPTO_RIPEMD160_HMAC: 1171 case CRYPTO_NULL_HMAC: 1172 case CRYPTO_MD5_KPDK: 1173 case CRYPTO_SHA1_KPDK: 1174 case CRYPTO_MD5: 1175 case CRYPTO_SHA1: 1176 case CRYPTO_SHA2_224: 1177 case CRYPTO_SHA2_256: 1178 case CRYPTO_SHA2_384: 1179 case CRYPTO_SHA2_512: 1180 case CRYPTO_BLAKE2B: 1181 case CRYPTO_BLAKE2S: 1182 case CRYPTO_POLY1305: 1183 if ((crp->crp_etype = swcr_authcompute(crd, sw, 1184 crp->crp_buf, crp->crp_flags)) != 0) 1185 goto done; 1186 break; 1187 1188 case CRYPTO_AES_NIST_GCM_16: 1189 case CRYPTO_AES_NIST_GMAC: 1190 case CRYPTO_AES_128_NIST_GMAC: 1191 case CRYPTO_AES_192_NIST_GMAC: 1192 case CRYPTO_AES_256_NIST_GMAC: 1193 crp->crp_etype = swcr_authenc(crp); 1194 goto done; 1195 1196 case CRYPTO_DEFLATE_COMP: 1197 if ((crp->crp_etype = swcr_compdec(crd, sw, 1198 crp->crp_buf, crp->crp_flags)) != 0) 1199 goto done; 1200 else 1201 crp->crp_olen = (int)sw->sw_size; 1202 break; 1203 1204 default: 1205 /* Unknown/unsupported algorithm */ 1206 crp->crp_etype = EINVAL; 1207 goto done; 1208 } 1209 } 1210 1211 done: 1212 crypto_done(crp); 1213 return 0; 1214 } 1215 1216 static void 1217 swcr_identify(driver_t *drv, device_t parent) 1218 { 1219 /* NB: order 10 is so we get attached after h/w devices */ 1220 if (device_find_child(parent, "cryptosoft", -1) == NULL && 1221 BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0) 1222 panic("cryptosoft: could not attach"); 1223 } 1224 1225 static int 1226 swcr_probe(device_t dev) 1227 { 1228 device_set_desc(dev, "software crypto"); 1229 return (BUS_PROBE_NOWILDCARD); 1230 } 1231 1232 static int 1233 swcr_attach(device_t dev) 1234 { 1235 memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN); 1236 memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN); 1237 1238 swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_data), 1239 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC); 1240 if (swcr_id < 0) { 1241 device_printf(dev, "cannot initialize!"); 1242 return ENOMEM; 1243 } 1244 #define REGISTER(alg) \ 1245 crypto_register(swcr_id, alg, 0,0) 1246 REGISTER(CRYPTO_DES_CBC); 1247 REGISTER(CRYPTO_3DES_CBC); 1248 REGISTER(CRYPTO_BLF_CBC); 1249 REGISTER(CRYPTO_CAST_CBC); 1250 REGISTER(CRYPTO_SKIPJACK_CBC); 1251 REGISTER(CRYPTO_NULL_CBC); 1252 REGISTER(CRYPTO_MD5_HMAC); 1253 REGISTER(CRYPTO_SHA1_HMAC); 1254 REGISTER(CRYPTO_SHA2_224_HMAC); 1255 REGISTER(CRYPTO_SHA2_256_HMAC); 1256 REGISTER(CRYPTO_SHA2_384_HMAC); 1257 REGISTER(CRYPTO_SHA2_512_HMAC); 1258 REGISTER(CRYPTO_RIPEMD160_HMAC); 1259 REGISTER(CRYPTO_NULL_HMAC); 1260 REGISTER(CRYPTO_MD5_KPDK); 1261 REGISTER(CRYPTO_SHA1_KPDK); 1262 REGISTER(CRYPTO_MD5); 1263 REGISTER(CRYPTO_SHA1); 1264 REGISTER(CRYPTO_SHA2_224); 1265 REGISTER(CRYPTO_SHA2_256); 1266 REGISTER(CRYPTO_SHA2_384); 1267 REGISTER(CRYPTO_SHA2_512); 1268 REGISTER(CRYPTO_RIJNDAEL128_CBC); 1269 REGISTER(CRYPTO_AES_XTS); 1270 REGISTER(CRYPTO_AES_ICM); 1271 REGISTER(CRYPTO_AES_NIST_GCM_16); 1272 REGISTER(CRYPTO_AES_NIST_GMAC); 1273 REGISTER(CRYPTO_AES_128_NIST_GMAC); 1274 REGISTER(CRYPTO_AES_192_NIST_GMAC); 1275 REGISTER(CRYPTO_AES_256_NIST_GMAC); 1276 REGISTER(CRYPTO_CAMELLIA_CBC); 1277 REGISTER(CRYPTO_DEFLATE_COMP); 1278 REGISTER(CRYPTO_BLAKE2B); 1279 REGISTER(CRYPTO_BLAKE2S); 1280 REGISTER(CRYPTO_CHACHA20); 1281 REGISTER(CRYPTO_POLY1305); 1282 #undef REGISTER 1283 1284 return 0; 1285 } 1286 1287 static int 1288 swcr_detach(device_t dev) 1289 { 1290 crypto_unregister_all(swcr_id); 1291 return 0; 1292 } 1293 1294 static device_method_t swcr_methods[] = { 1295 DEVMETHOD(device_identify, swcr_identify), 1296 DEVMETHOD(device_probe, swcr_probe), 1297 DEVMETHOD(device_attach, swcr_attach), 1298 DEVMETHOD(device_detach, swcr_detach), 1299 1300 DEVMETHOD(cryptodev_newsession, swcr_newsession), 1301 DEVMETHOD(cryptodev_freesession,swcr_freesession), 1302 DEVMETHOD(cryptodev_process, swcr_process), 1303 1304 {0, 0}, 1305 }; 1306 1307 static driver_t swcr_driver = { 1308 "cryptosoft", 1309 swcr_methods, 1310 0, /* NB: no softc */ 1311 }; 1312 static devclass_t swcr_devclass; 1313 1314 /* 1315 * NB: We explicitly reference the crypto module so we 1316 * get the necessary ordering when built as a loadable 1317 * module. This is required because we bundle the crypto 1318 * module code together with the cryptosoft driver (otherwise 1319 * normal module dependencies would handle things). 1320 */ 1321 extern int crypto_modevent(struct module *, int, void *); 1322 /* XXX where to attach */ 1323 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0); 1324 MODULE_VERSION(cryptosoft, 1); 1325 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1); 1326