1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2003-2012 Broadcom Corporation
5 * All Rights Reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
25 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
27 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
28 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include <sys/cdefs.h>
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/errno.h>
39 #include <sys/malloc.h>
40 #include <sys/kernel.h>
41 #include <sys/module.h>
42 #include <sys/mbuf.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/sysctl.h>
46 #include <sys/bus.h>
47 #include <sys/random.h>
48 #include <sys/rman.h>
49 #include <sys/uio.h>
50 #include <sys/kobj.h>
51
52 #include <dev/pci/pcivar.h>
53
54 #include <opencrypto/cryptodev.h>
55 #include <opencrypto/xform_auth.h>
56
57 #include "cryptodev_if.h"
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <mips/nlm/hal/haldefs.h>
63 #include <mips/nlm/hal/iomap.h>
64 #include <mips/nlm/xlp.h>
65 #include <mips/nlm/hal/sys.h>
66 #include <mips/nlm/hal/fmn.h>
67 #include <mips/nlm/hal/nlmsaelib.h>
68 #include <mips/nlm/dev/sec/nlmseclib.h>
69 #include <mips/nlm/hal/cop2.h>
70 #include <mips/nlm/hal/mips-extns.h>
71 #include <mips/nlm/msgring.h>
72
73 unsigned int creditleft;
74
75 static int xlp_sec_init(struct xlp_sec_softc *sc);
76 static int xlp_sec_probesession(device_t,
77 const struct crypto_session_params *);
78 static int xlp_sec_newsession(device_t , crypto_session_t,
79 const struct crypto_session_params *);
80 static int xlp_sec_process(device_t , struct cryptop *, int);
81 static void xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
82 const struct crypto_session_params *);
83 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
84 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
85 static void xlp_free_cmd_params(struct xlp_sec_command *);
86
87 static int xlp_sec_probe(device_t);
88 static int xlp_sec_attach(device_t);
89 static int xlp_sec_detach(device_t);
90
91 static device_method_t xlp_sec_methods[] = {
92 /* device interface */
93 DEVMETHOD(device_probe, xlp_sec_probe),
94 DEVMETHOD(device_attach, xlp_sec_attach),
95 DEVMETHOD(device_detach, xlp_sec_detach),
96
97 /* bus interface */
98 DEVMETHOD(bus_print_child, bus_generic_print_child),
99 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
100
101 /* crypto device methods */
102 DEVMETHOD(cryptodev_probesession, xlp_sec_probesession),
103 DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
104 DEVMETHOD(cryptodev_process, xlp_sec_process),
105
106 DEVMETHOD_END
107 };
108
109 static driver_t xlp_sec_driver = {
110 "nlmsec",
111 xlp_sec_methods,
112 sizeof(struct xlp_sec_softc)
113 };
114 static devclass_t xlp_sec_devclass;
115
116 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
117 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
118
119 void
120 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
121 struct nlm_fmn_msg *msg, void *data);
122
123 #ifdef NLM_SEC_DEBUG
124
125 #define extract_bits(x, bitshift, bitcnt) \
126 (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
127
128 void
print_crypto_params(struct xlp_sec_command * cmd,struct nlm_fmn_msg m)129 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
130 {
131 unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
132
133 msg0 = cmd->ctrlp->desc0;
134 msg1 = cmd->paramp->desc0;
135 msg2 = cmd->paramp->desc1;
136 msg3 = cmd->paramp->desc2;
137 msg4 = cmd->paramp->desc3;
138 msg5 = cmd->paramp->segment[0][0];
139 msg6 = cmd->paramp->segment[0][1];
140 msg7 = m.msg[0];
141 msg8 = m.msg[1];
142
143 printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
144 "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
145 msg5, msg6, msg7, msg8);
146
147 printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
148 (unsigned int)extract_bits(msg0, 61, 1),
149 (unsigned int)extract_bits(msg0, 52, 8),
150 (unsigned int)extract_bits(msg0, 43, 8),
151 (unsigned int)extract_bits(msg0, 34, 8),
152 (unsigned int)extract_bits(msg0, 25, 8),
153 (unsigned int)extract_bits(msg0, 0, 23));
154
155 printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
156 (unsigned int)extract_bits(msg1, 63, 1),
157 (unsigned int)extract_bits(msg1,62,1),
158 (unsigned int)extract_bits(msg1,60,1),
159 (unsigned int)extract_bits(msg1,59,1),
160 (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
161
162 printf("p1: clen %u hl %u\n", (unsigned int)extract_bits(msg2, 32, 32),
163 (unsigned int)extract_bits(msg2,0,32));
164
165 printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
166 (unsigned int)extract_bits(msg3, 45, 17),
167 (unsigned int)extract_bits(msg3, 42,3),
168 (unsigned int)extract_bits(msg3, 22,16),
169 (unsigned int)extract_bits(msg3, 19,3),
170 (unsigned int)extract_bits(msg3, 18,1),
171 (unsigned int)extract_bits(msg3, 0, 16));
172
173 printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
174 (unsigned int)extract_bits(msg4, 48,16),
175 (unsigned int)extract_bits(msg4,11,16),
176 (unsigned int)extract_bits(msg4,6,3),
177 (unsigned int)extract_bits(msg4,5,1));
178
179 printf("p4: sflen %d sddr %llx \n",
180 (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
181
182 printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
183 (unsigned int)extract_bits(msg6, 48, 16),
184 (unsigned int)extract_bits(msg6, 46, 1),
185 (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
186
187 printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
188 (unsigned int)extract_bits(msg7, 48, 16),
189 (unsigned int)extract_bits(msg7,46,2),
190 (unsigned int)extract_bits(msg7,45,1),
191 (unsigned int)extract_bits(msg7,40,5),
192 (extract_bits(msg7,0,34)<< 6));
193
194 printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
195 (unsigned int)extract_bits(msg8, 63, 1),
196 (unsigned int)extract_bits(msg8,56,5),
197 (unsigned int)extract_bits(msg8,43,12),
198 (extract_bits(msg8,0,34) << 6));
199
200 return;
201 }
202
203 void
print_cmd(struct xlp_sec_command * cmd)204 print_cmd(struct xlp_sec_command *cmd)
205 {
206 printf("session_num :%d\n",cmd->session_num);
207 printf("crp :0x%x\n",(uint32_t)cmd->crp);
208 printf("enccrd :0x%x\n",(uint32_t)cmd->enccrd);
209 printf("maccrd :0x%x\n",(uint32_t)cmd->maccrd);
210 printf("ses :%d\n",(uint32_t)cmd->ses);
211 printf("ctrlp :0x%x\n",(uint32_t)cmd->ctrlp);
212 printf("paramp :0x%x\n",(uint32_t)cmd->paramp);
213 printf("hashdest :0x%x\n",(uint32_t)cmd->hashdest);
214 printf("hashsrc :%d\n",cmd->hashsrc);
215 printf("hmacpad :%d\n",cmd->hmacpad);
216 printf("hashoff :%d\n",cmd->hashoff);
217 printf("hashlen :%d\n",cmd->hashlen);
218 printf("cipheroff :%d\n",cmd->cipheroff);
219 printf("cipherlen :%d\n",cmd->cipherlen);
220 printf("ivoff :%d\n",cmd->ivoff);
221 printf("ivlen :%d\n",cmd->ivlen);
222 printf("hashalg :%d\n",cmd->hashalg);
223 printf("hashmode :%d\n",cmd->hashmode);
224 printf("cipheralg :%d\n",cmd->cipheralg);
225 printf("ciphermode :%d\n",cmd->ciphermode);
226 printf("nsegs :%d\n",cmd->nsegs);
227 printf("hash_dst_len :%d\n",cmd->hash_dst_len);
228 }
229 #endif /* NLM_SEC_DEBUG */
230
231 static int
xlp_sec_init(struct xlp_sec_softc * sc)232 xlp_sec_init(struct xlp_sec_softc *sc)
233 {
234
235 /* Register interrupt handler for the SEC CMS messages */
236 if (register_msgring_handler(sc->sec_vc_start,
237 sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
238 printf("Couldn't register sec msgring handler\n");
239 return (-1);
240 }
241
242 /* Do the CMS credit initialization */
243 /* Currently it is configured by default to 50 when kernel comes up */
244
245 return (0);
246 }
247
248 /* This function is called from an interrupt handler */
249 void
nlm_xlpsec_msgring_handler(int vc,int size,int code,int src_id,struct nlm_fmn_msg * msg,void * data)250 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
251 struct nlm_fmn_msg *msg, void *data)
252 {
253 struct xlp_sec_command *cmd = NULL;
254 struct xlp_sec_softc *sc = NULL;
255 uint8_t hash[HASH_MAX_LEN];
256
257 KASSERT(code == FMN_SWCODE_CRYPTO,
258 ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
259 code, FMN_SWCODE_CRYPTO));
260
261 sc = (struct xlp_sec_softc *)data;
262 KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
263 ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
264 src_id, sc->sec_vc_start, sc->sec_vc_end));
265
266 cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
267 KASSERT(cmd != NULL && cmd->crp != NULL,
268 ("%s :cmd not received properly\n",__FUNCTION__));
269
270 KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
271 ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
272 (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
273 (int)CRYPTO_ERROR(msg->msg[1])));
274
275 /* If there are not enough credits to send, then send request
276 * will fail with ERESTART and the driver will be blocked until it is
277 * unblocked here after knowing that there are sufficient credits to
278 * send the request again.
279 */
280 if (sc->sc_needwakeup) {
281 atomic_add_int(&creditleft, sc->sec_msgsz);
282 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
283 crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
284 sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
285 }
286 }
287 if (cmd->hash_dst_len != 0) {
288 if (cmd->crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
289 crypto_copydata(cmd->crp, cmd->crp->crp_digest_start,
290 cmd->hash_dst_len, hash);
291 if (timingsafe_bcmp(cmd->hashdest, hash,
292 cmd->hash_dst_len) != 0)
293 cmd->crp->crp_etype = EBADMSG;
294 } else
295 crypto_copyback(cmd->crp, cmd->crp->crp_digest_start,
296 cmd->hash_dst_len, cmd->hashdest);
297 }
298
299 /* This indicates completion of the crypto operation */
300 crypto_done(cmd->crp);
301
302 xlp_free_cmd_params(cmd);
303
304 return;
305 }
306
307 static int
xlp_sec_probe(device_t dev)308 xlp_sec_probe(device_t dev)
309 {
310 struct xlp_sec_softc *sc;
311
312 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
313 pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
314 sc = device_get_softc(dev);
315 return (BUS_PROBE_DEFAULT);
316 }
317 return (ENXIO);
318 }
319
320 /*
321 * Attach an interface that successfully probed.
322 */
323 static int
xlp_sec_attach(device_t dev)324 xlp_sec_attach(device_t dev)
325 {
326 struct xlp_sec_softc *sc = device_get_softc(dev);
327 uint64_t base;
328 int qstart, qnum;
329 int freq, node;
330
331 sc->sc_dev = dev;
332
333 node = nlm_get_device_node(pci_get_slot(dev));
334 freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
335 if (bootverbose)
336 device_printf(dev, "SAE Freq: %dMHz\n", freq);
337 if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
338 device_set_desc(dev, "XLP Security Accelerator");
339 sc->sc_cid = crypto_get_driverid(dev,
340 sizeof(struct xlp_sec_session), CRYPTOCAP_F_HARDWARE);
341 if (sc->sc_cid < 0) {
342 printf("xlp_sec - error : could not get the driver"
343 " id\n");
344 goto error_exit;
345 }
346
347 base = nlm_get_sec_pcibase(node);
348 qstart = nlm_qidstart(base);
349 qnum = nlm_qnum(base);
350 sc->sec_vc_start = qstart;
351 sc->sec_vc_end = qstart + qnum - 1;
352 }
353
354 if (xlp_sec_init(sc) != 0)
355 goto error_exit;
356 if (bootverbose)
357 device_printf(dev, "SEC Initialization complete!\n");
358 return (0);
359
360 error_exit:
361 return (ENXIO);
362
363 }
364
365 /*
366 * Detach an interface that successfully probed.
367 */
368 static int
xlp_sec_detach(device_t dev)369 xlp_sec_detach(device_t dev)
370 {
371 return (0);
372 }
373
374 static bool
xlp_sec_auth_supported(const struct crypto_session_params * csp)375 xlp_sec_auth_supported(const struct crypto_session_params *csp)
376 {
377
378 switch (csp->csp_auth_alg) {
379 case CRYPTO_SHA1:
380 case CRYPTO_SHA1_HMAC:
381 break;
382 default:
383 return (false);
384 }
385 return (true);
386 }
387
388 static bool
xlp_sec_cipher_supported(const struct crypto_session_params * csp)389 xlp_sec_cipher_supported(const struct crypto_session_params *csp)
390 {
391
392 switch (csp->csp_cipher_alg) {
393 case CRYPTO_AES_CBC:
394 if (csp->csp_ivlen != XLP_SEC_AES_IV_LENGTH)
395 return (false);
396 break;
397 default:
398 return (false);
399 }
400
401 return (true);
402 }
403
404 static int
xlp_sec_probesession(device_t dev,const struct crypto_session_params * csp)405 xlp_sec_probesession(device_t dev, const struct crypto_session_params *csp)
406 {
407
408 if (csp->csp_flags != 0)
409 return (EINVAL);
410 switch (csp->csp_mode) {
411 case CSP_MODE_DIGEST:
412 if (!xlp_sec_auth_supported(csp))
413 return (EINVAL);
414 break;
415 case CSP_MODE_CIPHER:
416 if (!xlp_sec_cipher_supported(csp))
417 return (EINVAL);
418 break;
419 case CSP_MODE_ETA:
420 if (!xlp_sec_auth_supported(csp) ||
421 !xlp_sec_cipher_supported(csp))
422 return (EINVAL);
423 break;
424 default:
425 return (EINVAL);
426 }
427 return (CRYPTODEV_PROBE_HARDWARE);
428 }
429
430 static int
xlp_sec_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)431 xlp_sec_newsession(device_t dev, crypto_session_t cses,
432 const struct crypto_session_params *csp)
433 {
434 struct xlp_sec_session *ses;
435
436 ses = crypto_get_driver_session(cses);
437
438 if (csp->csp_auth_alg != 0) {
439 if (csp->csp_auth_mlen == 0)
440 ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
441 else
442 ses->hs_mlen = csp->csp_auth_mlen;
443 }
444
445 return (0);
446 }
447
448 /*
449 * XXX freesession routine should run a zero'd mac/encrypt key into context
450 * ram. to blow away any keys already stored there.
451 */
452
453 static void
xlp_copyiv(struct xlp_sec_softc * sc,struct xlp_sec_command * cmd,const struct crypto_session_params * csp)454 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
455 const struct crypto_session_params *csp)
456 {
457 struct cryptop *crp = NULL;
458
459 crp = cmd->crp;
460
461 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
462 memcpy(cmd->iv, crp->crp_iv, csp->csp_ivlen);
463 }
464
465 static int
xlp_get_nsegs(struct cryptop * crp,unsigned int * nsegs)466 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
467 {
468
469 switch (crp->crp_buf.cb_type) {
470 case CRYPTO_BUF_MBUF:
471 {
472 struct mbuf *m = NULL;
473
474 m = crp->crp_buf.cb_mbuf;
475 while (m != NULL) {
476 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
477 m = m->m_next;
478 }
479 break;
480 }
481 case CRYPTO_BUF_UIO:
482 {
483 struct uio *uio = NULL;
484 struct iovec *iov = NULL;
485 int iol = 0;
486
487 uio = crp->crp_buf.cb_uio;
488 iov = uio->uio_iov;
489 iol = uio->uio_iovcnt;
490 while (iol > 0) {
491 *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
492 iol--;
493 iov++;
494 }
495 break;
496 }
497 case CRYPTO_BUF_CONTIG:
498 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_buf.cb_buf_len);
499 break;
500 default:
501 return (EINVAL);
502 }
503 return (0);
504 }
505
506 static int
xlp_alloc_cmd_params(struct xlp_sec_command * cmd,unsigned int nsegs)507 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
508 {
509 int err = 0;
510
511 if(cmd == NULL) {
512 err = EINVAL;
513 goto error;
514 }
515 if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
516 M_NOWAIT | M_ZERO)) == NULL) {
517 err = ENOMEM;
518 goto error;
519 }
520 if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
521 err = EINVAL;
522 goto error;
523 }
524 /* (nsegs - 1) because one seg is part of the structure already */
525 if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
526 (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
527 err = ENOMEM;
528 goto error;
529 }
530 if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
531 err = EINVAL;
532 goto error;
533 }
534 if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
535 M_NOWAIT | M_ZERO)) == NULL) {
536 err = ENOMEM;
537 goto error;
538 }
539 if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
540 M_NOWAIT | M_ZERO)) == NULL) {
541 err = ENOMEM;
542 goto error;
543 }
544 error:
545 return (err);
546 }
547
548 static void
xlp_free_cmd_params(struct xlp_sec_command * cmd)549 xlp_free_cmd_params(struct xlp_sec_command *cmd)
550 {
551 if (cmd->ctrlp != NULL)
552 free(cmd->ctrlp, M_DEVBUF);
553 if (cmd->paramp != NULL)
554 free(cmd->paramp, M_DEVBUF);
555 if (cmd->iv != NULL)
556 free(cmd->iv, M_DEVBUF);
557 if (cmd->hashdest != NULL)
558 free(cmd->hashdest, M_DEVBUF);
559 if (cmd != NULL)
560 free(cmd, M_DEVBUF);
561 return;
562 }
563
564 static int
xlp_sec_process(device_t dev,struct cryptop * crp,int hint)565 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
566 {
567 struct xlp_sec_softc *sc = device_get_softc(dev);
568 const struct crypto_session_params *csp;
569 struct xlp_sec_command *cmd = NULL;
570 int err = -1, ret = 0;
571 struct xlp_sec_session *ses;
572 unsigned int nsegs = 0;
573
574 ses = crypto_get_driver_session(crp->crp_session);
575 csp = crypto_get_params(crp->crp_session);
576
577 /*
578 * This device only support AAD requests where the AAD is
579 * adjacent to the payload.
580 */
581 if (crp->crp_aad_length != 0 && crp->crp_payload_start !=
582 crp->crp_aad_start + crp->crp_aad_length) {
583 err = EFBIG;
584 goto errout;
585 }
586
587 if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
588 M_NOWAIT | M_ZERO)) == NULL) {
589 err = ENOMEM;
590 goto errout;
591 }
592
593 cmd->crp = crp;
594 cmd->ses = ses;
595 cmd->hash_dst_len = ses->hs_mlen;
596
597 if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
598 err = EINVAL;
599 goto errout;
600 }
601
602 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
603 /* Since IV is given as separate segment to avoid copy */
604 nsegs += 1;
605 }
606 cmd->nsegs = nsegs;
607
608 if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
609 goto errout;
610
611 switch (csp->csp_mode) {
612 case CSP_MODE_CIPHER:
613 if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
614 err = EINVAL;
615 goto errout;
616 }
617 cmd->cipheroff = crp->crp_payload_start;
618 cmd->cipherlen = crp->crp_payload_length;
619 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
620 cmd->cipheroff += cmd->ivlen;
621 cmd->ivoff = 0;
622 } else
623 cmd->ivoff = crp->crp_iv_start;
624 xlp_copyiv(sc, cmd, csp);
625 if ((err = nlm_crypto_do_cipher(sc, cmd, csp)) != 0)
626 goto errout;
627 break;
628 case CSP_MODE_DIGEST:
629 if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
630 err = EINVAL;
631 goto errout;
632 }
633 cmd->hashoff = crp->crp_payload_start;
634 cmd->hashlen = crp->crp_payload_length;
635 cmd->hmacpad = 0;
636 cmd->hashsrc = 0;
637 if ((err = nlm_crypto_do_digest(sc, cmd, csp)) != 0)
638 goto errout;
639 break;
640 case CSP_MODE_ETA:
641 if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
642 err = EINVAL;
643 goto errout;
644 }
645 if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
646 err = EINVAL;
647 goto errout;
648 }
649 if (crp->crp_aad_length != 0) {
650 cmd->hashoff = crp->crp_aad_start;
651 cmd->hashlen = crp->crp_aad_length +
652 crp->crp_payload_length;
653 } else {
654 cmd->hashoff = crp->crp_payload_start;
655 cmd->hashlen = crp->crp_payload_length;
656 }
657 cmd->hmacpad = 0;
658 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
659 cmd->hashsrc = 1;
660 else
661 cmd->hashsrc = 0;
662 cmd->cipheroff = crp->crp_payload_start;
663 cmd->cipherlen = crp->crp_payload_length;
664 if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
665 cmd->hashoff += cmd->ivlen;
666 cmd->cipheroff += cmd->ivlen;
667 cmd->ivoff = 0;
668 } else
669 cmd->ivoff = crp->crp_iv_start;
670 xlp_copyiv(sc, cmd, csp);
671 if ((err = nlm_crypto_do_cipher_digest(sc, cmd, csp)) != 0)
672 goto errout;
673 break;
674 default:
675 err = EINVAL;
676 goto errout;
677 }
678 return (0);
679 errout:
680 xlp_free_cmd_params(cmd);
681 if (err == ERESTART) {
682 sc->sc_needwakeup |= CRYPTO_SYMQ;
683 creditleft = 0;
684 return (err);
685 }
686 crp->crp_etype = err;
687 crypto_done(crp);
688 return (err);
689 }
690