1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (C) 2009-2011 Semihalf.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * CESA SRAM Memory Map:
31 *
32 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE
33 * | |
34 * | DATA |
35 * | |
36 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0)
37 * | struct cesa_sa_data |
38 * +------------------------+
39 * | struct cesa_sa_hdesc |
40 * +------------------------+ <= sc->sc_sram_base_va
41 */
42
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/bus.h>
49 #include <sys/endian.h>
50 #include <sys/kernel.h>
51 #include <sys/lock.h>
52 #include <sys/mbuf.h>
53 #include <sys/module.h>
54 #include <sys/mutex.h>
55 #include <sys/rman.h>
56
57 #include <machine/bus.h>
58 #include <machine/intr.h>
59 #include <machine/resource.h>
60 #include <machine/fdt.h>
61
62 #include <dev/fdt/simplebus.h>
63 #include <dev/fdt/fdt_common.h>
64 #include <dev/ofw/ofw_bus.h>
65 #include <dev/ofw/ofw_bus_subr.h>
66
67 #include <sys/md5.h>
68 #include <crypto/sha1.h>
69 #include <crypto/sha2/sha256.h>
70 #include <crypto/rijndael/rijndael.h>
71 #include <opencrypto/cryptodev.h>
72 #include "cryptodev_if.h"
73
74 #include <arm/mv/mvreg.h>
75 #include <arm/mv/mvvar.h>
76 #include "cesa.h"
77
78 static int cesa_probe(device_t);
79 static int cesa_attach(device_t);
80 static int cesa_attach_late(device_t);
81 static int cesa_detach(device_t);
82 static void cesa_intr(void *);
83 static int cesa_newsession(device_t, crypto_session_t, struct cryptoini *);
84 static int cesa_process(device_t, struct cryptop *, int);
85
86 static struct resource_spec cesa_res_spec[] = {
87 { SYS_RES_MEMORY, 0, RF_ACTIVE },
88 { SYS_RES_MEMORY, 1, RF_ACTIVE },
89 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE },
90 { -1, 0 }
91 };
92
93 static device_method_t cesa_methods[] = {
94 /* Device interface */
95 DEVMETHOD(device_probe, cesa_probe),
96 DEVMETHOD(device_attach, cesa_attach),
97 DEVMETHOD(device_detach, cesa_detach),
98
99 /* Crypto device methods */
100 DEVMETHOD(cryptodev_newsession, cesa_newsession),
101 DEVMETHOD(cryptodev_process, cesa_process),
102
103 DEVMETHOD_END
104 };
105
106 static driver_t cesa_driver = {
107 "cesa",
108 cesa_methods,
109 sizeof (struct cesa_softc)
110 };
111 static devclass_t cesa_devclass;
112
113 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0);
114 MODULE_DEPEND(cesa, crypto, 1, 1, 1);
115
116 static void
cesa_dump_cshd(struct cesa_softc * sc,struct cesa_sa_hdesc * cshd)117 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
118 {
119 #ifdef DEBUG
120 device_t dev;
121
122 dev = sc->sc_dev;
123 device_printf(dev, "CESA SA Hardware Descriptor:\n");
124 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
125 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src);
126 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst);
127 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
128 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key);
129 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
130 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
131 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src);
132 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst);
133 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
134 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
135 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
136 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
137 #endif
138 }
139
140 static void
cesa_alloc_dma_mem_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)141 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
142 {
143 struct cesa_dma_mem *cdm;
144
145 if (error)
146 return;
147
148 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1."));
149 cdm = arg;
150 cdm->cdm_paddr = segs->ds_addr;
151 }
152
153 static int
cesa_alloc_dma_mem(struct cesa_softc * sc,struct cesa_dma_mem * cdm,bus_size_t size)154 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm,
155 bus_size_t size)
156 {
157 int error;
158
159 KASSERT(cdm->cdm_vaddr == NULL,
160 ("%s(): DMA memory descriptor in use.", __func__));
161
162 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
163 PAGE_SIZE, 0, /* alignment, boundary */
164 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
165 BUS_SPACE_MAXADDR, /* highaddr */
166 NULL, NULL, /* filtfunc, filtfuncarg */
167 size, 1, /* maxsize, nsegments */
168 size, 0, /* maxsegsz, flags */
169 NULL, NULL, /* lockfunc, lockfuncarg */
170 &cdm->cdm_tag); /* dmat */
171 if (error) {
172 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
173 " %i!\n", error);
174
175 goto err1;
176 }
177
178 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr,
179 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map);
180 if (error) {
181 device_printf(sc->sc_dev, "failed to allocate DMA safe"
182 " memory, error %i!\n", error);
183
184 goto err2;
185 }
186
187 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr,
188 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT);
189 if (error) {
190 device_printf(sc->sc_dev, "cannot get address of the DMA"
191 " memory, error %i\n", error);
192
193 goto err3;
194 }
195
196 return (0);
197 err3:
198 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
199 err2:
200 bus_dma_tag_destroy(cdm->cdm_tag);
201 err1:
202 cdm->cdm_vaddr = NULL;
203 return (error);
204 }
205
206 static void
cesa_free_dma_mem(struct cesa_dma_mem * cdm)207 cesa_free_dma_mem(struct cesa_dma_mem *cdm)
208 {
209
210 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map);
211 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
212 bus_dma_tag_destroy(cdm->cdm_tag);
213 cdm->cdm_vaddr = NULL;
214 }
215
216 static void
cesa_sync_dma_mem(struct cesa_dma_mem * cdm,bus_dmasync_op_t op)217 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op)
218 {
219
220 /* Sync only if dma memory is valid */
221 if (cdm->cdm_vaddr != NULL)
222 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
223 }
224
225 static void
cesa_sync_desc(struct cesa_softc * sc,bus_dmasync_op_t op)226 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
227 {
228
229 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
230 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
231 cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
232 }
233
234 static struct cesa_request *
cesa_alloc_request(struct cesa_softc * sc)235 cesa_alloc_request(struct cesa_softc *sc)
236 {
237 struct cesa_request *cr;
238
239 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests);
240 if (!cr)
241 return (NULL);
242
243 STAILQ_INIT(&cr->cr_tdesc);
244 STAILQ_INIT(&cr->cr_sdesc);
245
246 return (cr);
247 }
248
249 static void
cesa_free_request(struct cesa_softc * sc,struct cesa_request * cr)250 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
251 {
252
253 /* Free TDMA descriptors assigned to this request */
254 CESA_LOCK(sc, tdesc);
255 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
256 CESA_UNLOCK(sc, tdesc);
257
258 /* Free SA descriptors assigned to this request */
259 CESA_LOCK(sc, sdesc);
260 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
261 CESA_UNLOCK(sc, sdesc);
262
263 /* Unload DMA memory associated with request */
264 if (cr->cr_dmap_loaded) {
265 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap);
266 cr->cr_dmap_loaded = 0;
267 }
268
269 CESA_GENERIC_FREE_LOCKED(sc, cr, requests);
270 }
271
272 static void
cesa_enqueue_request(struct cesa_softc * sc,struct cesa_request * cr)273 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr)
274 {
275
276 CESA_LOCK(sc, requests);
277 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq);
278 CESA_UNLOCK(sc, requests);
279 }
280
281 static struct cesa_tdma_desc *
cesa_alloc_tdesc(struct cesa_softc * sc)282 cesa_alloc_tdesc(struct cesa_softc *sc)
283 {
284 struct cesa_tdma_desc *ctd;
285
286 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
287
288 if (!ctd)
289 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
290 "Consider increasing CESA_TDMA_DESCRIPTORS.\n");
291
292 return (ctd);
293 }
294
295 static struct cesa_sa_desc *
cesa_alloc_sdesc(struct cesa_softc * sc,struct cesa_request * cr)296 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
297 {
298 struct cesa_sa_desc *csd;
299
300 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
301 if (!csd) {
302 device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
303 "Consider increasing CESA_SA_DESCRIPTORS.\n");
304 return (NULL);
305 }
306
307 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
308
309 /* Fill-in SA descriptor with default values */
310 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
311 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
312 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
313 csd->csd_cshd->cshd_enc_src = 0;
314 csd->csd_cshd->cshd_enc_dst = 0;
315 csd->csd_cshd->cshd_enc_dlen = 0;
316 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
317 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
318 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
319 csd->csd_cshd->cshd_mac_src = 0;
320 csd->csd_cshd->cshd_mac_dlen = 0;
321
322 return (csd);
323 }
324
325 static struct cesa_tdma_desc *
cesa_tdma_copy(struct cesa_softc * sc,bus_addr_t dst,bus_addr_t src,bus_size_t size)326 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
327 bus_size_t size)
328 {
329 struct cesa_tdma_desc *ctd;
330
331 ctd = cesa_alloc_tdesc(sc);
332 if (!ctd)
333 return (NULL);
334
335 ctd->ctd_cthd->cthd_dst = dst;
336 ctd->ctd_cthd->cthd_src = src;
337 ctd->ctd_cthd->cthd_byte_count = size;
338
339 /* Handle special control packet */
340 if (size != 0)
341 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
342 else
343 ctd->ctd_cthd->cthd_flags = 0;
344
345 return (ctd);
346 }
347
348 static struct cesa_tdma_desc *
cesa_tdma_copyin_sa_data(struct cesa_softc * sc,struct cesa_request * cr)349 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
350 {
351
352 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa +
353 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
354 sizeof(struct cesa_sa_data)));
355 }
356
357 static struct cesa_tdma_desc *
cesa_tdma_copyout_sa_data(struct cesa_softc * sc,struct cesa_request * cr)358 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
359 {
360
361 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa +
362 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
363 }
364
365 static struct cesa_tdma_desc *
cesa_tdma_copy_sdesc(struct cesa_softc * sc,struct cesa_sa_desc * csd)366 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
367 {
368
369 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr,
370 sizeof(struct cesa_sa_hdesc)));
371 }
372
373 static void
cesa_append_tdesc(struct cesa_request * cr,struct cesa_tdma_desc * ctd)374 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
375 {
376 struct cesa_tdma_desc *ctd_prev;
377
378 if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
379 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
380 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
381 }
382
383 ctd->ctd_cthd->cthd_next = 0;
384 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
385 }
386
387 static int
cesa_append_packet(struct cesa_softc * sc,struct cesa_request * cr,struct cesa_packet * cp,struct cesa_sa_desc * csd)388 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
389 struct cesa_packet *cp, struct cesa_sa_desc *csd)
390 {
391 struct cesa_tdma_desc *ctd, *tmp;
392
393 /* Copy SA descriptor for this packet */
394 ctd = cesa_tdma_copy_sdesc(sc, csd);
395 if (!ctd)
396 return (ENOMEM);
397
398 cesa_append_tdesc(cr, ctd);
399
400 /* Copy data to be processed */
401 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
402 cesa_append_tdesc(cr, ctd);
403 STAILQ_INIT(&cp->cp_copyin);
404
405 /* Insert control descriptor */
406 ctd = cesa_tdma_copy(sc, 0, 0, 0);
407 if (!ctd)
408 return (ENOMEM);
409
410 cesa_append_tdesc(cr, ctd);
411
412 /* Copy back results */
413 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
414 cesa_append_tdesc(cr, ctd);
415 STAILQ_INIT(&cp->cp_copyout);
416
417 return (0);
418 }
419
420 static int
cesa_set_mkey(struct cesa_session * cs,int alg,const uint8_t * mkey,int mklen)421 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
422 {
423 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
424 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
425 SHA1_CTX sha1ctx;
426 SHA256_CTX sha256ctx;
427 MD5_CTX md5ctx;
428 uint32_t *hout;
429 uint32_t *hin;
430 int i;
431
432 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
433 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
434 for (i = 0; i < mklen; i++) {
435 ipad[i] ^= mkey[i];
436 opad[i] ^= mkey[i];
437 }
438
439 hin = (uint32_t *)cs->cs_hiv_in;
440 hout = (uint32_t *)cs->cs_hiv_out;
441
442 switch (alg) {
443 case CRYPTO_MD5_HMAC:
444 MD5Init(&md5ctx);
445 MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN);
446 memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
447 MD5Init(&md5ctx);
448 MD5Update(&md5ctx, opad, MD5_BLOCK_LEN);
449 memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
450 break;
451 case CRYPTO_SHA1_HMAC:
452 SHA1Init(&sha1ctx);
453 SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN);
454 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
455 SHA1Init(&sha1ctx);
456 SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN);
457 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
458 break;
459 case CRYPTO_SHA2_256_HMAC:
460 SHA256_Init(&sha256ctx);
461 SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN);
462 memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state));
463 SHA256_Init(&sha256ctx);
464 SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN);
465 memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state));
466 break;
467 default:
468 return (EINVAL);
469 }
470
471 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
472 hin[i] = htobe32(hin[i]);
473 hout[i] = htobe32(hout[i]);
474 }
475
476 return (0);
477 }
478
479 static int
cesa_prep_aes_key(struct cesa_session * cs)480 cesa_prep_aes_key(struct cesa_session *cs)
481 {
482 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
483 uint32_t *dkey;
484 int i;
485
486 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
487
488 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
489 dkey = (uint32_t *)cs->cs_aes_dkey;
490
491 switch (cs->cs_klen) {
492 case 16:
493 cs->cs_config |= CESA_CSH_AES_KLEN_128;
494 for (i = 0; i < 4; i++)
495 *dkey++ = htobe32(ek[4 * 10 + i]);
496 break;
497 case 24:
498 cs->cs_config |= CESA_CSH_AES_KLEN_192;
499 for (i = 0; i < 4; i++)
500 *dkey++ = htobe32(ek[4 * 12 + i]);
501 for (i = 0; i < 2; i++)
502 *dkey++ = htobe32(ek[4 * 11 + 2 + i]);
503 break;
504 case 32:
505 cs->cs_config |= CESA_CSH_AES_KLEN_256;
506 for (i = 0; i < 4; i++)
507 *dkey++ = htobe32(ek[4 * 14 + i]);
508 for (i = 0; i < 4; i++)
509 *dkey++ = htobe32(ek[4 * 13 + i]);
510 break;
511 default:
512 return (EINVAL);
513 }
514
515 return (0);
516 }
517
518 static int
cesa_is_hash(int alg)519 cesa_is_hash(int alg)
520 {
521
522 switch (alg) {
523 case CRYPTO_MD5:
524 case CRYPTO_MD5_HMAC:
525 case CRYPTO_SHA1:
526 case CRYPTO_SHA1_HMAC:
527 case CRYPTO_SHA2_256_HMAC:
528 return (1);
529 default:
530 return (0);
531 }
532 }
533
534 static void
cesa_start_packet(struct cesa_packet * cp,unsigned int size)535 cesa_start_packet(struct cesa_packet *cp, unsigned int size)
536 {
537
538 cp->cp_size = size;
539 cp->cp_offset = 0;
540 STAILQ_INIT(&cp->cp_copyin);
541 STAILQ_INIT(&cp->cp_copyout);
542 }
543
544 static int
cesa_fill_packet(struct cesa_softc * sc,struct cesa_packet * cp,bus_dma_segment_t * seg)545 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
546 bus_dma_segment_t *seg)
547 {
548 struct cesa_tdma_desc *ctd;
549 unsigned int bsize;
550
551 /* Calculate size of block copy */
552 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
553
554 if (bsize > 0) {
555 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa +
556 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
557 if (!ctd)
558 return (-ENOMEM);
559
560 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
561
562 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa +
563 CESA_DATA(cp->cp_offset), bsize);
564 if (!ctd)
565 return (-ENOMEM);
566
567 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
568
569 seg->ds_len -= bsize;
570 seg->ds_addr += bsize;
571 cp->cp_offset += bsize;
572 }
573
574 return (bsize);
575 }
576
577 static void
cesa_create_chain_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)578 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
579 {
580 unsigned int mpsize, fragmented;
581 unsigned int mlen, mskip, tmlen;
582 struct cesa_chain_info *cci;
583 unsigned int elen, eskip;
584 unsigned int skip, len;
585 struct cesa_sa_desc *csd;
586 struct cesa_request *cr;
587 struct cesa_softc *sc;
588 struct cesa_packet cp;
589 bus_dma_segment_t seg;
590 uint32_t config;
591 int size;
592
593 cci = arg;
594 sc = cci->cci_sc;
595 cr = cci->cci_cr;
596
597 if (error) {
598 cci->cci_error = error;
599 return;
600 }
601
602 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
603 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
604 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
605 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
606
607 if (elen && mlen &&
608 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
609 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
610 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
611 /*
612 * Data alignment in the request does not meet CESA requiremnts
613 * for combined encryption/decryption and hashing. We have to
614 * split the request to separate operations and process them
615 * one by one.
616 */
617 config = cci->cci_config;
618 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
619 config &= ~CESA_CSHD_OP_MASK;
620
621 cci->cci_config = config | CESA_CSHD_MAC;
622 cci->cci_enc = NULL;
623 cci->cci_mac = cr->cr_mac;
624 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
625
626 cci->cci_config = config | CESA_CSHD_ENC;
627 cci->cci_enc = cr->cr_enc;
628 cci->cci_mac = NULL;
629 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
630 } else {
631 config &= ~CESA_CSHD_OP_MASK;
632
633 cci->cci_config = config | CESA_CSHD_ENC;
634 cci->cci_enc = cr->cr_enc;
635 cci->cci_mac = NULL;
636 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
637
638 cci->cci_config = config | CESA_CSHD_MAC;
639 cci->cci_enc = NULL;
640 cci->cci_mac = cr->cr_mac;
641 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
642 }
643
644 return;
645 }
646
647 tmlen = mlen;
648 fragmented = 0;
649 mpsize = CESA_MAX_PACKET_SIZE;
650 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
651
652 if (elen && mlen) {
653 skip = MIN(eskip, mskip);
654 len = MAX(elen + eskip, mlen + mskip) - skip;
655 } else if (elen) {
656 skip = eskip;
657 len = elen;
658 } else {
659 skip = mskip;
660 len = mlen;
661 }
662
663 /* Start first packet in chain */
664 cesa_start_packet(&cp, MIN(mpsize, len));
665
666 while (nseg-- && len > 0) {
667 seg = *(segs++);
668
669 /*
670 * Skip data in buffer on which neither ENC nor MAC operation
671 * is requested.
672 */
673 if (skip > 0) {
674 size = MIN(skip, seg.ds_len);
675 skip -= size;
676
677 seg.ds_addr += size;
678 seg.ds_len -= size;
679
680 if (eskip > 0)
681 eskip -= size;
682
683 if (mskip > 0)
684 mskip -= size;
685
686 if (seg.ds_len == 0)
687 continue;
688 }
689
690 while (1) {
691 /*
692 * Fill in current packet with data. Break if there is
693 * no more data in current DMA segment or an error
694 * occurred.
695 */
696 size = cesa_fill_packet(sc, &cp, &seg);
697 if (size <= 0) {
698 error = -size;
699 break;
700 }
701
702 len -= size;
703
704 /* If packet is full, append it to the chain */
705 if (cp.cp_size == cp.cp_offset) {
706 csd = cesa_alloc_sdesc(sc, cr);
707 if (!csd) {
708 error = ENOMEM;
709 break;
710 }
711
712 /* Create SA descriptor for this packet */
713 csd->csd_cshd->cshd_config = cci->cci_config;
714 csd->csd_cshd->cshd_mac_total_dlen = tmlen;
715
716 /*
717 * Enable fragmentation if request will not fit
718 * into one packet.
719 */
720 if (len > 0) {
721 if (!fragmented) {
722 fragmented = 1;
723 csd->csd_cshd->cshd_config |=
724 CESA_CSHD_FRAG_FIRST;
725 } else
726 csd->csd_cshd->cshd_config |=
727 CESA_CSHD_FRAG_MIDDLE;
728 } else if (fragmented)
729 csd->csd_cshd->cshd_config |=
730 CESA_CSHD_FRAG_LAST;
731
732 if (eskip < cp.cp_size && elen > 0) {
733 csd->csd_cshd->cshd_enc_src =
734 CESA_DATA(eskip);
735 csd->csd_cshd->cshd_enc_dst =
736 CESA_DATA(eskip);
737 csd->csd_cshd->cshd_enc_dlen =
738 MIN(elen, cp.cp_size - eskip);
739 }
740
741 if (mskip < cp.cp_size && mlen > 0) {
742 csd->csd_cshd->cshd_mac_src =
743 CESA_DATA(mskip);
744 csd->csd_cshd->cshd_mac_dlen =
745 MIN(mlen, cp.cp_size - mskip);
746 }
747
748 elen -= csd->csd_cshd->cshd_enc_dlen;
749 eskip -= MIN(eskip, cp.cp_size);
750 mlen -= csd->csd_cshd->cshd_mac_dlen;
751 mskip -= MIN(mskip, cp.cp_size);
752
753 cesa_dump_cshd(sc, csd->csd_cshd);
754
755 /* Append packet to the request */
756 error = cesa_append_packet(sc, cr, &cp, csd);
757 if (error)
758 break;
759
760 /* Start a new packet, as current is full */
761 cesa_start_packet(&cp, MIN(mpsize, len));
762 }
763 }
764
765 if (error)
766 break;
767 }
768
769 if (error) {
770 /*
771 * Move all allocated resources to the request. They will be
772 * freed later.
773 */
774 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
775 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
776 cci->cci_error = error;
777 }
778 }
779
780 static void
cesa_create_chain_cb2(void * arg,bus_dma_segment_t * segs,int nseg,bus_size_t size,int error)781 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
782 bus_size_t size, int error)
783 {
784
785 cesa_create_chain_cb(arg, segs, nseg, error);
786 }
787
788 static int
cesa_create_chain(struct cesa_softc * sc,struct cesa_request * cr)789 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
790 {
791 struct cesa_chain_info cci;
792 struct cesa_tdma_desc *ctd;
793 uint32_t config;
794 int error;
795
796 error = 0;
797 CESA_LOCK_ASSERT(sc, sessions);
798
799 /* Create request metadata */
800 if (cr->cr_enc) {
801 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
802 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
803 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
804 cr->cr_cs->cs_klen);
805 else
806 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
807 cr->cr_cs->cs_klen);
808 }
809
810 if (cr->cr_mac) {
811 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
812 CESA_MAX_HASH_LEN);
813 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
814 CESA_MAX_HASH_LEN);
815 }
816
817 ctd = cesa_tdma_copyin_sa_data(sc, cr);
818 if (!ctd)
819 return (ENOMEM);
820
821 cesa_append_tdesc(cr, ctd);
822
823 /* Prepare SA configuration */
824 config = cr->cr_cs->cs_config;
825
826 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
827 config |= CESA_CSHD_DECRYPT;
828 if (cr->cr_enc && !cr->cr_mac)
829 config |= CESA_CSHD_ENC;
830 if (!cr->cr_enc && cr->cr_mac)
831 config |= CESA_CSHD_MAC;
832 if (cr->cr_enc && cr->cr_mac)
833 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
834 CESA_CSHD_ENC_AND_MAC;
835
836 /* Create data packets */
837 cci.cci_sc = sc;
838 cci.cci_cr = cr;
839 cci.cci_enc = cr->cr_enc;
840 cci.cci_mac = cr->cr_mac;
841 cci.cci_config = config;
842 cci.cci_error = 0;
843
844 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
845 error = bus_dmamap_load_uio(sc->sc_data_dtag,
846 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
847 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
848 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
849 error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
850 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
851 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
852 else
853 error = bus_dmamap_load(sc->sc_data_dtag,
854 cr->cr_dmap, cr->cr_crp->crp_buf,
855 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
856 BUS_DMA_NOWAIT);
857
858 if (!error)
859 cr->cr_dmap_loaded = 1;
860
861 if (cci.cci_error)
862 error = cci.cci_error;
863
864 if (error)
865 return (error);
866
867 /* Read back request metadata */
868 ctd = cesa_tdma_copyout_sa_data(sc, cr);
869 if (!ctd)
870 return (ENOMEM);
871
872 cesa_append_tdesc(cr, ctd);
873
874 return (0);
875 }
876
877 static void
cesa_execute(struct cesa_softc * sc)878 cesa_execute(struct cesa_softc *sc)
879 {
880 struct cesa_tdma_desc *prev_ctd, *ctd;
881 struct cesa_request *prev_cr, *cr;
882
883 CESA_LOCK(sc, requests);
884
885 /*
886 * If ready list is empty, there is nothing to execute. If queued list
887 * is not empty, the hardware is busy and we cannot start another
888 * execution.
889 */
890 if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
891 !STAILQ_EMPTY(&sc->sc_queued_requests)) {
892 CESA_UNLOCK(sc, requests);
893 return;
894 }
895
896 /* Move all ready requests to queued list */
897 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
898 STAILQ_INIT(&sc->sc_ready_requests);
899
900 /* Create one execution chain from all requests on the list */
901 if (STAILQ_FIRST(&sc->sc_queued_requests) !=
902 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
903 prev_cr = NULL;
904 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
905 BUS_DMASYNC_POSTWRITE);
906
907 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
908 if (prev_cr) {
909 ctd = STAILQ_FIRST(&cr->cr_tdesc);
910 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
911 cesa_tdma_desc, ctd_stq);
912
913 prev_ctd->ctd_cthd->cthd_next =
914 ctd->ctd_cthd_paddr;
915 }
916
917 prev_cr = cr;
918 }
919
920 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
921 BUS_DMASYNC_PREWRITE);
922 }
923
924 /* Start chain execution in hardware */
925 cr = STAILQ_FIRST(&sc->sc_queued_requests);
926 ctd = STAILQ_FIRST(&cr->cr_tdesc);
927
928 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
929
930 if (sc->sc_soc_id == MV_DEV_88F6828 ||
931 sc->sc_soc_id == MV_DEV_88F6820 ||
932 sc->sc_soc_id == MV_DEV_88F6810)
933 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2);
934 else
935 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
936
937 CESA_UNLOCK(sc, requests);
938 }
939
940 static int
cesa_setup_sram(struct cesa_softc * sc)941 cesa_setup_sram(struct cesa_softc *sc)
942 {
943 phandle_t sram_node;
944 ihandle_t sram_ihandle;
945 pcell_t sram_handle, sram_reg[2];
946 void *sram_va;
947 int rv;
948
949 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle",
950 (void *)&sram_handle, sizeof(sram_handle));
951 if (rv <= 0)
952 return (rv);
953
954 sram_ihandle = (ihandle_t)sram_handle;
955 sram_node = OF_instance_to_package(sram_ihandle);
956
957 rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg));
958 if (rv <= 0)
959 return (rv);
960
961 sc->sc_sram_base_pa = sram_reg[0];
962 /* Store SRAM size to be able to unmap in detach() */
963 sc->sc_sram_size = sram_reg[1];
964
965 if (sc->sc_soc_id != MV_DEV_88F6828 &&
966 sc->sc_soc_id != MV_DEV_88F6820 &&
967 sc->sc_soc_id != MV_DEV_88F6810)
968 return (0);
969
970 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */
971 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
972 if (sram_va == NULL)
973 return (ENOMEM);
974 sc->sc_sram_base_va = (vm_offset_t)sram_va;
975
976 return (0);
977 }
978
979 /*
980 * Function: device_from_node
981 * This function returns appropriate device_t to phandle_t
982 * Parameters:
983 * root - device where you want to start search
984 * if you provide NULL here, function will take
985 * "root0" device as root.
986 * node - we are checking every device_t to be
987 * appropriate with this.
988 */
989 static device_t
device_from_node(device_t root,phandle_t node)990 device_from_node(device_t root, phandle_t node)
991 {
992 device_t *children, retval;
993 int nkid, i;
994
995 /* Nothing matches no node */
996 if (node == -1)
997 return (NULL);
998
999 if (root == NULL)
1000 /* Get root of device tree */
1001 if ((root = device_lookup_by_name("root0")) == NULL)
1002 return (NULL);
1003
1004 if (device_get_children(root, &children, &nkid) != 0)
1005 return (NULL);
1006
1007 retval = NULL;
1008 for (i = 0; i < nkid; i++) {
1009 /* Check if device and node matches */
1010 if (OFW_BUS_GET_NODE(root, children[i]) == node) {
1011 retval = children[i];
1012 break;
1013 }
1014 /* or go deeper */
1015 if ((retval = device_from_node(children[i], node)) != NULL)
1016 break;
1017 }
1018 free(children, M_TEMP);
1019
1020 return (retval);
1021 }
1022
1023 static int
cesa_setup_sram_armada(struct cesa_softc * sc)1024 cesa_setup_sram_armada(struct cesa_softc *sc)
1025 {
1026 phandle_t sram_node;
1027 ihandle_t sram_ihandle;
1028 pcell_t sram_handle[2];
1029 void *sram_va;
1030 int rv, j;
1031 struct resource_list rl;
1032 struct resource_list_entry *rle;
1033 struct simplebus_softc *ssc;
1034 device_t sdev;
1035
1036 /* Get refs to SRAMS from CESA node */
1037 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams",
1038 (void *)sram_handle, sizeof(sram_handle));
1039 if (rv <= 0)
1040 return (rv);
1041
1042 if (sc->sc_cesa_engine_id >= 2)
1043 return (ENXIO);
1044
1045 /* Get SRAM node on the basis of sc_cesa_engine_id */
1046 sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id];
1047 sram_node = OF_instance_to_package(sram_ihandle);
1048
1049 /* Get device_t of simplebus (sram_node parent) */
1050 sdev = device_from_node(NULL, OF_parent(sram_node));
1051 if (!sdev)
1052 return (ENXIO);
1053
1054 ssc = device_get_softc(sdev);
1055
1056 resource_list_init(&rl);
1057 /* Parse reg property to resource list */
1058 ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells,
1059 ssc->scells, &rl);
1060
1061 /* We expect only one resource */
1062 rle = resource_list_find(&rl, SYS_RES_MEMORY, 0);
1063 if (rle == NULL)
1064 return (ENXIO);
1065
1066 /* Remap through ranges property */
1067 for (j = 0; j < ssc->nranges; j++) {
1068 if (rle->start >= ssc->ranges[j].bus &&
1069 rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) {
1070 rle->start -= ssc->ranges[j].bus;
1071 rle->start += ssc->ranges[j].host;
1072 rle->end -= ssc->ranges[j].bus;
1073 rle->end += ssc->ranges[j].host;
1074 }
1075 }
1076
1077 sc->sc_sram_base_pa = rle->start;
1078 sc->sc_sram_size = rle->count;
1079
1080 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */
1081 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
1082 if (sram_va == NULL)
1083 return (ENOMEM);
1084 sc->sc_sram_base_va = (vm_offset_t)sram_va;
1085
1086 return (0);
1087 }
1088
1089 struct ofw_compat_data cesa_devices[] = {
1090 { "mrvl,cesa", (uintptr_t)true },
1091 { "marvell,armada-38x-crypto", (uintptr_t)true },
1092 { NULL, 0 }
1093 };
1094
1095 static int
cesa_probe(device_t dev)1096 cesa_probe(device_t dev)
1097 {
1098
1099 if (!ofw_bus_status_okay(dev))
1100 return (ENXIO);
1101
1102 if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data)
1103 return (ENXIO);
1104
1105 device_set_desc(dev, "Marvell Cryptographic Engine and Security "
1106 "Accelerator");
1107
1108 return (BUS_PROBE_DEFAULT);
1109 }
1110
1111 static int
cesa_attach(device_t dev)1112 cesa_attach(device_t dev)
1113 {
1114 static int engine_idx = 0;
1115 struct simplebus_devinfo *ndi;
1116 struct resource_list *rl;
1117 struct cesa_softc *sc;
1118
1119 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
1120 return (cesa_attach_late(dev));
1121
1122 /*
1123 * Get simplebus_devinfo which contains
1124 * resource list filled with adresses and
1125 * interrupts read form FDT.
1126 * Let's correct it by splitting resources
1127 * for each engine.
1128 */
1129 if ((ndi = device_get_ivars(dev)) == NULL)
1130 return (ENXIO);
1131
1132 rl = &ndi->rl;
1133
1134 switch (engine_idx) {
1135 case 0:
1136 /* Update regs values */
1137 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA0_TDMA_ADDR,
1138 CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
1139 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA0_CESA_ADDR,
1140 CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
1141
1142 /* Remove unused interrupt */
1143 resource_list_delete(rl, SYS_RES_IRQ, 1);
1144 break;
1145
1146 case 1:
1147 /* Update regs values */
1148 resource_list_add(rl, SYS_RES_MEMORY, 0, CESA1_TDMA_ADDR,
1149 CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
1150 resource_list_add(rl, SYS_RES_MEMORY, 1, CESA1_CESA_ADDR,
1151 CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
1152
1153 /* Remove unused interrupt */
1154 resource_list_delete(rl, SYS_RES_IRQ, 0);
1155 resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0;
1156 break;
1157
1158 default:
1159 device_printf(dev, "Bad cesa engine_idx\n");
1160 return (ENXIO);
1161 }
1162
1163 sc = device_get_softc(dev);
1164 sc->sc_cesa_engine_id = engine_idx;
1165
1166 /*
1167 * Call simplebus_add_device only once.
1168 * It will create second cesa driver instance
1169 * with the same FDT node as first instance.
1170 * When second driver reach this function,
1171 * it will be configured to use second cesa engine
1172 */
1173 if (engine_idx == 0)
1174 simplebus_add_device(device_get_parent(dev), ofw_bus_get_node(dev),
1175 0, "cesa", 1, NULL);
1176
1177 engine_idx++;
1178
1179 return (cesa_attach_late(dev));
1180 }
1181
1182 static int
cesa_attach_late(device_t dev)1183 cesa_attach_late(device_t dev)
1184 {
1185 struct cesa_softc *sc;
1186 uint32_t d, r, val;
1187 int error;
1188 int i;
1189
1190 sc = device_get_softc(dev);
1191 sc->sc_blocked = 0;
1192 sc->sc_error = 0;
1193 sc->sc_dev = dev;
1194
1195 soc_id(&d, &r);
1196
1197 switch (d) {
1198 case MV_DEV_88F6281:
1199 case MV_DEV_88F6282:
1200 /* Check if CESA peripheral device has power turned on */
1201 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) ==
1202 CPU_PM_CTRL_CRYPTO) {
1203 device_printf(dev, "not powered on\n");
1204 return (ENXIO);
1205 }
1206 sc->sc_tperr = 0;
1207 break;
1208 case MV_DEV_88F6828:
1209 case MV_DEV_88F6820:
1210 case MV_DEV_88F6810:
1211 sc->sc_tperr = 0;
1212 break;
1213 case MV_DEV_MV78100:
1214 case MV_DEV_MV78100_Z0:
1215 /* Check if CESA peripheral device has power turned on */
1216 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) !=
1217 CPU_PM_CTRL_CRYPTO) {
1218 device_printf(dev, "not powered on\n");
1219 return (ENXIO);
1220 }
1221 sc->sc_tperr = CESA_ICR_TPERR;
1222 break;
1223 default:
1224 return (ENXIO);
1225 }
1226
1227 sc->sc_soc_id = d;
1228
1229 /* Initialize mutexes */
1230 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
1231 "CESA Shared Data", MTX_DEF);
1232 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
1233 "CESA TDMA Descriptors Pool", MTX_DEF);
1234 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
1235 "CESA SA Descriptors Pool", MTX_DEF);
1236 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
1237 "CESA Requests Pool", MTX_DEF);
1238 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
1239 "CESA Sessions Pool", MTX_DEF);
1240
1241 /* Allocate I/O and IRQ resources */
1242 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res);
1243 if (error) {
1244 device_printf(dev, "could not allocate resources\n");
1245 goto err0;
1246 }
1247
1248 /* Acquire SRAM base address */
1249 if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
1250 error = cesa_setup_sram(sc);
1251 else
1252 error = cesa_setup_sram_armada(sc);
1253
1254 if (error) {
1255 device_printf(dev, "could not setup SRAM\n");
1256 goto err1;
1257 }
1258
1259 /* Setup interrupt handler */
1260 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET |
1261 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie));
1262 if (error) {
1263 device_printf(dev, "could not setup engine completion irq\n");
1264 goto err2;
1265 }
1266
1267 /* Create DMA tag for processed data */
1268 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1269 1, 0, /* alignment, boundary */
1270 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1271 BUS_SPACE_MAXADDR, /* highaddr */
1272 NULL, NULL, /* filtfunc, filtfuncarg */
1273 CESA_MAX_REQUEST_SIZE, /* maxsize */
1274 CESA_MAX_FRAGMENTS, /* nsegments */
1275 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */
1276 NULL, NULL, /* lockfunc, lockfuncarg */
1277 &sc->sc_data_dtag); /* dmat */
1278 if (error)
1279 goto err3;
1280
1281 /* Initialize data structures: TDMA Descriptors Pool */
1282 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm,
1283 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc));
1284 if (error)
1285 goto err4;
1286
1287 STAILQ_INIT(&sc->sc_free_tdesc);
1288 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
1289 sc->sc_tdesc[i].ctd_cthd =
1290 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
1291 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
1292 (i * sizeof(struct cesa_tdma_hdesc));
1293 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
1294 ctd_stq);
1295 }
1296
1297 /* Initialize data structures: SA Descriptors Pool */
1298 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm,
1299 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc));
1300 if (error)
1301 goto err5;
1302
1303 STAILQ_INIT(&sc->sc_free_sdesc);
1304 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) {
1305 sc->sc_sdesc[i].csd_cshd =
1306 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
1307 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
1308 (i * sizeof(struct cesa_sa_hdesc));
1309 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
1310 csd_stq);
1311 }
1312
1313 /* Initialize data structures: Requests Pool */
1314 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm,
1315 CESA_REQUESTS * sizeof(struct cesa_sa_data));
1316 if (error)
1317 goto err6;
1318
1319 STAILQ_INIT(&sc->sc_free_requests);
1320 STAILQ_INIT(&sc->sc_ready_requests);
1321 STAILQ_INIT(&sc->sc_queued_requests);
1322 for (i = 0; i < CESA_REQUESTS; i++) {
1323 sc->sc_requests[i].cr_csd =
1324 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i;
1325 sc->sc_requests[i].cr_csd_paddr =
1326 sc->sc_requests_cdm.cdm_paddr +
1327 (i * sizeof(struct cesa_sa_data));
1328
1329 /* Preallocate DMA maps */
1330 error = bus_dmamap_create(sc->sc_data_dtag, 0,
1331 &sc->sc_requests[i].cr_dmap);
1332 if (error && i > 0) {
1333 i--;
1334 do {
1335 bus_dmamap_destroy(sc->sc_data_dtag,
1336 sc->sc_requests[i].cr_dmap);
1337 } while (i--);
1338
1339 goto err7;
1340 }
1341
1342 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i],
1343 cr_stq);
1344 }
1345
1346 /*
1347 * Initialize TDMA:
1348 * - Burst limit: 128 bytes,
1349 * - Outstanding reads enabled,
1350 * - No byte-swap.
1351 */
1352 val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 |
1353 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE;
1354
1355 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1356 sc->sc_soc_id == MV_DEV_88F6820 ||
1357 sc->sc_soc_id == MV_DEV_88F6810)
1358 val |= CESA_TDMA_NUM_OUTSTAND;
1359
1360 CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val);
1361
1362 /*
1363 * Initialize SA:
1364 * - SA descriptor is present at beginning of CESA SRAM,
1365 * - Multi-packet chain mode,
1366 * - Cooperation with TDMA enabled.
1367 */
1368 CESA_REG_WRITE(sc, CESA_SA_DPR, 0);
1369 CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA |
1370 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE);
1371
1372 /* Unmask interrupts */
1373 CESA_REG_WRITE(sc, CESA_ICR, 0);
1374 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr);
1375 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
1376 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS |
1377 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT |
1378 CESA_TDMA_EMR_DATA_ERROR);
1379
1380 /* Register in OCF */
1381 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session),
1382 CRYPTOCAP_F_HARDWARE);
1383 if (sc->sc_cid < 0) {
1384 device_printf(dev, "could not get crypto driver id\n");
1385 goto err8;
1386 }
1387
1388 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
1389 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
1390 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
1391 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
1392 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
1393 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
1394 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
1395 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1396 sc->sc_soc_id == MV_DEV_88F6820 ||
1397 sc->sc_soc_id == MV_DEV_88F6810)
1398 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
1399
1400 return (0);
1401 err8:
1402 for (i = 0; i < CESA_REQUESTS; i++)
1403 bus_dmamap_destroy(sc->sc_data_dtag,
1404 sc->sc_requests[i].cr_dmap);
1405 err7:
1406 cesa_free_dma_mem(&sc->sc_requests_cdm);
1407 err6:
1408 cesa_free_dma_mem(&sc->sc_sdesc_cdm);
1409 err5:
1410 cesa_free_dma_mem(&sc->sc_tdesc_cdm);
1411 err4:
1412 bus_dma_tag_destroy(sc->sc_data_dtag);
1413 err3:
1414 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
1415 err2:
1416 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1417 sc->sc_soc_id == MV_DEV_88F6820 ||
1418 sc->sc_soc_id == MV_DEV_88F6810)
1419 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
1420 err1:
1421 bus_release_resources(dev, cesa_res_spec, sc->sc_res);
1422 err0:
1423 mtx_destroy(&sc->sc_sessions_lock);
1424 mtx_destroy(&sc->sc_requests_lock);
1425 mtx_destroy(&sc->sc_sdesc_lock);
1426 mtx_destroy(&sc->sc_tdesc_lock);
1427 mtx_destroy(&sc->sc_sc_lock);
1428 return (ENXIO);
1429 }
1430
1431 static int
cesa_detach(device_t dev)1432 cesa_detach(device_t dev)
1433 {
1434 struct cesa_softc *sc;
1435 int i;
1436
1437 sc = device_get_softc(dev);
1438
1439 /* TODO: Wait for queued requests completion before shutdown. */
1440
1441 /* Mask interrupts */
1442 CESA_REG_WRITE(sc, CESA_ICM, 0);
1443 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0);
1444
1445 /* Unregister from OCF */
1446 crypto_unregister_all(sc->sc_cid);
1447
1448 /* Free DMA Maps */
1449 for (i = 0; i < CESA_REQUESTS; i++)
1450 bus_dmamap_destroy(sc->sc_data_dtag,
1451 sc->sc_requests[i].cr_dmap);
1452
1453 /* Free DMA Memory */
1454 cesa_free_dma_mem(&sc->sc_requests_cdm);
1455 cesa_free_dma_mem(&sc->sc_sdesc_cdm);
1456 cesa_free_dma_mem(&sc->sc_tdesc_cdm);
1457
1458 /* Free DMA Tag */
1459 bus_dma_tag_destroy(sc->sc_data_dtag);
1460
1461 /* Stop interrupt */
1462 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
1463
1464 /* Relase I/O and IRQ resources */
1465 bus_release_resources(dev, cesa_res_spec, sc->sc_res);
1466
1467 /* Unmap SRAM memory */
1468 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1469 sc->sc_soc_id == MV_DEV_88F6820 ||
1470 sc->sc_soc_id == MV_DEV_88F6810)
1471 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
1472
1473 /* Destroy mutexes */
1474 mtx_destroy(&sc->sc_sessions_lock);
1475 mtx_destroy(&sc->sc_requests_lock);
1476 mtx_destroy(&sc->sc_sdesc_lock);
1477 mtx_destroy(&sc->sc_tdesc_lock);
1478 mtx_destroy(&sc->sc_sc_lock);
1479
1480 return (0);
1481 }
1482
1483 static void
cesa_intr(void * arg)1484 cesa_intr(void *arg)
1485 {
1486 STAILQ_HEAD(, cesa_request) requests;
1487 struct cesa_request *cr, *tmp;
1488 struct cesa_softc *sc;
1489 uint32_t ecr, icr;
1490 int blocked;
1491
1492 sc = arg;
1493
1494 /* Ack interrupt */
1495 ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR);
1496 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
1497 icr = CESA_REG_READ(sc, CESA_ICR);
1498 CESA_REG_WRITE(sc, CESA_ICR, 0);
1499
1500 /* Check for TDMA errors */
1501 if (ecr & CESA_TDMA_ECR_MISS) {
1502 device_printf(sc->sc_dev, "TDMA Miss error detected!\n");
1503 sc->sc_error = EIO;
1504 }
1505
1506 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) {
1507 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n");
1508 sc->sc_error = EIO;
1509 }
1510
1511 if (ecr & CESA_TDMA_ECR_BOTH_HIT) {
1512 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n");
1513 sc->sc_error = EIO;
1514 }
1515
1516 if (ecr & CESA_TDMA_ECR_DATA_ERROR) {
1517 device_printf(sc->sc_dev, "TDMA Data error detected!\n");
1518 sc->sc_error = EIO;
1519 }
1520
1521 /* Check for CESA errors */
1522 if (icr & sc->sc_tperr) {
1523 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n");
1524 sc->sc_error = EIO;
1525 }
1526
1527 /* If there is nothing more to do, return */
1528 if ((icr & CESA_ICR_ACCTDMA) == 0)
1529 return;
1530
1531 /* Get all finished requests */
1532 CESA_LOCK(sc, requests);
1533 STAILQ_INIT(&requests);
1534 STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
1535 STAILQ_INIT(&sc->sc_queued_requests);
1536 CESA_UNLOCK(sc, requests);
1537
1538 /* Execute all ready requests */
1539 cesa_execute(sc);
1540
1541 /* Process completed requests */
1542 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD |
1543 BUS_DMASYNC_POSTWRITE);
1544
1545 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) {
1546 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap,
1547 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1548
1549 cr->cr_crp->crp_etype = sc->sc_error;
1550 if (cr->cr_mac)
1551 crypto_copyback(cr->cr_crp->crp_flags,
1552 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
1553 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
1554
1555 crypto_done(cr->cr_crp);
1556 cesa_free_request(sc, cr);
1557 }
1558
1559 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD |
1560 BUS_DMASYNC_PREWRITE);
1561
1562 sc->sc_error = 0;
1563
1564 /* Unblock driver if it ran out of resources */
1565 CESA_LOCK(sc, sc);
1566 blocked = sc->sc_blocked;
1567 sc->sc_blocked = 0;
1568 CESA_UNLOCK(sc, sc);
1569
1570 if (blocked)
1571 crypto_unblock(sc->sc_cid, blocked);
1572 }
1573
1574 static int
cesa_newsession(device_t dev,crypto_session_t cses,struct cryptoini * cri)1575 cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
1576 {
1577 struct cesa_session *cs;
1578 struct cesa_softc *sc;
1579 struct cryptoini *enc;
1580 struct cryptoini *mac;
1581 int error;
1582
1583 sc = device_get_softc(dev);
1584 enc = NULL;
1585 mac = NULL;
1586 error = 0;
1587
1588 /* Check and parse input */
1589 if (cesa_is_hash(cri->cri_alg))
1590 mac = cri;
1591 else
1592 enc = cri;
1593
1594 cri = cri->cri_next;
1595
1596 if (cri) {
1597 if (!enc && !cesa_is_hash(cri->cri_alg))
1598 enc = cri;
1599
1600 if (!mac && cesa_is_hash(cri->cri_alg))
1601 mac = cri;
1602
1603 if (cri->cri_next || !(enc && mac))
1604 return (EINVAL);
1605 }
1606
1607 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
1608 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
1609 return (E2BIG);
1610
1611 /* Allocate session */
1612 cs = crypto_get_driver_session(cses);
1613
1614 /* Prepare CESA configuration */
1615 cs->cs_config = 0;
1616 cs->cs_ivlen = 1;
1617 cs->cs_mblen = 1;
1618
1619 if (enc) {
1620 switch (enc->cri_alg) {
1621 case CRYPTO_AES_CBC:
1622 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
1623 cs->cs_ivlen = AES_BLOCK_LEN;
1624 break;
1625 case CRYPTO_DES_CBC:
1626 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
1627 cs->cs_ivlen = DES_BLOCK_LEN;
1628 break;
1629 case CRYPTO_3DES_CBC:
1630 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
1631 CESA_CSHD_CBC;
1632 cs->cs_ivlen = DES3_BLOCK_LEN;
1633 break;
1634 default:
1635 error = EINVAL;
1636 break;
1637 }
1638 }
1639
1640 if (!error && mac) {
1641 switch (mac->cri_alg) {
1642 case CRYPTO_MD5:
1643 cs->cs_mblen = 1;
1644 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
1645 mac->cri_mlen;
1646 cs->cs_config |= CESA_CSHD_MD5;
1647 break;
1648 case CRYPTO_MD5_HMAC:
1649 cs->cs_mblen = MD5_BLOCK_LEN;
1650 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
1651 mac->cri_mlen;
1652 cs->cs_config |= CESA_CSHD_MD5_HMAC;
1653 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
1654 cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
1655 break;
1656 case CRYPTO_SHA1:
1657 cs->cs_mblen = 1;
1658 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
1659 mac->cri_mlen;
1660 cs->cs_config |= CESA_CSHD_SHA1;
1661 break;
1662 case CRYPTO_SHA1_HMAC:
1663 cs->cs_mblen = SHA1_BLOCK_LEN;
1664 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
1665 mac->cri_mlen;
1666 cs->cs_config |= CESA_CSHD_SHA1_HMAC;
1667 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
1668 cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
1669 break;
1670 case CRYPTO_SHA2_256_HMAC:
1671 cs->cs_mblen = SHA2_256_BLOCK_LEN;
1672 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN :
1673 mac->cri_mlen;
1674 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
1675 break;
1676 default:
1677 error = EINVAL;
1678 break;
1679 }
1680 }
1681
1682 /* Save cipher key */
1683 if (!error && enc && enc->cri_key) {
1684 cs->cs_klen = enc->cri_klen / 8;
1685 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
1686 if (enc->cri_alg == CRYPTO_AES_CBC)
1687 error = cesa_prep_aes_key(cs);
1688 }
1689
1690 /* Save digest key */
1691 if (!error && mac && mac->cri_key)
1692 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
1693 mac->cri_klen / 8);
1694
1695 if (error)
1696 return (error);
1697
1698 return (0);
1699 }
1700
1701 static int
cesa_process(device_t dev,struct cryptop * crp,int hint)1702 cesa_process(device_t dev, struct cryptop *crp, int hint)
1703 {
1704 struct cesa_request *cr;
1705 struct cesa_session *cs;
1706 struct cryptodesc *crd;
1707 struct cryptodesc *enc;
1708 struct cryptodesc *mac;
1709 struct cesa_softc *sc;
1710 int error;
1711
1712 sc = device_get_softc(dev);
1713 crd = crp->crp_desc;
1714 enc = NULL;
1715 mac = NULL;
1716 error = 0;
1717
1718 cs = crypto_get_driver_session(crp->crp_session);
1719
1720 /* Check and parse input */
1721 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
1722 crp->crp_etype = E2BIG;
1723 crypto_done(crp);
1724 return (0);
1725 }
1726
1727 if (cesa_is_hash(crd->crd_alg))
1728 mac = crd;
1729 else
1730 enc = crd;
1731
1732 crd = crd->crd_next;
1733
1734 if (crd) {
1735 if (!enc && !cesa_is_hash(crd->crd_alg))
1736 enc = crd;
1737
1738 if (!mac && cesa_is_hash(crd->crd_alg))
1739 mac = crd;
1740
1741 if (crd->crd_next || !(enc && mac)) {
1742 crp->crp_etype = EINVAL;
1743 crypto_done(crp);
1744 return (0);
1745 }
1746 }
1747
1748 /*
1749 * Get request descriptor. Block driver if there is no free
1750 * descriptors in pool.
1751 */
1752 cr = cesa_alloc_request(sc);
1753 if (!cr) {
1754 CESA_LOCK(sc, sc);
1755 sc->sc_blocked = CRYPTO_SYMQ;
1756 CESA_UNLOCK(sc, sc);
1757 return (ERESTART);
1758 }
1759
1760 /* Prepare request */
1761 cr->cr_crp = crp;
1762 cr->cr_enc = enc;
1763 cr->cr_mac = mac;
1764 cr->cr_cs = cs;
1765
1766 CESA_LOCK(sc, sessions);
1767 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1768
1769 if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
1770 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1771 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
1772 else
1773 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
1774
1775 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
1776 crypto_copyback(crp->crp_flags, crp->crp_buf,
1777 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
1778 } else if (enc) {
1779 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1780 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
1781 else
1782 crypto_copydata(crp->crp_flags, crp->crp_buf,
1783 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
1784 }
1785
1786 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
1787 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
1788 cs->cs_klen = enc->crd_klen / 8;
1789 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
1790 if (enc->crd_alg == CRYPTO_AES_CBC)
1791 error = cesa_prep_aes_key(cs);
1792 } else
1793 error = E2BIG;
1794 }
1795
1796 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
1797 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
1798 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
1799 mac->crd_klen / 8);
1800 else
1801 error = E2BIG;
1802 }
1803
1804 /* Convert request to chain of TDMA and SA descriptors */
1805 if (!error)
1806 error = cesa_create_chain(sc, cr);
1807
1808 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1809 CESA_UNLOCK(sc, sessions);
1810
1811 if (error) {
1812 cesa_free_request(sc, cr);
1813 crp->crp_etype = error;
1814 crypto_done(crp);
1815 return (0);
1816 }
1817
1818 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD |
1819 BUS_DMASYNC_PREWRITE);
1820
1821 /* Enqueue request to execution */
1822 cesa_enqueue_request(sc, cr);
1823
1824 /* Start execution, if we have no more requests in queue */
1825 if ((hint & CRYPTO_HINT_MORE) == 0)
1826 cesa_execute(sc);
1827
1828 return (0);
1829 }
1830