1 /* $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $ */
2
3 /*-
4 * SPDX-License-Identifier: BSD-3-Clause
5 *
6 * Invertex AEON / Hifn 7751 driver
7 * Copyright (c) 1999 Invertex Inc. All rights reserved.
8 * Copyright (c) 1999 Theo de Raadt
9 * Copyright (c) 2000-2001 Network Security Technologies, Inc.
10 * http://www.netsec.net
11 * Copyright (c) 2003 Hifn Inc.
12 *
13 * This driver is based on a previous driver by Invertex, for which they
14 * requested: Please send any comments, feedback, bug-fixes, or feature
15 * requests to [email protected].
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * 1. Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * 2. Redistributions in binary form must reproduce the above copyright
24 * notice, this list of conditions and the following disclaimer in the
25 * documentation and/or other materials provided with the distribution.
26 * 3. The name of the author may not be used to endorse or promote products
27 * derived from this software without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
30 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
32 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
33 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
34 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
35 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
36 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
37 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
38 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 *
40 * Effort sponsored in part by the Defense Advanced Research Projects
41 * Agency (DARPA) and Air Force Research Laboratory, Air Force
42 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
43 */
44
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47
48 /*
49 * Driver for various Hifn encryption processors.
50 */
51 #include "opt_hifn.h"
52
53 #include <sys/param.h>
54 #include <sys/systm.h>
55 #include <sys/proc.h>
56 #include <sys/errno.h>
57 #include <sys/malloc.h>
58 #include <sys/kernel.h>
59 #include <sys/module.h>
60 #include <sys/mbuf.h>
61 #include <sys/lock.h>
62 #include <sys/mutex.h>
63 #include <sys/sysctl.h>
64
65 #include <vm/vm.h>
66 #include <vm/pmap.h>
67
68 #include <machine/bus.h>
69 #include <machine/resource.h>
70 #include <sys/bus.h>
71 #include <sys/rman.h>
72
73 #include <opencrypto/cryptodev.h>
74 #include <sys/random.h>
75 #include <sys/kobj.h>
76
77 #include "cryptodev_if.h"
78
79 #include <dev/pci/pcivar.h>
80 #include <dev/pci/pcireg.h>
81
82 #ifdef HIFN_RNDTEST
83 #include <dev/rndtest/rndtest.h>
84 #endif
85 #include <dev/hifn/hifn7751reg.h>
86 #include <dev/hifn/hifn7751var.h>
87
88 #ifdef HIFN_VULCANDEV
89 #include <sys/conf.h>
90 #include <sys/uio.h>
91
92 static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
93 #endif
94
95 /*
96 * Prototypes and count for the pci_device structure
97 */
98 static int hifn_probe(device_t);
99 static int hifn_attach(device_t);
100 static int hifn_detach(device_t);
101 static int hifn_suspend(device_t);
102 static int hifn_resume(device_t);
103 static int hifn_shutdown(device_t);
104
105 static int hifn_newsession(device_t, crypto_session_t, struct cryptoini *);
106 static int hifn_process(device_t, struct cryptop *, int);
107
108 static device_method_t hifn_methods[] = {
109 /* Device interface */
110 DEVMETHOD(device_probe, hifn_probe),
111 DEVMETHOD(device_attach, hifn_attach),
112 DEVMETHOD(device_detach, hifn_detach),
113 DEVMETHOD(device_suspend, hifn_suspend),
114 DEVMETHOD(device_resume, hifn_resume),
115 DEVMETHOD(device_shutdown, hifn_shutdown),
116
117 /* crypto device methods */
118 DEVMETHOD(cryptodev_newsession, hifn_newsession),
119 DEVMETHOD(cryptodev_process, hifn_process),
120
121 DEVMETHOD_END
122 };
123 static driver_t hifn_driver = {
124 "hifn",
125 hifn_methods,
126 sizeof (struct hifn_softc)
127 };
128 static devclass_t hifn_devclass;
129
130 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
131 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
132 #ifdef HIFN_RNDTEST
133 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
134 #endif
135
136 static void hifn_reset_board(struct hifn_softc *, int);
137 static void hifn_reset_puc(struct hifn_softc *);
138 static void hifn_puc_wait(struct hifn_softc *);
139 static int hifn_enable_crypto(struct hifn_softc *);
140 static void hifn_set_retry(struct hifn_softc *sc);
141 static void hifn_init_dma(struct hifn_softc *);
142 static void hifn_init_pci_registers(struct hifn_softc *);
143 static int hifn_sramsize(struct hifn_softc *);
144 static int hifn_dramsize(struct hifn_softc *);
145 static int hifn_ramtype(struct hifn_softc *);
146 static void hifn_sessions(struct hifn_softc *);
147 static void hifn_intr(void *);
148 static u_int hifn_write_command(struct hifn_command *, u_int8_t *);
149 static u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
150 static void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
151 static int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
152 static int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
153 static int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
154 static int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
155 static int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
156 static int hifn_init_pubrng(struct hifn_softc *);
157 static void hifn_rng(void *);
158 static void hifn_tick(void *);
159 static void hifn_abort(struct hifn_softc *);
160 static void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
161
162 static void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
163 static void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
164
165 static __inline u_int32_t
READ_REG_0(struct hifn_softc * sc,bus_size_t reg)166 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
167 {
168 u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
169 sc->sc_bar0_lastreg = (bus_size_t) -1;
170 return (v);
171 }
172 #define WRITE_REG_0(sc, reg, val) hifn_write_reg_0(sc, reg, val)
173
174 static __inline u_int32_t
READ_REG_1(struct hifn_softc * sc,bus_size_t reg)175 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
176 {
177 u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
178 sc->sc_bar1_lastreg = (bus_size_t) -1;
179 return (v);
180 }
181 #define WRITE_REG_1(sc, reg, val) hifn_write_reg_1(sc, reg, val)
182
183 static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0,
184 "Hifn driver parameters");
185
186 #ifdef HIFN_DEBUG
187 static int hifn_debug = 0;
188 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
189 0, "control debugging msgs");
190 #endif
191
192 static struct hifn_stats hifnstats;
193 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
194 hifn_stats, "driver statistics");
195 static int hifn_maxbatch = 1;
196 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
197 0, "max ops to batch w/o interrupt");
198
199 /*
200 * Probe for a supported device. The PCI vendor and device
201 * IDs are used to detect devices we know how to handle.
202 */
203 static int
hifn_probe(device_t dev)204 hifn_probe(device_t dev)
205 {
206 if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
207 pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
208 return (BUS_PROBE_DEFAULT);
209 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
210 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
211 pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
212 pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
213 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
214 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
215 return (BUS_PROBE_DEFAULT);
216 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
217 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
218 return (BUS_PROBE_DEFAULT);
219 return (ENXIO);
220 }
221
222 static void
hifn_dmamap_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)223 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
224 {
225 bus_addr_t *paddr = (bus_addr_t*) arg;
226 *paddr = segs->ds_addr;
227 }
228
229 static const char*
hifn_partname(struct hifn_softc * sc)230 hifn_partname(struct hifn_softc *sc)
231 {
232 /* XXX sprintf numbers when not decoded */
233 switch (pci_get_vendor(sc->sc_dev)) {
234 case PCI_VENDOR_HIFN:
235 switch (pci_get_device(sc->sc_dev)) {
236 case PCI_PRODUCT_HIFN_6500: return "Hifn 6500";
237 case PCI_PRODUCT_HIFN_7751: return "Hifn 7751";
238 case PCI_PRODUCT_HIFN_7811: return "Hifn 7811";
239 case PCI_PRODUCT_HIFN_7951: return "Hifn 7951";
240 case PCI_PRODUCT_HIFN_7955: return "Hifn 7955";
241 case PCI_PRODUCT_HIFN_7956: return "Hifn 7956";
242 }
243 return "Hifn unknown-part";
244 case PCI_VENDOR_INVERTEX:
245 switch (pci_get_device(sc->sc_dev)) {
246 case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
247 }
248 return "Invertex unknown-part";
249 case PCI_VENDOR_NETSEC:
250 switch (pci_get_device(sc->sc_dev)) {
251 case PCI_PRODUCT_NETSEC_7751: return "NetSec 7751";
252 }
253 return "NetSec unknown-part";
254 }
255 return "Unknown-vendor unknown-part";
256 }
257
258 static void
default_harvest(struct rndtest_state * rsp,void * buf,u_int count)259 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
260 {
261 /* MarkM: FIX!! Check that this does not swamp the harvester! */
262 random_harvest_queue(buf, count, RANDOM_PURE_HIFN);
263 }
264
265 static u_int
checkmaxmin(device_t dev,const char * what,u_int v,u_int min,u_int max)266 checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
267 {
268 if (v > max) {
269 device_printf(dev, "Warning, %s %u out of range, "
270 "using max %u\n", what, v, max);
271 v = max;
272 } else if (v < min) {
273 device_printf(dev, "Warning, %s %u out of range, "
274 "using min %u\n", what, v, min);
275 v = min;
276 }
277 return v;
278 }
279
280 /*
281 * Select PLL configuration for 795x parts. This is complicated in
282 * that we cannot determine the optimal parameters without user input.
283 * The reference clock is derived from an external clock through a
284 * multiplier. The external clock is either the host bus (i.e. PCI)
285 * or an external clock generator. When using the PCI bus we assume
286 * the clock is either 33 or 66 MHz; for an external source we cannot
287 * tell the speed.
288 *
289 * PLL configuration is done with a string: "pci" for PCI bus, or "ext"
290 * for an external source, followed by the frequency. We calculate
291 * the appropriate multiplier and PLL register contents accordingly.
292 * When no configuration is given we default to "pci66" since that
293 * always will allow the card to work. If a card is using the PCI
294 * bus clock and in a 33MHz slot then it will be operating at half
295 * speed until the correct information is provided.
296 *
297 * We use a default setting of "ext66" because according to Mike Ham
298 * of HiFn, almost every board in existence has an external crystal
299 * populated at 66Mhz. Using PCI can be a problem on modern motherboards,
300 * because PCI33 can have clocks from 0 to 33Mhz, and some have
301 * non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
302 */
303 static void
hifn_getpllconfig(device_t dev,u_int * pll)304 hifn_getpllconfig(device_t dev, u_int *pll)
305 {
306 const char *pllspec;
307 u_int freq, mul, fl, fh;
308 u_int32_t pllconfig;
309 char *nxt;
310
311 if (resource_string_value("hifn", device_get_unit(dev),
312 "pllconfig", &pllspec))
313 pllspec = "ext66";
314 fl = 33, fh = 66;
315 pllconfig = 0;
316 if (strncmp(pllspec, "ext", 3) == 0) {
317 pllspec += 3;
318 pllconfig |= HIFN_PLL_REF_SEL;
319 switch (pci_get_device(dev)) {
320 case PCI_PRODUCT_HIFN_7955:
321 case PCI_PRODUCT_HIFN_7956:
322 fl = 20, fh = 100;
323 break;
324 #ifdef notyet
325 case PCI_PRODUCT_HIFN_7954:
326 fl = 20, fh = 66;
327 break;
328 #endif
329 }
330 } else if (strncmp(pllspec, "pci", 3) == 0)
331 pllspec += 3;
332 freq = strtoul(pllspec, &nxt, 10);
333 if (nxt == pllspec)
334 freq = 66;
335 else
336 freq = checkmaxmin(dev, "frequency", freq, fl, fh);
337 /*
338 * Calculate multiplier. We target a Fck of 266 MHz,
339 * allowing only even values, possibly rounded down.
340 * Multipliers > 8 must set the charge pump current.
341 */
342 mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
343 pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
344 if (mul > 8)
345 pllconfig |= HIFN_PLL_IS;
346 *pll = pllconfig;
347 }
348
349 /*
350 * Attach an interface that successfully probed.
351 */
352 static int
hifn_attach(device_t dev)353 hifn_attach(device_t dev)
354 {
355 struct hifn_softc *sc = device_get_softc(dev);
356 caddr_t kva;
357 int rseg, rid;
358 char rbase;
359 u_int16_t ena, rev;
360
361 sc->sc_dev = dev;
362
363 mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
364
365 /* XXX handle power management */
366
367 /*
368 * The 7951 and 795x have a random number generator and
369 * public key support; note this.
370 */
371 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
372 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
373 pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
374 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
375 sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
376 /*
377 * The 7811 has a random number generator and
378 * we also note it's identity 'cuz of some quirks.
379 */
380 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
381 pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
382 sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
383
384 /*
385 * The 795x parts support AES.
386 */
387 if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
388 (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
389 pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
390 sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
391 /*
392 * Select PLL configuration. This depends on the
393 * bus and board design and must be manually configured
394 * if the default setting is unacceptable.
395 */
396 hifn_getpllconfig(dev, &sc->sc_pllconfig);
397 }
398
399 /*
400 * Setup PCI resources. Note that we record the bus
401 * tag and handle for each register mapping, this is
402 * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
403 * and WRITE_REG_1 macros throughout the driver.
404 */
405 pci_enable_busmaster(dev);
406
407 rid = HIFN_BAR0;
408 sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
409 RF_ACTIVE);
410 if (sc->sc_bar0res == NULL) {
411 device_printf(dev, "cannot map bar%d register space\n", 0);
412 goto fail_pci;
413 }
414 sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
415 sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
416 sc->sc_bar0_lastreg = (bus_size_t) -1;
417
418 rid = HIFN_BAR1;
419 sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
420 RF_ACTIVE);
421 if (sc->sc_bar1res == NULL) {
422 device_printf(dev, "cannot map bar%d register space\n", 1);
423 goto fail_io0;
424 }
425 sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
426 sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
427 sc->sc_bar1_lastreg = (bus_size_t) -1;
428
429 hifn_set_retry(sc);
430
431 /*
432 * Setup the area where the Hifn DMA's descriptors
433 * and associated data structures.
434 */
435 if (bus_dma_tag_create(bus_get_dma_tag(dev), /* PCI parent */
436 1, 0, /* alignment,boundary */
437 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
438 BUS_SPACE_MAXADDR, /* highaddr */
439 NULL, NULL, /* filter, filterarg */
440 HIFN_MAX_DMALEN, /* maxsize */
441 MAX_SCATTER, /* nsegments */
442 HIFN_MAX_SEGLEN, /* maxsegsize */
443 BUS_DMA_ALLOCNOW, /* flags */
444 NULL, /* lockfunc */
445 NULL, /* lockarg */
446 &sc->sc_dmat)) {
447 device_printf(dev, "cannot allocate DMA tag\n");
448 goto fail_io1;
449 }
450 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
451 device_printf(dev, "cannot create dma map\n");
452 bus_dma_tag_destroy(sc->sc_dmat);
453 goto fail_io1;
454 }
455 if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
456 device_printf(dev, "cannot alloc dma buffer\n");
457 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
458 bus_dma_tag_destroy(sc->sc_dmat);
459 goto fail_io1;
460 }
461 if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
462 sizeof (*sc->sc_dma),
463 hifn_dmamap_cb, &sc->sc_dma_physaddr,
464 BUS_DMA_NOWAIT)) {
465 device_printf(dev, "cannot load dma map\n");
466 bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
467 bus_dma_tag_destroy(sc->sc_dmat);
468 goto fail_io1;
469 }
470 sc->sc_dma = (struct hifn_dma *)kva;
471 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
472
473 KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
474 KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
475 KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
476 KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
477
478 /*
479 * Reset the board and do the ``secret handshake''
480 * to enable the crypto support. Then complete the
481 * initialization procedure by setting up the interrupt
482 * and hooking in to the system crypto support so we'll
483 * get used for system services like the crypto device,
484 * IPsec, RNG device, etc.
485 */
486 hifn_reset_board(sc, 0);
487
488 if (hifn_enable_crypto(sc) != 0) {
489 device_printf(dev, "crypto enabling failed\n");
490 goto fail_mem;
491 }
492 hifn_reset_puc(sc);
493
494 hifn_init_dma(sc);
495 hifn_init_pci_registers(sc);
496
497 /* XXX can't dynamically determine ram type for 795x; force dram */
498 if (sc->sc_flags & HIFN_IS_7956)
499 sc->sc_drammodel = 1;
500 else if (hifn_ramtype(sc))
501 goto fail_mem;
502
503 if (sc->sc_drammodel == 0)
504 hifn_sramsize(sc);
505 else
506 hifn_dramsize(sc);
507
508 /*
509 * Workaround for NetSec 7751 rev A: half ram size because two
510 * of the address lines were left floating
511 */
512 if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
513 pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
514 pci_get_revid(dev) == 0x61) /*XXX???*/
515 sc->sc_ramsize >>= 1;
516
517 /*
518 * Arrange the interrupt line.
519 */
520 rid = 0;
521 sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
522 RF_SHAREABLE|RF_ACTIVE);
523 if (sc->sc_irq == NULL) {
524 device_printf(dev, "could not map interrupt\n");
525 goto fail_mem;
526 }
527 /*
528 * NB: Network code assumes we are blocked with splimp()
529 * so make sure the IRQ is marked appropriately.
530 */
531 if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
532 NULL, hifn_intr, sc, &sc->sc_intrhand)) {
533 device_printf(dev, "could not setup interrupt\n");
534 goto fail_intr2;
535 }
536
537 hifn_sessions(sc);
538
539 /*
540 * NB: Keep only the low 16 bits; this masks the chip id
541 * from the 7951.
542 */
543 rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
544
545 rseg = sc->sc_ramsize / 1024;
546 rbase = 'K';
547 if (sc->sc_ramsize >= (1024 * 1024)) {
548 rbase = 'M';
549 rseg /= 1024;
550 }
551 device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
552 hifn_partname(sc), rev,
553 rseg, rbase, sc->sc_drammodel ? 'd' : 's');
554 if (sc->sc_flags & HIFN_IS_7956)
555 printf(", pll=0x%x<%s clk, %ux mult>",
556 sc->sc_pllconfig,
557 sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
558 2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
559 printf("\n");
560
561 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct hifn_session),
562 CRYPTOCAP_F_HARDWARE);
563 if (sc->sc_cid < 0) {
564 device_printf(dev, "could not get crypto driver id\n");
565 goto fail_intr;
566 }
567
568 WRITE_REG_0(sc, HIFN_0_PUCNFG,
569 READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
570 ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
571
572 switch (ena) {
573 case HIFN_PUSTAT_ENA_2:
574 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
575 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
576 if (sc->sc_flags & HIFN_HAS_AES)
577 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
578 /*FALLTHROUGH*/
579 case HIFN_PUSTAT_ENA_1:
580 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
581 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
582 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
583 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
584 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
585 break;
586 }
587
588 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
589 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
590
591 if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
592 hifn_init_pubrng(sc);
593
594 callout_init(&sc->sc_tickto, 1);
595 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
596
597 return (0);
598
599 fail_intr:
600 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
601 fail_intr2:
602 /* XXX don't store rid */
603 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
604 fail_mem:
605 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
606 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
607 bus_dma_tag_destroy(sc->sc_dmat);
608
609 /* Turn off DMA polling */
610 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
611 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
612 fail_io1:
613 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
614 fail_io0:
615 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
616 fail_pci:
617 mtx_destroy(&sc->sc_mtx);
618 return (ENXIO);
619 }
620
621 /*
622 * Detach an interface that successfully probed.
623 */
624 static int
hifn_detach(device_t dev)625 hifn_detach(device_t dev)
626 {
627 struct hifn_softc *sc = device_get_softc(dev);
628
629 KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
630
631 /* disable interrupts */
632 WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
633
634 /*XXX other resources */
635 callout_stop(&sc->sc_tickto);
636 callout_stop(&sc->sc_rngto);
637 #ifdef HIFN_RNDTEST
638 if (sc->sc_rndtest)
639 rndtest_detach(sc->sc_rndtest);
640 #endif
641
642 /* Turn off DMA polling */
643 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
644 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
645
646 crypto_unregister_all(sc->sc_cid);
647
648 bus_generic_detach(dev); /*XXX should be no children, right? */
649
650 bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
651 /* XXX don't store rid */
652 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
653
654 bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
655 bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
656 bus_dma_tag_destroy(sc->sc_dmat);
657
658 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
659 bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
660
661 mtx_destroy(&sc->sc_mtx);
662
663 return (0);
664 }
665
666 /*
667 * Stop all chip I/O so that the kernel's probe routines don't
668 * get confused by errant DMAs when rebooting.
669 */
670 static int
hifn_shutdown(device_t dev)671 hifn_shutdown(device_t dev)
672 {
673 #ifdef notyet
674 hifn_stop(device_get_softc(dev));
675 #endif
676 return (0);
677 }
678
679 /*
680 * Device suspend routine. Stop the interface and save some PCI
681 * settings in case the BIOS doesn't restore them properly on
682 * resume.
683 */
684 static int
hifn_suspend(device_t dev)685 hifn_suspend(device_t dev)
686 {
687 struct hifn_softc *sc = device_get_softc(dev);
688 #ifdef notyet
689 hifn_stop(sc);
690 #endif
691 sc->sc_suspended = 1;
692
693 return (0);
694 }
695
696 /*
697 * Device resume routine. Restore some PCI settings in case the BIOS
698 * doesn't, re-enable busmastering, and restart the interface if
699 * appropriate.
700 */
701 static int
hifn_resume(device_t dev)702 hifn_resume(device_t dev)
703 {
704 struct hifn_softc *sc = device_get_softc(dev);
705 #ifdef notyet
706 /* reinitialize interface if necessary */
707 if (ifp->if_flags & IFF_UP)
708 rl_init(sc);
709 #endif
710 sc->sc_suspended = 0;
711
712 return (0);
713 }
714
715 static int
hifn_init_pubrng(struct hifn_softc * sc)716 hifn_init_pubrng(struct hifn_softc *sc)
717 {
718 u_int32_t r;
719 int i;
720
721 #ifdef HIFN_RNDTEST
722 sc->sc_rndtest = rndtest_attach(sc->sc_dev);
723 if (sc->sc_rndtest)
724 sc->sc_harvest = rndtest_harvest;
725 else
726 sc->sc_harvest = default_harvest;
727 #else
728 sc->sc_harvest = default_harvest;
729 #endif
730 if ((sc->sc_flags & HIFN_IS_7811) == 0) {
731 /* Reset 7951 public key/rng engine */
732 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
733 READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
734
735 for (i = 0; i < 100; i++) {
736 DELAY(1000);
737 if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
738 HIFN_PUBRST_RESET) == 0)
739 break;
740 }
741
742 if (i == 100) {
743 device_printf(sc->sc_dev, "public key init failed\n");
744 return (1);
745 }
746 }
747
748 /* Enable the rng, if available */
749 if (sc->sc_flags & HIFN_HAS_RNG) {
750 if (sc->sc_flags & HIFN_IS_7811) {
751 r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
752 if (r & HIFN_7811_RNGENA_ENA) {
753 r &= ~HIFN_7811_RNGENA_ENA;
754 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
755 }
756 WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
757 HIFN_7811_RNGCFG_DEFL);
758 r |= HIFN_7811_RNGENA_ENA;
759 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
760 } else
761 WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
762 READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
763 HIFN_RNGCFG_ENA);
764
765 sc->sc_rngfirst = 1;
766 if (hz >= 100)
767 sc->sc_rnghz = hz / 100;
768 else
769 sc->sc_rnghz = 1;
770 callout_init(&sc->sc_rngto, 1);
771 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
772 }
773
774 /* Enable public key engine, if available */
775 if (sc->sc_flags & HIFN_HAS_PUBLIC) {
776 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
777 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
778 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
779 #ifdef HIFN_VULCANDEV
780 sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0,
781 UID_ROOT, GID_WHEEL, 0666,
782 "vulcanpk");
783 sc->sc_pkdev->si_drv1 = sc;
784 #endif
785 }
786
787 return (0);
788 }
789
790 static void
hifn_rng(void * vsc)791 hifn_rng(void *vsc)
792 {
793 #define RANDOM_BITS(n) (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
794 struct hifn_softc *sc = vsc;
795 u_int32_t sts, num[2];
796 int i;
797
798 if (sc->sc_flags & HIFN_IS_7811) {
799 /* ONLY VALID ON 7811!!!! */
800 for (i = 0; i < 5; i++) {
801 sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
802 if (sts & HIFN_7811_RNGSTS_UFL) {
803 device_printf(sc->sc_dev,
804 "RNG underflow: disabling\n");
805 return;
806 }
807 if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
808 break;
809
810 /*
811 * There are at least two words in the RNG FIFO
812 * at this point.
813 */
814 num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
815 num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
816 /* NB: discard first data read */
817 if (sc->sc_rngfirst)
818 sc->sc_rngfirst = 0;
819 else
820 (*sc->sc_harvest)(sc->sc_rndtest,
821 num, sizeof (num));
822 }
823 } else {
824 num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
825
826 /* NB: discard first data read */
827 if (sc->sc_rngfirst)
828 sc->sc_rngfirst = 0;
829 else
830 (*sc->sc_harvest)(sc->sc_rndtest,
831 num, sizeof (num[0]));
832 }
833
834 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
835 #undef RANDOM_BITS
836 }
837
838 static void
hifn_puc_wait(struct hifn_softc * sc)839 hifn_puc_wait(struct hifn_softc *sc)
840 {
841 int i;
842 int reg = HIFN_0_PUCTRL;
843
844 if (sc->sc_flags & HIFN_IS_7956) {
845 reg = HIFN_0_PUCTRL2;
846 }
847
848 for (i = 5000; i > 0; i--) {
849 DELAY(1);
850 if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
851 break;
852 }
853 if (!i)
854 device_printf(sc->sc_dev, "proc unit did not reset\n");
855 }
856
857 /*
858 * Reset the processing unit.
859 */
860 static void
hifn_reset_puc(struct hifn_softc * sc)861 hifn_reset_puc(struct hifn_softc *sc)
862 {
863 /* Reset processing unit */
864 int reg = HIFN_0_PUCTRL;
865
866 if (sc->sc_flags & HIFN_IS_7956) {
867 reg = HIFN_0_PUCTRL2;
868 }
869 WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
870
871 hifn_puc_wait(sc);
872 }
873
874 /*
875 * Set the Retry and TRDY registers; note that we set them to
876 * zero because the 7811 locks up when forced to retry (section
877 * 3.6 of "Specification Update SU-0014-04". Not clear if we
878 * should do this for all Hifn parts, but it doesn't seem to hurt.
879 */
880 static void
hifn_set_retry(struct hifn_softc * sc)881 hifn_set_retry(struct hifn_softc *sc)
882 {
883 /* NB: RETRY only responds to 8-bit reads/writes */
884 pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
885 pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 1);
886 }
887
888 /*
889 * Resets the board. Values in the regesters are left as is
890 * from the reset (i.e. initial values are assigned elsewhere).
891 */
892 static void
hifn_reset_board(struct hifn_softc * sc,int full)893 hifn_reset_board(struct hifn_softc *sc, int full)
894 {
895 u_int32_t reg;
896
897 /*
898 * Set polling in the DMA configuration register to zero. 0x7 avoids
899 * resetting the board and zeros out the other fields.
900 */
901 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
902 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
903
904 /*
905 * Now that polling has been disabled, we have to wait 1 ms
906 * before resetting the board.
907 */
908 DELAY(1000);
909
910 /* Reset the DMA unit */
911 if (full) {
912 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
913 DELAY(1000);
914 } else {
915 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
916 HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
917 hifn_reset_puc(sc);
918 }
919
920 KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
921 bzero(sc->sc_dma, sizeof(*sc->sc_dma));
922
923 /* Bring dma unit out of reset */
924 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
925 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
926
927 hifn_puc_wait(sc);
928 hifn_set_retry(sc);
929
930 if (sc->sc_flags & HIFN_IS_7811) {
931 for (reg = 0; reg < 1000; reg++) {
932 if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
933 HIFN_MIPSRST_CRAMINIT)
934 break;
935 DELAY(1000);
936 }
937 if (reg == 1000)
938 printf(": cram init timeout\n");
939 } else {
940 /* set up DMA configuration register #2 */
941 /* turn off all PK and BAR0 swaps */
942 WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
943 (3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
944 (3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
945 (2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
946 (2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
947 }
948
949 }
950
951 static u_int32_t
hifn_next_signature(u_int32_t a,u_int cnt)952 hifn_next_signature(u_int32_t a, u_int cnt)
953 {
954 int i;
955 u_int32_t v;
956
957 for (i = 0; i < cnt; i++) {
958
959 /* get the parity */
960 v = a & 0x80080125;
961 v ^= v >> 16;
962 v ^= v >> 8;
963 v ^= v >> 4;
964 v ^= v >> 2;
965 v ^= v >> 1;
966
967 a = (v & 1) ^ (a << 1);
968 }
969
970 return a;
971 }
972
973 struct pci2id {
974 u_short pci_vendor;
975 u_short pci_prod;
976 char card_id[13];
977 };
978 static struct pci2id pci2id[] = {
979 {
980 PCI_VENDOR_HIFN,
981 PCI_PRODUCT_HIFN_7951,
982 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
983 0x00, 0x00, 0x00, 0x00, 0x00 }
984 }, {
985 PCI_VENDOR_HIFN,
986 PCI_PRODUCT_HIFN_7955,
987 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
988 0x00, 0x00, 0x00, 0x00, 0x00 }
989 }, {
990 PCI_VENDOR_HIFN,
991 PCI_PRODUCT_HIFN_7956,
992 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
993 0x00, 0x00, 0x00, 0x00, 0x00 }
994 }, {
995 PCI_VENDOR_NETSEC,
996 PCI_PRODUCT_NETSEC_7751,
997 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
998 0x00, 0x00, 0x00, 0x00, 0x00 }
999 }, {
1000 PCI_VENDOR_INVERTEX,
1001 PCI_PRODUCT_INVERTEX_AEON,
1002 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1003 0x00, 0x00, 0x00, 0x00, 0x00 }
1004 }, {
1005 PCI_VENDOR_HIFN,
1006 PCI_PRODUCT_HIFN_7811,
1007 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1008 0x00, 0x00, 0x00, 0x00, 0x00 }
1009 }, {
1010 /*
1011 * Other vendors share this PCI ID as well, such as
1012 * http://www.powercrypt.com, and obviously they also
1013 * use the same key.
1014 */
1015 PCI_VENDOR_HIFN,
1016 PCI_PRODUCT_HIFN_7751,
1017 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1018 0x00, 0x00, 0x00, 0x00, 0x00 }
1019 },
1020 };
1021
1022 /*
1023 * Checks to see if crypto is already enabled. If crypto isn't enable,
1024 * "hifn_enable_crypto" is called to enable it. The check is important,
1025 * as enabling crypto twice will lock the board.
1026 */
1027 static int
hifn_enable_crypto(struct hifn_softc * sc)1028 hifn_enable_crypto(struct hifn_softc *sc)
1029 {
1030 u_int32_t dmacfg, ramcfg, encl, addr, i;
1031 char *offtbl = NULL;
1032
1033 for (i = 0; i < nitems(pci2id); i++) {
1034 if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
1035 pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
1036 offtbl = pci2id[i].card_id;
1037 break;
1038 }
1039 }
1040 if (offtbl == NULL) {
1041 device_printf(sc->sc_dev, "Unknown card!\n");
1042 return (1);
1043 }
1044
1045 ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1046 dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
1047
1048 /*
1049 * The RAM config register's encrypt level bit needs to be set before
1050 * every read performed on the encryption level register.
1051 */
1052 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1053
1054 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1055
1056 /*
1057 * Make sure we don't re-unlock. Two unlocks kills chip until the
1058 * next reboot.
1059 */
1060 if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
1061 #ifdef HIFN_DEBUG
1062 if (hifn_debug)
1063 device_printf(sc->sc_dev,
1064 "Strong crypto already enabled!\n");
1065 #endif
1066 goto report;
1067 }
1068
1069 if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
1070 #ifdef HIFN_DEBUG
1071 if (hifn_debug)
1072 device_printf(sc->sc_dev,
1073 "Unknown encryption level 0x%x\n", encl);
1074 #endif
1075 return 1;
1076 }
1077
1078 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
1079 HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
1080 DELAY(1000);
1081 addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
1082 DELAY(1000);
1083 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
1084 DELAY(1000);
1085
1086 for (i = 0; i <= 12; i++) {
1087 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
1088 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
1089
1090 DELAY(1000);
1091 }
1092
1093 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1094 encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1095
1096 #ifdef HIFN_DEBUG
1097 if (hifn_debug) {
1098 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
1099 device_printf(sc->sc_dev, "Engine is permanently "
1100 "locked until next system reset!\n");
1101 else
1102 device_printf(sc->sc_dev, "Engine enabled "
1103 "successfully!\n");
1104 }
1105 #endif
1106
1107 report:
1108 WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
1109 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
1110
1111 switch (encl) {
1112 case HIFN_PUSTAT_ENA_1:
1113 case HIFN_PUSTAT_ENA_2:
1114 break;
1115 case HIFN_PUSTAT_ENA_0:
1116 default:
1117 device_printf(sc->sc_dev, "disabled");
1118 break;
1119 }
1120
1121 return 0;
1122 }
1123
1124 /*
1125 * Give initial values to the registers listed in the "Register Space"
1126 * section of the HIFN Software Development reference manual.
1127 */
1128 static void
hifn_init_pci_registers(struct hifn_softc * sc)1129 hifn_init_pci_registers(struct hifn_softc *sc)
1130 {
1131 /* write fixed values needed by the Initialization registers */
1132 WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1133 WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1134 WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1135
1136 /* write all 4 ring address registers */
1137 WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
1138 offsetof(struct hifn_dma, cmdr[0]));
1139 WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
1140 offsetof(struct hifn_dma, srcr[0]));
1141 WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
1142 offsetof(struct hifn_dma, dstr[0]));
1143 WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
1144 offsetof(struct hifn_dma, resr[0]));
1145
1146 DELAY(2000);
1147
1148 /* write status register */
1149 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1150 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1151 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1152 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1153 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1154 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1155 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1156 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1157 HIFN_DMACSR_S_WAIT |
1158 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1159 HIFN_DMACSR_C_WAIT |
1160 HIFN_DMACSR_ENGINE |
1161 ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1162 HIFN_DMACSR_PUBDONE : 0) |
1163 ((sc->sc_flags & HIFN_IS_7811) ?
1164 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1165
1166 sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1167 sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1168 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1169 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1170 ((sc->sc_flags & HIFN_IS_7811) ?
1171 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1172 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1173 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1174
1175
1176 if (sc->sc_flags & HIFN_IS_7956) {
1177 u_int32_t pll;
1178
1179 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1180 HIFN_PUCNFG_TCALLPHASES |
1181 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1182
1183 /* turn off the clocks and insure bypass is set */
1184 pll = READ_REG_1(sc, HIFN_1_PLL);
1185 pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
1186 | HIFN_PLL_BP | HIFN_PLL_MBSET;
1187 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1188 DELAY(10*1000); /* 10ms */
1189
1190 /* change configuration */
1191 pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
1192 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1193 DELAY(10*1000); /* 10ms */
1194
1195 /* disable bypass */
1196 pll &= ~HIFN_PLL_BP;
1197 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1198 /* enable clocks with new configuration */
1199 pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
1200 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1201 } else {
1202 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1203 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1204 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1205 (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1206 }
1207
1208 WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1209 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1210 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1211 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1212 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1213 }
1214
1215 /*
1216 * The maximum number of sessions supported by the card
1217 * is dependent on the amount of context ram, which
1218 * encryption algorithms are enabled, and how compression
1219 * is configured. This should be configured before this
1220 * routine is called.
1221 */
1222 static void
hifn_sessions(struct hifn_softc * sc)1223 hifn_sessions(struct hifn_softc *sc)
1224 {
1225 u_int32_t pucnfg;
1226 int ctxsize;
1227
1228 pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1229
1230 if (pucnfg & HIFN_PUCNFG_COMPSING) {
1231 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1232 ctxsize = 128;
1233 else
1234 ctxsize = 512;
1235 /*
1236 * 7955/7956 has internal context memory of 32K
1237 */
1238 if (sc->sc_flags & HIFN_IS_7956)
1239 sc->sc_maxses = 32768 / ctxsize;
1240 else
1241 sc->sc_maxses = 1 +
1242 ((sc->sc_ramsize - 32768) / ctxsize);
1243 } else
1244 sc->sc_maxses = sc->sc_ramsize / 16384;
1245
1246 if (sc->sc_maxses > 2048)
1247 sc->sc_maxses = 2048;
1248 }
1249
1250 /*
1251 * Determine ram type (sram or dram). Board should be just out of a reset
1252 * state when this is called.
1253 */
1254 static int
hifn_ramtype(struct hifn_softc * sc)1255 hifn_ramtype(struct hifn_softc *sc)
1256 {
1257 u_int8_t data[8], dataexpect[8];
1258 int i;
1259
1260 for (i = 0; i < sizeof(data); i++)
1261 data[i] = dataexpect[i] = 0x55;
1262 if (hifn_writeramaddr(sc, 0, data))
1263 return (-1);
1264 if (hifn_readramaddr(sc, 0, data))
1265 return (-1);
1266 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1267 sc->sc_drammodel = 1;
1268 return (0);
1269 }
1270
1271 for (i = 0; i < sizeof(data); i++)
1272 data[i] = dataexpect[i] = 0xaa;
1273 if (hifn_writeramaddr(sc, 0, data))
1274 return (-1);
1275 if (hifn_readramaddr(sc, 0, data))
1276 return (-1);
1277 if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1278 sc->sc_drammodel = 1;
1279 return (0);
1280 }
1281
1282 return (0);
1283 }
1284
1285 #define HIFN_SRAM_MAX (32 << 20)
1286 #define HIFN_SRAM_STEP_SIZE 16384
1287 #define HIFN_SRAM_GRANULARITY (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1288
1289 static int
hifn_sramsize(struct hifn_softc * sc)1290 hifn_sramsize(struct hifn_softc *sc)
1291 {
1292 u_int32_t a;
1293 u_int8_t data[8];
1294 u_int8_t dataexpect[sizeof(data)];
1295 int32_t i;
1296
1297 for (i = 0; i < sizeof(data); i++)
1298 data[i] = dataexpect[i] = i ^ 0x5a;
1299
1300 for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1301 a = i * HIFN_SRAM_STEP_SIZE;
1302 bcopy(&i, data, sizeof(i));
1303 hifn_writeramaddr(sc, a, data);
1304 }
1305
1306 for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1307 a = i * HIFN_SRAM_STEP_SIZE;
1308 bcopy(&i, dataexpect, sizeof(i));
1309 if (hifn_readramaddr(sc, a, data) < 0)
1310 return (0);
1311 if (bcmp(data, dataexpect, sizeof(data)) != 0)
1312 return (0);
1313 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1314 }
1315
1316 return (0);
1317 }
1318
1319 /*
1320 * XXX For dram boards, one should really try all of the
1321 * HIFN_PUCNFG_DSZ_*'s. This just assumes that PUCNFG
1322 * is already set up correctly.
1323 */
1324 static int
hifn_dramsize(struct hifn_softc * sc)1325 hifn_dramsize(struct hifn_softc *sc)
1326 {
1327 u_int32_t cnfg;
1328
1329 if (sc->sc_flags & HIFN_IS_7956) {
1330 /*
1331 * 7955/7956 have a fixed internal ram of only 32K.
1332 */
1333 sc->sc_ramsize = 32768;
1334 } else {
1335 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1336 HIFN_PUCNFG_DRAMMASK;
1337 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1338 }
1339 return (0);
1340 }
1341
1342 static void
hifn_alloc_slot(struct hifn_softc * sc,int * cmdp,int * srcp,int * dstp,int * resp)1343 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
1344 {
1345 struct hifn_dma *dma = sc->sc_dma;
1346
1347 if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
1348 sc->sc_cmdi = 0;
1349 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1350 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1351 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1352 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1353 }
1354 *cmdp = sc->sc_cmdi++;
1355 sc->sc_cmdk = sc->sc_cmdi;
1356
1357 if (sc->sc_srci == HIFN_D_SRC_RSIZE) {
1358 sc->sc_srci = 0;
1359 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1360 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1361 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1362 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1363 }
1364 *srcp = sc->sc_srci++;
1365 sc->sc_srck = sc->sc_srci;
1366
1367 if (sc->sc_dsti == HIFN_D_DST_RSIZE) {
1368 sc->sc_dsti = 0;
1369 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1370 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1371 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1372 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1373 }
1374 *dstp = sc->sc_dsti++;
1375 sc->sc_dstk = sc->sc_dsti;
1376
1377 if (sc->sc_resi == HIFN_D_RES_RSIZE) {
1378 sc->sc_resi = 0;
1379 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1380 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1381 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1382 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1383 }
1384 *resp = sc->sc_resi++;
1385 sc->sc_resk = sc->sc_resi;
1386 }
1387
1388 static int
hifn_writeramaddr(struct hifn_softc * sc,int addr,u_int8_t * data)1389 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1390 {
1391 struct hifn_dma *dma = sc->sc_dma;
1392 hifn_base_command_t wc;
1393 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1394 int r, cmdi, resi, srci, dsti;
1395
1396 wc.masks = htole16(3 << 13);
1397 wc.session_num = htole16(addr >> 14);
1398 wc.total_source_count = htole16(8);
1399 wc.total_dest_count = htole16(addr & 0x3fff);
1400
1401 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1402
1403 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1404 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1405 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1406
1407 /* build write command */
1408 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1409 *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
1410 bcopy(data, &dma->test_src, sizeof(dma->test_src));
1411
1412 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
1413 + offsetof(struct hifn_dma, test_src));
1414 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
1415 + offsetof(struct hifn_dma, test_dst));
1416
1417 dma->cmdr[cmdi].l = htole32(16 | masks);
1418 dma->srcr[srci].l = htole32(8 | masks);
1419 dma->dstr[dsti].l = htole32(4 | masks);
1420 dma->resr[resi].l = htole32(4 | masks);
1421
1422 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1423 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1424
1425 for (r = 10000; r >= 0; r--) {
1426 DELAY(10);
1427 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1428 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1429 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1430 break;
1431 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1432 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1433 }
1434 if (r == 0) {
1435 device_printf(sc->sc_dev, "writeramaddr -- "
1436 "result[%d](addr %d) still valid\n", resi, addr);
1437 r = -1;
1438 return (-1);
1439 } else
1440 r = 0;
1441
1442 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1443 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1444 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1445
1446 return (r);
1447 }
1448
1449 static int
hifn_readramaddr(struct hifn_softc * sc,int addr,u_int8_t * data)1450 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1451 {
1452 struct hifn_dma *dma = sc->sc_dma;
1453 hifn_base_command_t rc;
1454 const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1455 int r, cmdi, srci, dsti, resi;
1456
1457 rc.masks = htole16(2 << 13);
1458 rc.session_num = htole16(addr >> 14);
1459 rc.total_source_count = htole16(addr & 0x3fff);
1460 rc.total_dest_count = htole16(8);
1461
1462 hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1463
1464 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1465 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1466 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1467
1468 bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1469 *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
1470
1471 dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
1472 offsetof(struct hifn_dma, test_src));
1473 dma->test_src = 0;
1474 dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr +
1475 offsetof(struct hifn_dma, test_dst));
1476 dma->test_dst = 0;
1477 dma->cmdr[cmdi].l = htole32(8 | masks);
1478 dma->srcr[srci].l = htole32(8 | masks);
1479 dma->dstr[dsti].l = htole32(8 | masks);
1480 dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1481
1482 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1483 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1484
1485 for (r = 10000; r >= 0; r--) {
1486 DELAY(10);
1487 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1488 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1489 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1490 break;
1491 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1492 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1493 }
1494 if (r == 0) {
1495 device_printf(sc->sc_dev, "readramaddr -- "
1496 "result[%d](addr %d) still valid\n", resi, addr);
1497 r = -1;
1498 } else {
1499 r = 0;
1500 bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
1501 }
1502
1503 WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1504 HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1505 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1506
1507 return (r);
1508 }
1509
1510 /*
1511 * Initialize the descriptor rings.
1512 */
1513 static void
hifn_init_dma(struct hifn_softc * sc)1514 hifn_init_dma(struct hifn_softc *sc)
1515 {
1516 struct hifn_dma *dma = sc->sc_dma;
1517 int i;
1518
1519 hifn_set_retry(sc);
1520
1521 /* initialize static pointer values */
1522 for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1523 dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
1524 offsetof(struct hifn_dma, command_bufs[i][0]));
1525 for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1526 dma->resr[i].p = htole32(sc->sc_dma_physaddr +
1527 offsetof(struct hifn_dma, result_bufs[i][0]));
1528
1529 dma->cmdr[HIFN_D_CMD_RSIZE].p =
1530 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
1531 dma->srcr[HIFN_D_SRC_RSIZE].p =
1532 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
1533 dma->dstr[HIFN_D_DST_RSIZE].p =
1534 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
1535 dma->resr[HIFN_D_RES_RSIZE].p =
1536 htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
1537
1538 sc->sc_cmdu = sc->sc_srcu = sc->sc_dstu = sc->sc_resu = 0;
1539 sc->sc_cmdi = sc->sc_srci = sc->sc_dsti = sc->sc_resi = 0;
1540 sc->sc_cmdk = sc->sc_srck = sc->sc_dstk = sc->sc_resk = 0;
1541 }
1542
1543 /*
1544 * Writes out the raw command buffer space. Returns the
1545 * command buffer size.
1546 */
1547 static u_int
hifn_write_command(struct hifn_command * cmd,u_int8_t * buf)1548 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1549 {
1550 u_int8_t *buf_pos;
1551 hifn_base_command_t *base_cmd;
1552 hifn_mac_command_t *mac_cmd;
1553 hifn_crypt_command_t *cry_cmd;
1554 int using_mac, using_crypt, len, ivlen;
1555 u_int32_t dlen, slen;
1556
1557 buf_pos = buf;
1558 using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1559 using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1560
1561 base_cmd = (hifn_base_command_t *)buf_pos;
1562 base_cmd->masks = htole16(cmd->base_masks);
1563 slen = cmd->src_mapsize;
1564 if (cmd->sloplen)
1565 dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
1566 else
1567 dlen = cmd->dst_mapsize;
1568 base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1569 base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1570 dlen >>= 16;
1571 slen >>= 16;
1572 base_cmd->session_num = htole16(
1573 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1574 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1575 buf_pos += sizeof(hifn_base_command_t);
1576
1577 if (using_mac) {
1578 mac_cmd = (hifn_mac_command_t *)buf_pos;
1579 dlen = cmd->maccrd->crd_len;
1580 mac_cmd->source_count = htole16(dlen & 0xffff);
1581 dlen >>= 16;
1582 mac_cmd->masks = htole16(cmd->mac_masks |
1583 ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1584 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1585 mac_cmd->reserved = 0;
1586 buf_pos += sizeof(hifn_mac_command_t);
1587 }
1588
1589 if (using_crypt) {
1590 cry_cmd = (hifn_crypt_command_t *)buf_pos;
1591 dlen = cmd->enccrd->crd_len;
1592 cry_cmd->source_count = htole16(dlen & 0xffff);
1593 dlen >>= 16;
1594 cry_cmd->masks = htole16(cmd->cry_masks |
1595 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1596 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1597 cry_cmd->reserved = 0;
1598 buf_pos += sizeof(hifn_crypt_command_t);
1599 }
1600
1601 if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1602 bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
1603 buf_pos += HIFN_MAC_KEY_LENGTH;
1604 }
1605
1606 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1607 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1608 case HIFN_CRYPT_CMD_ALG_3DES:
1609 bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
1610 buf_pos += HIFN_3DES_KEY_LENGTH;
1611 break;
1612 case HIFN_CRYPT_CMD_ALG_DES:
1613 bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
1614 buf_pos += HIFN_DES_KEY_LENGTH;
1615 break;
1616 case HIFN_CRYPT_CMD_ALG_RC4:
1617 len = 256;
1618 do {
1619 int clen;
1620
1621 clen = MIN(cmd->cklen, len);
1622 bcopy(cmd->ck, buf_pos, clen);
1623 len -= clen;
1624 buf_pos += clen;
1625 } while (len > 0);
1626 bzero(buf_pos, 4);
1627 buf_pos += 4;
1628 break;
1629 case HIFN_CRYPT_CMD_ALG_AES:
1630 /*
1631 * AES keys are variable 128, 192 and
1632 * 256 bits (16, 24 and 32 bytes).
1633 */
1634 bcopy(cmd->ck, buf_pos, cmd->cklen);
1635 buf_pos += cmd->cklen;
1636 break;
1637 }
1638 }
1639
1640 if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1641 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1642 case HIFN_CRYPT_CMD_ALG_AES:
1643 ivlen = HIFN_AES_IV_LENGTH;
1644 break;
1645 default:
1646 ivlen = HIFN_IV_LENGTH;
1647 break;
1648 }
1649 bcopy(cmd->iv, buf_pos, ivlen);
1650 buf_pos += ivlen;
1651 }
1652
1653 if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
1654 bzero(buf_pos, 8);
1655 buf_pos += 8;
1656 }
1657
1658 return (buf_pos - buf);
1659 }
1660
1661 static int
hifn_dmamap_aligned(struct hifn_operand * op)1662 hifn_dmamap_aligned(struct hifn_operand *op)
1663 {
1664 int i;
1665
1666 for (i = 0; i < op->nsegs; i++) {
1667 if (op->segs[i].ds_addr & 3)
1668 return (0);
1669 if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
1670 return (0);
1671 }
1672 return (1);
1673 }
1674
1675 static __inline int
hifn_dmamap_dstwrap(struct hifn_softc * sc,int idx)1676 hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
1677 {
1678 struct hifn_dma *dma = sc->sc_dma;
1679
1680 if (++idx == HIFN_D_DST_RSIZE) {
1681 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1682 HIFN_D_MASKDONEIRQ);
1683 HIFN_DSTR_SYNC(sc, idx,
1684 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1685 idx = 0;
1686 }
1687 return (idx);
1688 }
1689
1690 static int
hifn_dmamap_load_dst(struct hifn_softc * sc,struct hifn_command * cmd)1691 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1692 {
1693 struct hifn_dma *dma = sc->sc_dma;
1694 struct hifn_operand *dst = &cmd->dst;
1695 u_int32_t p, l;
1696 int idx, used = 0, i;
1697
1698 idx = sc->sc_dsti;
1699 for (i = 0; i < dst->nsegs - 1; i++) {
1700 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1701 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1702 HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
1703 HIFN_DSTR_SYNC(sc, idx,
1704 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1705 used++;
1706
1707 idx = hifn_dmamap_dstwrap(sc, idx);
1708 }
1709
1710 if (cmd->sloplen == 0) {
1711 p = dst->segs[i].ds_addr;
1712 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1713 dst->segs[i].ds_len;
1714 } else {
1715 p = sc->sc_dma_physaddr +
1716 offsetof(struct hifn_dma, slop[cmd->slopidx]);
1717 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1718 sizeof(u_int32_t);
1719
1720 if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
1721 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1722 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1723 HIFN_D_MASKDONEIRQ |
1724 (dst->segs[i].ds_len - cmd->sloplen));
1725 HIFN_DSTR_SYNC(sc, idx,
1726 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1727 used++;
1728
1729 idx = hifn_dmamap_dstwrap(sc, idx);
1730 }
1731 }
1732 dma->dstr[idx].p = htole32(p);
1733 dma->dstr[idx].l = htole32(l);
1734 HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1735 used++;
1736
1737 idx = hifn_dmamap_dstwrap(sc, idx);
1738
1739 sc->sc_dsti = idx;
1740 sc->sc_dstu += used;
1741 return (idx);
1742 }
1743
1744 static __inline int
hifn_dmamap_srcwrap(struct hifn_softc * sc,int idx)1745 hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
1746 {
1747 struct hifn_dma *dma = sc->sc_dma;
1748
1749 if (++idx == HIFN_D_SRC_RSIZE) {
1750 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1751 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1752 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1753 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1754 idx = 0;
1755 }
1756 return (idx);
1757 }
1758
1759 static int
hifn_dmamap_load_src(struct hifn_softc * sc,struct hifn_command * cmd)1760 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1761 {
1762 struct hifn_dma *dma = sc->sc_dma;
1763 struct hifn_operand *src = &cmd->src;
1764 int idx, i;
1765 u_int32_t last = 0;
1766
1767 idx = sc->sc_srci;
1768 for (i = 0; i < src->nsegs; i++) {
1769 if (i == src->nsegs - 1)
1770 last = HIFN_D_LAST;
1771
1772 dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
1773 dma->srcr[idx].l = htole32(src->segs[i].ds_len |
1774 HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1775 HIFN_SRCR_SYNC(sc, idx,
1776 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1777
1778 idx = hifn_dmamap_srcwrap(sc, idx);
1779 }
1780 sc->sc_srci = idx;
1781 sc->sc_srcu += src->nsegs;
1782 return (idx);
1783 }
1784
1785 static void
hifn_op_cb(void * arg,bus_dma_segment_t * seg,int nsegs,bus_size_t mapsize,int error)1786 hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1787 {
1788 struct hifn_operand *op = arg;
1789
1790 KASSERT(nsegs <= MAX_SCATTER,
1791 ("hifn_op_cb: too many DMA segments (%u > %u) "
1792 "returned when mapping operand", nsegs, MAX_SCATTER));
1793 op->mapsize = mapsize;
1794 op->nsegs = nsegs;
1795 bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1796 }
1797
1798 static int
hifn_crypto(struct hifn_softc * sc,struct hifn_command * cmd,struct cryptop * crp,int hint)1799 hifn_crypto(
1800 struct hifn_softc *sc,
1801 struct hifn_command *cmd,
1802 struct cryptop *crp,
1803 int hint)
1804 {
1805 struct hifn_dma *dma = sc->sc_dma;
1806 u_int32_t cmdlen, csr;
1807 int cmdi, resi, err = 0;
1808
1809 /*
1810 * need 1 cmd, and 1 res
1811 *
1812 * NB: check this first since it's easy.
1813 */
1814 HIFN_LOCK(sc);
1815 if ((sc->sc_cmdu + 1) > HIFN_D_CMD_RSIZE ||
1816 (sc->sc_resu + 1) > HIFN_D_RES_RSIZE) {
1817 #ifdef HIFN_DEBUG
1818 if (hifn_debug) {
1819 device_printf(sc->sc_dev,
1820 "cmd/result exhaustion, cmdu %u resu %u\n",
1821 sc->sc_cmdu, sc->sc_resu);
1822 }
1823 #endif
1824 hifnstats.hst_nomem_cr++;
1825 HIFN_UNLOCK(sc);
1826 return (ERESTART);
1827 }
1828
1829 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
1830 hifnstats.hst_nomem_map++;
1831 HIFN_UNLOCK(sc);
1832 return (ENOMEM);
1833 }
1834
1835 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1836 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1837 cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1838 hifnstats.hst_nomem_load++;
1839 err = ENOMEM;
1840 goto err_srcmap1;
1841 }
1842 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1843 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1844 cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1845 hifnstats.hst_nomem_load++;
1846 err = ENOMEM;
1847 goto err_srcmap1;
1848 }
1849 } else {
1850 err = EINVAL;
1851 goto err_srcmap1;
1852 }
1853
1854 if (hifn_dmamap_aligned(&cmd->src)) {
1855 cmd->sloplen = cmd->src_mapsize & 3;
1856 cmd->dst = cmd->src;
1857 } else {
1858 if (crp->crp_flags & CRYPTO_F_IOV) {
1859 err = EINVAL;
1860 goto err_srcmap;
1861 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1862 int totlen, len;
1863 struct mbuf *m, *m0, *mlast;
1864
1865 KASSERT(cmd->dst_m == cmd->src_m,
1866 ("hifn_crypto: dst_m initialized improperly"));
1867 hifnstats.hst_unaligned++;
1868 /*
1869 * Source is not aligned on a longword boundary.
1870 * Copy the data to insure alignment. If we fail
1871 * to allocate mbufs or clusters while doing this
1872 * we return ERESTART so the operation is requeued
1873 * at the crypto later, but only if there are
1874 * ops already posted to the hardware; otherwise we
1875 * have no guarantee that we'll be re-entered.
1876 */
1877 totlen = cmd->src_mapsize;
1878 if (cmd->src_m->m_flags & M_PKTHDR) {
1879 len = MHLEN;
1880 MGETHDR(m0, M_NOWAIT, MT_DATA);
1881 if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
1882 m_free(m0);
1883 m0 = NULL;
1884 }
1885 } else {
1886 len = MLEN;
1887 MGET(m0, M_NOWAIT, MT_DATA);
1888 }
1889 if (m0 == NULL) {
1890 hifnstats.hst_nomem_mbuf++;
1891 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1892 goto err_srcmap;
1893 }
1894 if (totlen >= MINCLSIZE) {
1895 if (!(MCLGET(m0, M_NOWAIT))) {
1896 hifnstats.hst_nomem_mcl++;
1897 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1898 m_freem(m0);
1899 goto err_srcmap;
1900 }
1901 len = MCLBYTES;
1902 }
1903 totlen -= len;
1904 m0->m_pkthdr.len = m0->m_len = len;
1905 mlast = m0;
1906
1907 while (totlen > 0) {
1908 MGET(m, M_NOWAIT, MT_DATA);
1909 if (m == NULL) {
1910 hifnstats.hst_nomem_mbuf++;
1911 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1912 m_freem(m0);
1913 goto err_srcmap;
1914 }
1915 len = MLEN;
1916 if (totlen >= MINCLSIZE) {
1917 if (!(MCLGET(m, M_NOWAIT))) {
1918 hifnstats.hst_nomem_mcl++;
1919 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1920 mlast->m_next = m;
1921 m_freem(m0);
1922 goto err_srcmap;
1923 }
1924 len = MCLBYTES;
1925 }
1926
1927 m->m_len = len;
1928 m0->m_pkthdr.len += len;
1929 totlen -= len;
1930
1931 mlast->m_next = m;
1932 mlast = m;
1933 }
1934 cmd->dst_m = m0;
1935 }
1936 }
1937
1938 if (cmd->dst_map == NULL) {
1939 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1940 hifnstats.hst_nomem_map++;
1941 err = ENOMEM;
1942 goto err_srcmap;
1943 }
1944 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1945 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1946 cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1947 hifnstats.hst_nomem_map++;
1948 err = ENOMEM;
1949 goto err_dstmap1;
1950 }
1951 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1952 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1953 cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1954 hifnstats.hst_nomem_load++;
1955 err = ENOMEM;
1956 goto err_dstmap1;
1957 }
1958 }
1959 }
1960
1961 #ifdef HIFN_DEBUG
1962 if (hifn_debug) {
1963 device_printf(sc->sc_dev,
1964 "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1965 READ_REG_1(sc, HIFN_1_DMA_CSR),
1966 READ_REG_1(sc, HIFN_1_DMA_IER),
1967 sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu,
1968 cmd->src_nsegs, cmd->dst_nsegs);
1969 }
1970 #endif
1971
1972 if (cmd->src_map == cmd->dst_map) {
1973 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1974 BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1975 } else {
1976 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1977 BUS_DMASYNC_PREWRITE);
1978 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1979 BUS_DMASYNC_PREREAD);
1980 }
1981
1982 /*
1983 * need N src, and N dst
1984 */
1985 if ((sc->sc_srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
1986 (sc->sc_dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
1987 #ifdef HIFN_DEBUG
1988 if (hifn_debug) {
1989 device_printf(sc->sc_dev,
1990 "src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
1991 sc->sc_srcu, cmd->src_nsegs,
1992 sc->sc_dstu, cmd->dst_nsegs);
1993 }
1994 #endif
1995 hifnstats.hst_nomem_sd++;
1996 err = ERESTART;
1997 goto err_dstmap;
1998 }
1999
2000 if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
2001 sc->sc_cmdi = 0;
2002 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
2003 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2004 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
2005 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2006 }
2007 cmdi = sc->sc_cmdi++;
2008 cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
2009 HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
2010
2011 /* .p for command/result already set */
2012 dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
2013 HIFN_D_MASKDONEIRQ);
2014 HIFN_CMDR_SYNC(sc, cmdi,
2015 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2016 sc->sc_cmdu++;
2017
2018 /*
2019 * We don't worry about missing an interrupt (which a "command wait"
2020 * interrupt salvages us from), unless there is more than one command
2021 * in the queue.
2022 */
2023 if (sc->sc_cmdu > 1) {
2024 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
2025 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2026 }
2027
2028 hifnstats.hst_ipackets++;
2029 hifnstats.hst_ibytes += cmd->src_mapsize;
2030
2031 hifn_dmamap_load_src(sc, cmd);
2032
2033 /*
2034 * Unlike other descriptors, we don't mask done interrupt from
2035 * result descriptor.
2036 */
2037 #ifdef HIFN_DEBUG
2038 if (hifn_debug)
2039 printf("load res\n");
2040 #endif
2041 if (sc->sc_resi == HIFN_D_RES_RSIZE) {
2042 sc->sc_resi = 0;
2043 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
2044 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2045 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
2046 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2047 }
2048 resi = sc->sc_resi++;
2049 KASSERT(sc->sc_hifn_commands[resi] == NULL,
2050 ("hifn_crypto: command slot %u busy", resi));
2051 sc->sc_hifn_commands[resi] = cmd;
2052 HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
2053 if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
2054 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2055 HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
2056 sc->sc_curbatch++;
2057 if (sc->sc_curbatch > hifnstats.hst_maxbatch)
2058 hifnstats.hst_maxbatch = sc->sc_curbatch;
2059 hifnstats.hst_totbatch++;
2060 } else {
2061 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2062 HIFN_D_VALID | HIFN_D_LAST);
2063 sc->sc_curbatch = 0;
2064 }
2065 HIFN_RESR_SYNC(sc, resi,
2066 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2067 sc->sc_resu++;
2068
2069 if (cmd->sloplen)
2070 cmd->slopidx = resi;
2071
2072 hifn_dmamap_load_dst(sc, cmd);
2073
2074 csr = 0;
2075 if (sc->sc_c_busy == 0) {
2076 csr |= HIFN_DMACSR_C_CTRL_ENA;
2077 sc->sc_c_busy = 1;
2078 }
2079 if (sc->sc_s_busy == 0) {
2080 csr |= HIFN_DMACSR_S_CTRL_ENA;
2081 sc->sc_s_busy = 1;
2082 }
2083 if (sc->sc_r_busy == 0) {
2084 csr |= HIFN_DMACSR_R_CTRL_ENA;
2085 sc->sc_r_busy = 1;
2086 }
2087 if (sc->sc_d_busy == 0) {
2088 csr |= HIFN_DMACSR_D_CTRL_ENA;
2089 sc->sc_d_busy = 1;
2090 }
2091 if (csr)
2092 WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
2093
2094 #ifdef HIFN_DEBUG
2095 if (hifn_debug) {
2096 device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
2097 READ_REG_1(sc, HIFN_1_DMA_CSR),
2098 READ_REG_1(sc, HIFN_1_DMA_IER));
2099 }
2100 #endif
2101
2102 sc->sc_active = 5;
2103 HIFN_UNLOCK(sc);
2104 KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
2105 return (err); /* success */
2106
2107 err_dstmap:
2108 if (cmd->src_map != cmd->dst_map)
2109 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2110 err_dstmap1:
2111 if (cmd->src_map != cmd->dst_map)
2112 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2113 err_srcmap:
2114 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2115 if (cmd->src_m != cmd->dst_m)
2116 m_freem(cmd->dst_m);
2117 }
2118 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2119 err_srcmap1:
2120 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2121 HIFN_UNLOCK(sc);
2122 return (err);
2123 }
2124
2125 static void
hifn_tick(void * vsc)2126 hifn_tick(void* vsc)
2127 {
2128 struct hifn_softc *sc = vsc;
2129
2130 HIFN_LOCK(sc);
2131 if (sc->sc_active == 0) {
2132 u_int32_t r = 0;
2133
2134 if (sc->sc_cmdu == 0 && sc->sc_c_busy) {
2135 sc->sc_c_busy = 0;
2136 r |= HIFN_DMACSR_C_CTRL_DIS;
2137 }
2138 if (sc->sc_srcu == 0 && sc->sc_s_busy) {
2139 sc->sc_s_busy = 0;
2140 r |= HIFN_DMACSR_S_CTRL_DIS;
2141 }
2142 if (sc->sc_dstu == 0 && sc->sc_d_busy) {
2143 sc->sc_d_busy = 0;
2144 r |= HIFN_DMACSR_D_CTRL_DIS;
2145 }
2146 if (sc->sc_resu == 0 && sc->sc_r_busy) {
2147 sc->sc_r_busy = 0;
2148 r |= HIFN_DMACSR_R_CTRL_DIS;
2149 }
2150 if (r)
2151 WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
2152 } else
2153 sc->sc_active--;
2154 HIFN_UNLOCK(sc);
2155 callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
2156 }
2157
2158 static void
hifn_intr(void * arg)2159 hifn_intr(void *arg)
2160 {
2161 struct hifn_softc *sc = arg;
2162 struct hifn_dma *dma;
2163 u_int32_t dmacsr, restart;
2164 int i, u;
2165
2166 dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
2167
2168 /* Nothing in the DMA unit interrupted */
2169 if ((dmacsr & sc->sc_dmaier) == 0)
2170 return;
2171
2172 HIFN_LOCK(sc);
2173
2174 dma = sc->sc_dma;
2175
2176 #ifdef HIFN_DEBUG
2177 if (hifn_debug) {
2178 device_printf(sc->sc_dev,
2179 "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
2180 dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
2181 sc->sc_cmdi, sc->sc_srci, sc->sc_dsti, sc->sc_resi,
2182 sc->sc_cmdk, sc->sc_srck, sc->sc_dstk, sc->sc_resk,
2183 sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
2184 }
2185 #endif
2186
2187 WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2188
2189 if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2190 (dmacsr & HIFN_DMACSR_PUBDONE))
2191 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2192 READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2193
2194 restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
2195 if (restart)
2196 device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
2197
2198 if (sc->sc_flags & HIFN_IS_7811) {
2199 if (dmacsr & HIFN_DMACSR_ILLR)
2200 device_printf(sc->sc_dev, "illegal read\n");
2201 if (dmacsr & HIFN_DMACSR_ILLW)
2202 device_printf(sc->sc_dev, "illegal write\n");
2203 }
2204
2205 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2206 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2207 if (restart) {
2208 device_printf(sc->sc_dev, "abort, resetting.\n");
2209 hifnstats.hst_abort++;
2210 hifn_abort(sc);
2211 HIFN_UNLOCK(sc);
2212 return;
2213 }
2214
2215 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (sc->sc_cmdu == 0)) {
2216 /*
2217 * If no slots to process and we receive a "waiting on
2218 * command" interrupt, we disable the "waiting on command"
2219 * (by clearing it).
2220 */
2221 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2222 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2223 }
2224
2225 /* clear the rings */
2226 i = sc->sc_resk; u = sc->sc_resu;
2227 while (u != 0) {
2228 HIFN_RESR_SYNC(sc, i,
2229 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2230 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2231 HIFN_RESR_SYNC(sc, i,
2232 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2233 break;
2234 }
2235
2236 if (i != HIFN_D_RES_RSIZE) {
2237 struct hifn_command *cmd;
2238 u_int8_t *macbuf = NULL;
2239
2240 HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2241 cmd = sc->sc_hifn_commands[i];
2242 KASSERT(cmd != NULL,
2243 ("hifn_intr: null command slot %u", i));
2244 sc->sc_hifn_commands[i] = NULL;
2245
2246 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2247 macbuf = dma->result_bufs[i];
2248 macbuf += 12;
2249 }
2250
2251 hifn_callback(sc, cmd, macbuf);
2252 hifnstats.hst_opackets++;
2253 u--;
2254 }
2255
2256 if (++i == (HIFN_D_RES_RSIZE + 1))
2257 i = 0;
2258 }
2259 sc->sc_resk = i; sc->sc_resu = u;
2260
2261 i = sc->sc_srck; u = sc->sc_srcu;
2262 while (u != 0) {
2263 if (i == HIFN_D_SRC_RSIZE)
2264 i = 0;
2265 HIFN_SRCR_SYNC(sc, i,
2266 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2267 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2268 HIFN_SRCR_SYNC(sc, i,
2269 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2270 break;
2271 }
2272 i++, u--;
2273 }
2274 sc->sc_srck = i; sc->sc_srcu = u;
2275
2276 i = sc->sc_cmdk; u = sc->sc_cmdu;
2277 while (u != 0) {
2278 HIFN_CMDR_SYNC(sc, i,
2279 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2280 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2281 HIFN_CMDR_SYNC(sc, i,
2282 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2283 break;
2284 }
2285 if (i != HIFN_D_CMD_RSIZE) {
2286 u--;
2287 HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2288 }
2289 if (++i == (HIFN_D_CMD_RSIZE + 1))
2290 i = 0;
2291 }
2292 sc->sc_cmdk = i; sc->sc_cmdu = u;
2293
2294 HIFN_UNLOCK(sc);
2295
2296 if (sc->sc_needwakeup) { /* XXX check high watermark */
2297 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
2298 #ifdef HIFN_DEBUG
2299 if (hifn_debug)
2300 device_printf(sc->sc_dev,
2301 "wakeup crypto (%x) u %d/%d/%d/%d\n",
2302 sc->sc_needwakeup,
2303 sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
2304 #endif
2305 sc->sc_needwakeup &= ~wakeup;
2306 crypto_unblock(sc->sc_cid, wakeup);
2307 }
2308 }
2309
2310 /*
2311 * Allocate a new 'session' and return an encoded session id. 'sidp'
2312 * contains our registration id, and should contain an encoded session
2313 * id on successful allocation.
2314 */
2315 static int
hifn_newsession(device_t dev,crypto_session_t cses,struct cryptoini * cri)2316 hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
2317 {
2318 struct hifn_softc *sc = device_get_softc(dev);
2319 struct cryptoini *c;
2320 int mac = 0, cry = 0;
2321 struct hifn_session *ses;
2322
2323 KASSERT(sc != NULL, ("hifn_newsession: null softc"));
2324 if (cri == NULL || sc == NULL)
2325 return (EINVAL);
2326
2327 ses = crypto_get_driver_session(cses);
2328
2329 for (c = cri; c != NULL; c = c->cri_next) {
2330 switch (c->cri_alg) {
2331 case CRYPTO_MD5:
2332 case CRYPTO_SHA1:
2333 case CRYPTO_MD5_HMAC:
2334 case CRYPTO_SHA1_HMAC:
2335 if (mac)
2336 return (EINVAL);
2337 mac = 1;
2338 ses->hs_mlen = c->cri_mlen;
2339 if (ses->hs_mlen == 0) {
2340 switch (c->cri_alg) {
2341 case CRYPTO_MD5:
2342 case CRYPTO_MD5_HMAC:
2343 ses->hs_mlen = 16;
2344 break;
2345 case CRYPTO_SHA1:
2346 case CRYPTO_SHA1_HMAC:
2347 ses->hs_mlen = 20;
2348 break;
2349 }
2350 }
2351 break;
2352 case CRYPTO_DES_CBC:
2353 case CRYPTO_3DES_CBC:
2354 case CRYPTO_AES_CBC:
2355 /* XXX this may read fewer, does it matter? */
2356 read_random(ses->hs_iv,
2357 c->cri_alg == CRYPTO_AES_CBC ?
2358 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2359 /*FALLTHROUGH*/
2360 case CRYPTO_ARC4:
2361 if (cry)
2362 return (EINVAL);
2363 cry = 1;
2364 break;
2365 default:
2366 return (EINVAL);
2367 }
2368 }
2369 if (mac == 0 && cry == 0)
2370 return (EINVAL);
2371 return (0);
2372 }
2373
2374 /*
2375 * XXX freesession routine should run a zero'd mac/encrypt key into context
2376 * ram. to blow away any keys already stored there.
2377 */
2378
2379 static int
hifn_process(device_t dev,struct cryptop * crp,int hint)2380 hifn_process(device_t dev, struct cryptop *crp, int hint)
2381 {
2382 struct hifn_softc *sc = device_get_softc(dev);
2383 struct hifn_command *cmd = NULL;
2384 int err, ivlen;
2385 struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2386 struct hifn_session *ses;
2387
2388 if (crp == NULL || crp->crp_callback == NULL) {
2389 hifnstats.hst_invalid++;
2390 return (EINVAL);
2391 }
2392
2393 ses = crypto_get_driver_session(crp->crp_session);
2394 cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
2395 if (cmd == NULL) {
2396 hifnstats.hst_nomem++;
2397 err = ENOMEM;
2398 goto errout;
2399 }
2400
2401 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2402 cmd->src_m = (struct mbuf *)crp->crp_buf;
2403 cmd->dst_m = (struct mbuf *)crp->crp_buf;
2404 } else if (crp->crp_flags & CRYPTO_F_IOV) {
2405 cmd->src_io = (struct uio *)crp->crp_buf;
2406 cmd->dst_io = (struct uio *)crp->crp_buf;
2407 } else {
2408 err = EINVAL;
2409 goto errout; /* XXX we don't handle contiguous buffers! */
2410 }
2411
2412 crd1 = crp->crp_desc;
2413 if (crd1 == NULL) {
2414 err = EINVAL;
2415 goto errout;
2416 }
2417 crd2 = crd1->crd_next;
2418
2419 if (crd2 == NULL) {
2420 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
2421 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2422 crd1->crd_alg == CRYPTO_SHA1 ||
2423 crd1->crd_alg == CRYPTO_MD5) {
2424 maccrd = crd1;
2425 enccrd = NULL;
2426 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2427 crd1->crd_alg == CRYPTO_3DES_CBC ||
2428 crd1->crd_alg == CRYPTO_AES_CBC ||
2429 crd1->crd_alg == CRYPTO_ARC4) {
2430 if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2431 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2432 maccrd = NULL;
2433 enccrd = crd1;
2434 } else {
2435 err = EINVAL;
2436 goto errout;
2437 }
2438 } else {
2439 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
2440 crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2441 crd1->crd_alg == CRYPTO_MD5 ||
2442 crd1->crd_alg == CRYPTO_SHA1) &&
2443 (crd2->crd_alg == CRYPTO_DES_CBC ||
2444 crd2->crd_alg == CRYPTO_3DES_CBC ||
2445 crd2->crd_alg == CRYPTO_AES_CBC ||
2446 crd2->crd_alg == CRYPTO_ARC4) &&
2447 ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2448 cmd->base_masks = HIFN_BASE_CMD_DECODE;
2449 maccrd = crd1;
2450 enccrd = crd2;
2451 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2452 crd1->crd_alg == CRYPTO_ARC4 ||
2453 crd1->crd_alg == CRYPTO_3DES_CBC ||
2454 crd1->crd_alg == CRYPTO_AES_CBC) &&
2455 (crd2->crd_alg == CRYPTO_MD5_HMAC ||
2456 crd2->crd_alg == CRYPTO_SHA1_HMAC ||
2457 crd2->crd_alg == CRYPTO_MD5 ||
2458 crd2->crd_alg == CRYPTO_SHA1) &&
2459 (crd1->crd_flags & CRD_F_ENCRYPT)) {
2460 enccrd = crd1;
2461 maccrd = crd2;
2462 } else {
2463 /*
2464 * We cannot order the 7751 as requested
2465 */
2466 err = EINVAL;
2467 goto errout;
2468 }
2469 }
2470
2471 if (enccrd) {
2472 cmd->enccrd = enccrd;
2473 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2474 switch (enccrd->crd_alg) {
2475 case CRYPTO_ARC4:
2476 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2477 break;
2478 case CRYPTO_DES_CBC:
2479 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2480 HIFN_CRYPT_CMD_MODE_CBC |
2481 HIFN_CRYPT_CMD_NEW_IV;
2482 break;
2483 case CRYPTO_3DES_CBC:
2484 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2485 HIFN_CRYPT_CMD_MODE_CBC |
2486 HIFN_CRYPT_CMD_NEW_IV;
2487 break;
2488 case CRYPTO_AES_CBC:
2489 cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2490 HIFN_CRYPT_CMD_MODE_CBC |
2491 HIFN_CRYPT_CMD_NEW_IV;
2492 break;
2493 default:
2494 err = EINVAL;
2495 goto errout;
2496 }
2497 if (enccrd->crd_alg != CRYPTO_ARC4) {
2498 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2499 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2500 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2501 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2502 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2503 else
2504 bcopy(ses->hs_iv, cmd->iv, ivlen);
2505
2506 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2507 == 0) {
2508 crypto_copyback(crp->crp_flags,
2509 crp->crp_buf, enccrd->crd_inject,
2510 ivlen, cmd->iv);
2511 }
2512 } else {
2513 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2514 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2515 else {
2516 crypto_copydata(crp->crp_flags,
2517 crp->crp_buf, enccrd->crd_inject,
2518 ivlen, cmd->iv);
2519 }
2520 }
2521 }
2522
2523 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
2524 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2525 cmd->ck = enccrd->crd_key;
2526 cmd->cklen = enccrd->crd_klen >> 3;
2527 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2528
2529 /*
2530 * Need to specify the size for the AES key in the masks.
2531 */
2532 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2533 HIFN_CRYPT_CMD_ALG_AES) {
2534 switch (cmd->cklen) {
2535 case 16:
2536 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2537 break;
2538 case 24:
2539 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2540 break;
2541 case 32:
2542 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2543 break;
2544 default:
2545 err = EINVAL;
2546 goto errout;
2547 }
2548 }
2549 }
2550
2551 if (maccrd) {
2552 cmd->maccrd = maccrd;
2553 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2554
2555 switch (maccrd->crd_alg) {
2556 case CRYPTO_MD5:
2557 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2558 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2559 HIFN_MAC_CMD_POS_IPSEC;
2560 break;
2561 case CRYPTO_MD5_HMAC:
2562 cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2563 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2564 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2565 break;
2566 case CRYPTO_SHA1:
2567 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2568 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2569 HIFN_MAC_CMD_POS_IPSEC;
2570 break;
2571 case CRYPTO_SHA1_HMAC:
2572 cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2573 HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2574 HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2575 break;
2576 }
2577
2578 if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
2579 maccrd->crd_alg == CRYPTO_MD5_HMAC) {
2580 cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2581 bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
2582 bzero(cmd->mac + (maccrd->crd_klen >> 3),
2583 HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2584 }
2585 }
2586
2587 cmd->crp = crp;
2588 cmd->session = ses;
2589 cmd->softc = sc;
2590
2591 err = hifn_crypto(sc, cmd, crp, hint);
2592 if (!err) {
2593 return 0;
2594 } else if (err == ERESTART) {
2595 /*
2596 * There weren't enough resources to dispatch the request
2597 * to the part. Notify the caller so they'll requeue this
2598 * request and resubmit it again soon.
2599 */
2600 #ifdef HIFN_DEBUG
2601 if (hifn_debug)
2602 device_printf(sc->sc_dev, "requeue request\n");
2603 #endif
2604 free(cmd, M_DEVBUF);
2605 sc->sc_needwakeup |= CRYPTO_SYMQ;
2606 return (err);
2607 }
2608
2609 errout:
2610 if (cmd != NULL)
2611 free(cmd, M_DEVBUF);
2612 if (err == EINVAL)
2613 hifnstats.hst_invalid++;
2614 else
2615 hifnstats.hst_nomem++;
2616 crp->crp_etype = err;
2617 crypto_done(crp);
2618 return (err);
2619 }
2620
2621 static void
hifn_abort(struct hifn_softc * sc)2622 hifn_abort(struct hifn_softc *sc)
2623 {
2624 struct hifn_dma *dma = sc->sc_dma;
2625 struct hifn_command *cmd;
2626 struct cryptop *crp;
2627 int i, u;
2628
2629 i = sc->sc_resk; u = sc->sc_resu;
2630 while (u != 0) {
2631 cmd = sc->sc_hifn_commands[i];
2632 KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
2633 sc->sc_hifn_commands[i] = NULL;
2634 crp = cmd->crp;
2635
2636 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2637 /* Salvage what we can. */
2638 u_int8_t *macbuf;
2639
2640 if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2641 macbuf = dma->result_bufs[i];
2642 macbuf += 12;
2643 } else
2644 macbuf = NULL;
2645 hifnstats.hst_opackets++;
2646 hifn_callback(sc, cmd, macbuf);
2647 } else {
2648 if (cmd->src_map == cmd->dst_map) {
2649 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2650 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2651 } else {
2652 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2653 BUS_DMASYNC_POSTWRITE);
2654 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2655 BUS_DMASYNC_POSTREAD);
2656 }
2657
2658 if (cmd->src_m != cmd->dst_m) {
2659 m_freem(cmd->src_m);
2660 crp->crp_buf = (caddr_t)cmd->dst_m;
2661 }
2662
2663 /* non-shared buffers cannot be restarted */
2664 if (cmd->src_map != cmd->dst_map) {
2665 /*
2666 * XXX should be EAGAIN, delayed until
2667 * after the reset.
2668 */
2669 crp->crp_etype = ENOMEM;
2670 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2671 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2672 } else
2673 crp->crp_etype = ENOMEM;
2674
2675 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2676 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2677
2678 free(cmd, M_DEVBUF);
2679 if (crp->crp_etype != EAGAIN)
2680 crypto_done(crp);
2681 }
2682
2683 if (++i == HIFN_D_RES_RSIZE)
2684 i = 0;
2685 u--;
2686 }
2687 sc->sc_resk = i; sc->sc_resu = u;
2688
2689 hifn_reset_board(sc, 1);
2690 hifn_init_dma(sc);
2691 hifn_init_pci_registers(sc);
2692 }
2693
2694 static void
hifn_callback(struct hifn_softc * sc,struct hifn_command * cmd,u_int8_t * macbuf)2695 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
2696 {
2697 struct hifn_dma *dma = sc->sc_dma;
2698 struct cryptop *crp = cmd->crp;
2699 struct cryptodesc *crd;
2700 struct mbuf *m;
2701 int totlen, i, u, ivlen;
2702
2703 if (cmd->src_map == cmd->dst_map) {
2704 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2705 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2706 } else {
2707 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2708 BUS_DMASYNC_POSTWRITE);
2709 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2710 BUS_DMASYNC_POSTREAD);
2711 }
2712
2713 if (crp->crp_flags & CRYPTO_F_IMBUF) {
2714 if (cmd->src_m != cmd->dst_m) {
2715 crp->crp_buf = (caddr_t)cmd->dst_m;
2716 totlen = cmd->src_mapsize;
2717 for (m = cmd->dst_m; m != NULL; m = m->m_next) {
2718 if (totlen < m->m_len) {
2719 m->m_len = totlen;
2720 totlen = 0;
2721 } else
2722 totlen -= m->m_len;
2723 }
2724 cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
2725 m_freem(cmd->src_m);
2726 }
2727 }
2728
2729 if (cmd->sloplen != 0) {
2730 crypto_copyback(crp->crp_flags, crp->crp_buf,
2731 cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
2732 (caddr_t)&dma->slop[cmd->slopidx]);
2733 }
2734
2735 i = sc->sc_dstk; u = sc->sc_dstu;
2736 while (u != 0) {
2737 if (i == HIFN_D_DST_RSIZE)
2738 i = 0;
2739 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2740 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2741 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2742 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2743 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2744 break;
2745 }
2746 i++, u--;
2747 }
2748 sc->sc_dstk = i; sc->sc_dstu = u;
2749
2750 hifnstats.hst_obytes += cmd->dst_mapsize;
2751
2752 if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2753 HIFN_BASE_CMD_CRYPT) {
2754 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2755 if (crd->crd_alg != CRYPTO_DES_CBC &&
2756 crd->crd_alg != CRYPTO_3DES_CBC &&
2757 crd->crd_alg != CRYPTO_AES_CBC)
2758 continue;
2759 ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
2760 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2761 crypto_copydata(crp->crp_flags, crp->crp_buf,
2762 crd->crd_skip + crd->crd_len - ivlen, ivlen,
2763 cmd->session->hs_iv);
2764 break;
2765 }
2766 }
2767
2768 if (macbuf != NULL) {
2769 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2770 int len;
2771
2772 if (crd->crd_alg != CRYPTO_MD5 &&
2773 crd->crd_alg != CRYPTO_SHA1 &&
2774 crd->crd_alg != CRYPTO_MD5_HMAC &&
2775 crd->crd_alg != CRYPTO_SHA1_HMAC) {
2776 continue;
2777 }
2778 len = cmd->session->hs_mlen;
2779 crypto_copyback(crp->crp_flags, crp->crp_buf,
2780 crd->crd_inject, len, macbuf);
2781 break;
2782 }
2783 }
2784
2785 if (cmd->src_map != cmd->dst_map) {
2786 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2787 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2788 }
2789 bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2790 bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2791 free(cmd, M_DEVBUF);
2792 crypto_done(crp);
2793 }
2794
2795 /*
2796 * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
2797 * and Group 1 registers; avoid conditions that could create
2798 * burst writes by doing a read in between the writes.
2799 *
2800 * NB: The read we interpose is always to the same register;
2801 * we do this because reading from an arbitrary (e.g. last)
2802 * register may not always work.
2803 */
2804 static void
hifn_write_reg_0(struct hifn_softc * sc,bus_size_t reg,u_int32_t val)2805 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2806 {
2807 if (sc->sc_flags & HIFN_IS_7811) {
2808 if (sc->sc_bar0_lastreg == reg - 4)
2809 bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
2810 sc->sc_bar0_lastreg = reg;
2811 }
2812 bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
2813 }
2814
2815 static void
hifn_write_reg_1(struct hifn_softc * sc,bus_size_t reg,u_int32_t val)2816 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2817 {
2818 if (sc->sc_flags & HIFN_IS_7811) {
2819 if (sc->sc_bar1_lastreg == reg - 4)
2820 bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
2821 sc->sc_bar1_lastreg = reg;
2822 }
2823 bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
2824 }
2825
2826 #ifdef HIFN_VULCANDEV
2827 /*
2828 * this code provides support for mapping the PK engine's register
2829 * into a userspace program.
2830 *
2831 */
2832 static int
vulcanpk_mmap(struct cdev * dev,vm_ooffset_t offset,vm_paddr_t * paddr,int nprot,vm_memattr_t * memattr)2833 vulcanpk_mmap(struct cdev *dev, vm_ooffset_t offset,
2834 vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
2835 {
2836 struct hifn_softc *sc;
2837 vm_paddr_t pd;
2838 void *b;
2839
2840 sc = dev->si_drv1;
2841
2842 pd = rman_get_start(sc->sc_bar1res);
2843 b = rman_get_virtual(sc->sc_bar1res);
2844
2845 #if 0
2846 printf("vpk mmap: %p(%016llx) offset=%lld\n", b,
2847 (unsigned long long)pd, offset);
2848 hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
2849 #endif
2850
2851 if (offset == 0) {
2852 *paddr = pd;
2853 return (0);
2854 }
2855 return (-1);
2856 }
2857
2858 static struct cdevsw vulcanpk_cdevsw = {
2859 .d_version = D_VERSION,
2860 .d_mmap = vulcanpk_mmap,
2861 .d_name = "vulcanpk",
2862 };
2863 #endif /* HIFN_VULCANDEV */
2864