1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2003-2012 Broadcom Corporation
5 * All Rights Reserved
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
25 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
27 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
28 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33
34 #include <sys/cdefs.h>
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/errno.h>
39 #include <sys/endian.h>
40 #include <sys/malloc.h>
41 #include <sys/kernel.h>
42 #include <sys/module.h>
43 #include <sys/mbuf.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/sysctl.h>
47 #include <sys/bus.h>
48 #include <sys/random.h>
49 #include <sys/rman.h>
50 #include <sys/uio.h>
51 #include <sys/kobj.h>
52
53 #include <dev/pci/pcivar.h>
54
55 #include <opencrypto/cryptodev.h>
56
57 #include "cryptodev_if.h"
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61
62 #include <mips/nlm/hal/haldefs.h>
63 #include <mips/nlm/hal/iomap.h>
64 #include <mips/nlm/xlp.h>
65 #include <mips/nlm/hal/sys.h>
66 #include <mips/nlm/hal/fmn.h>
67 #include <mips/nlm/hal/nlmsaelib.h>
68 #include <mips/nlm/dev/sec/rsa_ucode.h>
69 #include <mips/nlm/hal/cop2.h>
70 #include <mips/nlm/hal/mips-extns.h>
71 #include <mips/nlm/msgring.h>
72 #include <mips/nlm/dev/sec/nlmrsalib.h>
73
74 #ifdef NLM_RSA_DEBUG
75 static void print_krp_params(struct cryptkop *krp);
76 #endif
77
78 static int xlp_rsa_init(struct xlp_rsa_softc *sc, int node);
79 static int xlp_rsa_kprocess(device_t , struct cryptkop *, int);
80 static int xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits);
81 static void xlp_free_cmd_params(struct xlp_rsa_command *cmd);
82 static int xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst,
83 uint32_t paramsize, uint8_t result);
84
85 static int xlp_rsa_probe(device_t);
86 static int xlp_rsa_attach(device_t);
87 static int xlp_rsa_detach(device_t);
88
89 static device_method_t xlp_rsa_methods[] = {
90 /* device interface */
91 DEVMETHOD(device_probe, xlp_rsa_probe),
92 DEVMETHOD(device_attach, xlp_rsa_attach),
93 DEVMETHOD(device_detach, xlp_rsa_detach),
94
95 /* bus interface */
96 DEVMETHOD(bus_print_child, bus_generic_print_child),
97 DEVMETHOD(bus_driver_added, bus_generic_driver_added),
98
99 /* crypto device methods */
100 DEVMETHOD(cryptodev_kprocess, xlp_rsa_kprocess),
101
102 DEVMETHOD_END
103 };
104
105 static driver_t xlp_rsa_driver = {
106 "nlmrsa",
107 xlp_rsa_methods,
108 sizeof(struct xlp_rsa_softc)
109 };
110 static devclass_t xlp_rsa_devclass;
111
112 DRIVER_MODULE(nlmrsa, pci, xlp_rsa_driver, xlp_rsa_devclass, 0, 0);
113 MODULE_DEPEND(nlmrsa, crypto, 1, 1, 1);
114
115 #ifdef NLM_RSA_DEBUG
116 static void
print_krp_params(struct cryptkop * krp)117 print_krp_params(struct cryptkop *krp)
118 {
119 int i;
120
121 printf("krp->krp_op :%d\n", krp->krp_op);
122 printf("krp->krp_status :%d\n", krp->krp_status);
123 printf("krp->krp_iparams:%d\n", krp->krp_iparams);
124 printf("krp->krp_oparams:%d\n", krp->krp_oparams);
125 for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
126 printf("krp->krp_param[%d].crp_p :0x%llx\n", i,
127 (unsigned long long)krp->krp_param[i].crp_p);
128 printf("krp->krp_param[%d].crp_nbits :%d\n", i,
129 krp->krp_param[i].crp_nbits);
130 printf("krp->krp_param[%d].crp_nbytes :%d\n", i,
131 howmany(krp->krp_param[i].crp_nbits, 8));
132 }
133 }
134 #endif
135
136 static int
xlp_rsa_init(struct xlp_rsa_softc * sc,int node)137 xlp_rsa_init(struct xlp_rsa_softc *sc, int node)
138 {
139 struct xlp_rsa_command *cmd = NULL;
140 uint32_t fbvc, dstvc, endsel, regval;
141 struct nlm_fmn_msg m;
142 int err, ret, i;
143 uint64_t base;
144
145 /* Register interrupt handler for the RSA/ECC CMS messages */
146 if (register_msgring_handler(sc->rsaecc_vc_start,
147 sc->rsaecc_vc_end, nlm_xlprsaecc_msgring_handler, sc) != 0) {
148 err = -1;
149 printf("Couldn't register rsa/ecc msgring handler\n");
150 goto errout;
151 }
152 fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
153 /* Do the CMS credit initialization */
154 /* Currently it is configured by default to 50 when kernel comes up */
155
156 #if BYTE_ORDER == LITTLE_ENDIAN
157 for (i = 0; i < nitems(nlm_rsa_ucode_data); i++)
158 nlm_rsa_ucode_data[i] = htobe64(nlm_rsa_ucode_data[i]);
159 #endif
160 for (dstvc = sc->rsaecc_vc_start; dstvc <= sc->rsaecc_vc_end; dstvc++) {
161 cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
162 M_NOWAIT | M_ZERO);
163 KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
164 cmd->rsasrc = contigmalloc(sizeof(nlm_rsa_ucode_data),
165 M_DEVBUF,
166 (M_WAITOK | M_ZERO),
167 0UL /* low address */, -1UL /* high address */,
168 XLP_L2L3_CACHELINE_SIZE /* alignment */,
169 0UL /* boundary */);
170 KASSERT(cmd->rsasrc != NULL,
171 ("%s:cmd->rsasrc is NULL\n", __func__));
172 memcpy(cmd->rsasrc, nlm_rsa_ucode_data,
173 sizeof(nlm_rsa_ucode_data));
174 m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, 0x70, 0,
175 vtophys(cmd->rsasrc));
176 m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
177 vtophys(cmd->rsasrc));
178 /* Software scratch pad */
179 m.msg[2] = (uintptr_t)cmd;
180 m.msg[3] = 0;
181
182 ret = nlm_fmn_msgsend(dstvc, 3, FMN_SWCODE_RSA, &m);
183 if (ret != 0) {
184 err = -1;
185 printf("%s: msgsnd failed (%x)\n", __func__, ret);
186 goto errout;
187 }
188 }
189 /* Configure so that all VCs send request to all RSA pipes */
190 base = nlm_get_rsa_regbase(node);
191 if (nlm_is_xlp3xx()) {
192 endsel = 1;
193 regval = 0xFFFF;
194 } else {
195 endsel = 3;
196 regval = 0x07FFFFFF;
197 }
198 for (i = 0; i < endsel; i++)
199 nlm_write_rsa_reg(base, RSA_ENG_SEL_0 + i, regval);
200 return (0);
201 errout:
202 xlp_free_cmd_params(cmd);
203 return (err);
204 }
205
206 /* This function is called from an interrupt handler */
207 void
nlm_xlprsaecc_msgring_handler(int vc,int size,int code,int src_id,struct nlm_fmn_msg * msg,void * data)208 nlm_xlprsaecc_msgring_handler(int vc, int size, int code, int src_id,
209 struct nlm_fmn_msg *msg, void *data)
210 {
211 struct xlp_rsa_command *cmd;
212 struct xlp_rsa_softc *sc;
213 struct crparam *outparam;
214 int ostart;
215
216 KASSERT(code == FMN_SWCODE_RSA,
217 ("%s: bad code = %d, expected code = %d\n", __func__, code,
218 FMN_SWCODE_RSA));
219
220 sc = data;
221 KASSERT(src_id >= sc->rsaecc_vc_start && src_id <= sc->rsaecc_vc_end,
222 ("%s: bad src_id = %d, expect %d - %d\n", __func__,
223 src_id, sc->rsaecc_vc_start, sc->rsaecc_vc_end));
224
225 cmd = (struct xlp_rsa_command *)(uintptr_t)msg->msg[1];
226 KASSERT(cmd != NULL, ("%s:cmd not received properly\n", __func__));
227
228 if (RSA_ERROR(msg->msg[0]) != 0) {
229 printf("%s: Message rcv msg0 %llx msg1 %llx err %x \n",
230 __func__, (unsigned long long)msg->msg[0],
231 (unsigned long long)msg->msg[1],
232 (int)RSA_ERROR(msg->msg[0]));
233 cmd->krp->krp_status = EBADMSG;
234 }
235
236 if (cmd->krp != NULL) {
237 ostart = cmd->krp->krp_iparams;
238 outparam = &cmd->krp->krp_param[ostart];
239 xlp_rsa_inp2hwformat(cmd->rsasrc + cmd->rsaopsize * ostart,
240 outparam->crp_p,
241 howmany(outparam->crp_nbits, 8),
242 1);
243 crypto_kdone(cmd->krp);
244 }
245
246 xlp_free_cmd_params(cmd);
247 }
248
249 static int
xlp_rsa_probe(device_t dev)250 xlp_rsa_probe(device_t dev)
251 {
252 struct xlp_rsa_softc *sc;
253
254 if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
255 pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
256 sc = device_get_softc(dev);
257 return (BUS_PROBE_DEFAULT);
258 }
259 return (ENXIO);
260 }
261
262 /*
263 * Attach an interface that successfully probed.
264 */
265 static int
xlp_rsa_attach(device_t dev)266 xlp_rsa_attach(device_t dev)
267 {
268 struct xlp_rsa_softc *sc = device_get_softc(dev);
269 uint64_t base;
270 int qstart, qnum;
271 int freq, node;
272
273 sc->sc_dev = dev;
274
275 node = nlm_get_device_node(pci_get_slot(dev));
276 freq = nlm_set_device_frequency(node, DFS_DEVICE_RSA, 250);
277 if (bootverbose)
278 device_printf(dev, "RSA Freq: %dMHz\n", freq);
279 if (pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
280 device_set_desc(dev, "XLP RSA/ECC Accelerator");
281 sc->sc_cid = crypto_get_driverid(dev,
282 sizeof(struct xlp_rsa_session), CRYPTOCAP_F_HARDWARE);
283 if (sc->sc_cid < 0) {
284 printf("xlp_rsaecc-err:couldn't get the driver id\n");
285 goto error_exit;
286 }
287 if (crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0) != 0)
288 goto error_exit;
289
290 base = nlm_get_rsa_pcibase(node);
291 qstart = nlm_qidstart(base);
292 qnum = nlm_qnum(base);
293 sc->rsaecc_vc_start = qstart;
294 sc->rsaecc_vc_end = qstart + qnum - 1;
295 }
296 if (xlp_rsa_init(sc, node) != 0)
297 goto error_exit;
298 device_printf(dev, "RSA Initialization complete!\n");
299 gone_in_dev(dev, 14, "Asymmetric crypto");
300 return (0);
301
302 error_exit:
303 return (ENXIO);
304 }
305
306 /*
307 * Detach an interface that successfully probed.
308 */
309 static int
xlp_rsa_detach(device_t dev)310 xlp_rsa_detach(device_t dev)
311 {
312 return (0);
313 }
314
315 /*
316 * XXX freesession should run a zero'd mac/encrypt key into context ram.
317 * XXX to blow away any keys already stored there.
318 */
319
320 static void
xlp_free_cmd_params(struct xlp_rsa_command * cmd)321 xlp_free_cmd_params(struct xlp_rsa_command *cmd)
322 {
323
324 if (cmd == NULL)
325 return;
326 if (cmd->rsasrc != NULL) {
327 if (cmd->krp == NULL) /* Micro code load */
328 contigfree(cmd->rsasrc, sizeof(nlm_rsa_ucode_data),
329 M_DEVBUF);
330 else
331 free(cmd->rsasrc, M_DEVBUF);
332 }
333 free(cmd, M_DEVBUF);
334 }
335
336 static int
xlp_get_rsa_opsize(struct xlp_rsa_command * cmd,unsigned int bits)337 xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits)
338 {
339
340 if (bits == 0 || bits > 8192)
341 return (-1);
342 /* XLP hardware expects always a fixed size with unused bytes
343 * zeroed out in the input data */
344 if (bits <= 512) {
345 cmd->rsatype = 0x40;
346 cmd->rsaopsize = 64;
347 } else if (bits <= 1024) {
348 cmd->rsatype = 0x41;
349 cmd->rsaopsize = 128;
350 } else if (bits <= 2048) {
351 cmd->rsatype = 0x42;
352 cmd->rsaopsize = 256;
353 } else if (bits <= 4096) {
354 cmd->rsatype = 0x43;
355 cmd->rsaopsize = 512;
356 } else if (bits <= 8192) {
357 cmd->rsatype = 0x44;
358 cmd->rsaopsize = 1024;
359 }
360 return (0);
361 }
362
363 static int
xlp_rsa_inp2hwformat(uint8_t * src,uint8_t * dst,uint32_t paramsize,uint8_t result)364 xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst, uint32_t paramsize,
365 uint8_t result)
366 {
367 uint32_t pdwords, pbytes;
368 int i, j, k;
369
370 pdwords = paramsize / 8;
371 pbytes = paramsize % 8;
372
373 for (i = 0, k = 0; i < pdwords; i++) {
374 /* copy dwords of inp/hw to hw/out format */
375 for (j = 7; j >= 0; j--, k++)
376 dst[i * 8 + j] = src[k];
377 }
378 if (pbytes) {
379 if (result == 0) {
380 /* copy rem bytes of input data to hw format */
381 for (j = 7; k < paramsize; j--, k++)
382 dst[i * 8 + j] = src[k];
383 } else {
384 /* copy rem bytes of hw data to exp output format */
385 for (j = 7; k < paramsize; j--, k++)
386 dst[k] = src[i * 8 + j];
387 }
388 }
389
390 return (0);
391 }
392
393 static int
nlm_crypto_complete_rsa_request(struct xlp_rsa_softc * sc,struct xlp_rsa_command * cmd)394 nlm_crypto_complete_rsa_request(struct xlp_rsa_softc *sc,
395 struct xlp_rsa_command *cmd)
396 {
397 unsigned int fbvc;
398 struct nlm_fmn_msg m;
399 int ret;
400
401 fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
402
403 m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, cmd->rsatype,
404 cmd->rsafn, vtophys(cmd->rsasrc));
405 m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
406 vtophys(cmd->rsasrc + cmd->rsaopsize * cmd->krp->krp_iparams));
407 /* Software scratch pad */
408 m.msg[2] = (uintptr_t)cmd;
409 m.msg[3] = 0;
410
411 /* Send the message to rsa engine vc */
412 ret = nlm_fmn_msgsend(sc->rsaecc_vc_start, 3, FMN_SWCODE_RSA, &m);
413 if (ret != 0) {
414 #ifdef NLM_SEC_DEBUG
415 printf("%s: msgsnd failed (%x)\n", __func__, ret);
416 #endif
417 return (ERESTART);
418 }
419 return (0);
420 }
421
422 static int
xlp_rsa_kprocess(device_t dev,struct cryptkop * krp,int hint)423 xlp_rsa_kprocess(device_t dev, struct cryptkop *krp, int hint)
424 {
425 struct xlp_rsa_softc *sc = device_get_softc(dev);
426 struct xlp_rsa_command *cmd;
427 struct crparam *kp;
428 int err, i;
429
430 if (krp == NULL || krp->krp_callback == NULL)
431 return (EINVAL);
432
433 cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
434 M_NOWAIT | M_ZERO);
435 KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
436 cmd->krp = krp;
437
438 #ifdef NLM_RSA_DEBUG
439 print_krp_params(krp);
440 #endif
441 err = EOPNOTSUPP;
442 switch (krp->krp_op) {
443 case CRK_MOD_EXP:
444 if (krp->krp_iparams == 3 && krp->krp_oparams == 1)
445 break;
446 goto errout;
447 default:
448 device_printf(dev, "Op:%d not yet supported\n", krp->krp_op);
449 goto errout;
450 }
451
452 err = xlp_get_rsa_opsize(cmd,
453 krp->krp_param[krp->krp_iparams - 1].crp_nbits);
454 if (err != 0) {
455 err = EINVAL;
456 goto errout;
457 }
458 cmd->rsafn = 0; /* Mod Exp */
459 cmd->rsasrc = malloc(
460 cmd->rsaopsize * (krp->krp_iparams + krp->krp_oparams),
461 M_DEVBUF,
462 M_NOWAIT | M_ZERO);
463 if (cmd->rsasrc == NULL) {
464 err = ENOMEM;
465 goto errout;
466 }
467
468 for (i = 0, kp = krp->krp_param; i < krp->krp_iparams; i++, kp++) {
469 KASSERT(kp->crp_nbits != 0,
470 ("%s: parameter[%d]'s length is zero\n", __func__, i));
471 xlp_rsa_inp2hwformat(kp->crp_p,
472 cmd->rsasrc + i * cmd->rsaopsize,
473 howmany(kp->crp_nbits, 8), 0);
474 }
475 err = nlm_crypto_complete_rsa_request(sc, cmd);
476 if (err != 0)
477 goto errout;
478
479 return (0);
480 errout:
481 xlp_free_cmd_params(cmd);
482 krp->krp_status = err;
483 crypto_kdone(krp);
484 return (err);
485 }
486