1 /*-
2 * Copyright (c) 2018 Conrad Meyer <[email protected]>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/kobj.h>
34 #include <sys/lock.h>
35 #include <sys/malloc.h>
36 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/rwlock.h>
39 #include <sys/smp.h>
40
41 #include <blake2.h>
42
43 #include <opencrypto/cryptodev.h>
44 #include <cryptodev_if.h>
45
46 #include <machine/fpu.h>
47
48 struct blake2_session {
49 size_t mlen;
50 };
51 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
52
53 struct blake2_softc {
54 bool dying;
55 int32_t cid;
56 struct rwlock lock;
57 };
58
59 static struct mtx_padalign *ctx_mtx;
60 static struct fpu_kern_ctx **ctx_fpu;
61
62 #define ACQUIRE_CTX(i, ctx) \
63 do { \
64 (i) = PCPU_GET(cpuid); \
65 mtx_lock(&ctx_mtx[(i)]); \
66 (ctx) = ctx_fpu[(i)]; \
67 } while (0)
68 #define RELEASE_CTX(i, ctx) \
69 do { \
70 mtx_unlock(&ctx_mtx[(i)]); \
71 (i) = -1; \
72 (ctx) = NULL; \
73 } while (0)
74
75 static int blake2_cipher_setup(struct blake2_session *ses,
76 const struct crypto_session_params *csp);
77 static int blake2_cipher_process(struct blake2_session *ses,
78 struct cryptop *crp);
79
80 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
81
82 static void
blake2_identify(driver_t * drv,device_t parent)83 blake2_identify(driver_t *drv, device_t parent)
84 {
85
86 /* NB: order 10 is so we get attached after h/w devices */
87 if (device_find_child(parent, "blaketwo", -1) == NULL &&
88 BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
89 panic("blaketwo: could not attach");
90 }
91
92 static int
blake2_probe(device_t dev)93 blake2_probe(device_t dev)
94 {
95 device_set_desc(dev, "Blake2");
96 return (0);
97 }
98
99 static void
blake2_cleanctx(void)100 blake2_cleanctx(void)
101 {
102 int i;
103
104 /* XXX - no way to return driverid */
105 CPU_FOREACH(i) {
106 if (ctx_fpu[i] != NULL) {
107 mtx_destroy(&ctx_mtx[i]);
108 fpu_kern_free_ctx(ctx_fpu[i]);
109 }
110 ctx_fpu[i] = NULL;
111 }
112 free(ctx_mtx, M_BLAKE2);
113 ctx_mtx = NULL;
114 free(ctx_fpu, M_BLAKE2);
115 ctx_fpu = NULL;
116 }
117
118 static int
blake2_attach(device_t dev)119 blake2_attach(device_t dev)
120 {
121 struct blake2_softc *sc;
122 int i;
123
124 sc = device_get_softc(dev);
125 sc->dying = false;
126
127 sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
128 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
129 CRYPTOCAP_F_ACCEL_SOFTWARE);
130 if (sc->cid < 0) {
131 device_printf(dev, "Could not get crypto driver id.\n");
132 return (ENOMEM);
133 }
134
135 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
136 M_WAITOK | M_ZERO);
137 ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
138 M_WAITOK | M_ZERO);
139
140 CPU_FOREACH(i) {
141 #ifdef __amd64__
142 ctx_fpu[i] = fpu_kern_alloc_ctx_domain(
143 pcpu_find(i)->pc_domain, FPU_KERN_NORMAL);
144 #else
145 ctx_fpu[i] = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
146 #endif
147 mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
148 }
149
150 rw_init(&sc->lock, "blake2_lock");
151
152 return (0);
153 }
154
155 static int
blake2_detach(device_t dev)156 blake2_detach(device_t dev)
157 {
158 struct blake2_softc *sc;
159
160 sc = device_get_softc(dev);
161
162 rw_wlock(&sc->lock);
163 sc->dying = true;
164 rw_wunlock(&sc->lock);
165 crypto_unregister_all(sc->cid);
166
167 rw_destroy(&sc->lock);
168
169 blake2_cleanctx();
170
171 return (0);
172 }
173
174 static int
blake2_probesession(device_t dev,const struct crypto_session_params * csp)175 blake2_probesession(device_t dev, const struct crypto_session_params *csp)
176 {
177
178 if (csp->csp_flags != 0)
179 return (EINVAL);
180 switch (csp->csp_mode) {
181 case CSP_MODE_DIGEST:
182 switch (csp->csp_auth_alg) {
183 case CRYPTO_BLAKE2B:
184 case CRYPTO_BLAKE2S:
185 break;
186 default:
187 return (EINVAL);
188 }
189 break;
190 default:
191 return (EINVAL);
192 }
193 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
194 }
195
196 static int
blake2_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)197 blake2_newsession(device_t dev, crypto_session_t cses,
198 const struct crypto_session_params *csp)
199 {
200 struct blake2_softc *sc;
201 struct blake2_session *ses;
202 int error;
203
204 sc = device_get_softc(dev);
205
206 ses = crypto_get_driver_session(cses);
207
208 rw_rlock(&sc->lock);
209 if (sc->dying) {
210 rw_runlock(&sc->lock);
211 return (EINVAL);
212 }
213 rw_runlock(&sc->lock);
214
215 error = blake2_cipher_setup(ses, csp);
216 if (error != 0) {
217 CRYPTDEB("setup failed");
218 return (error);
219 }
220
221 return (0);
222 }
223
224 static int
blake2_process(device_t dev,struct cryptop * crp,int hint __unused)225 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
226 {
227 struct blake2_session *ses;
228 int error;
229
230 ses = crypto_get_driver_session(crp->crp_session);
231 error = blake2_cipher_process(ses, crp);
232
233 crp->crp_etype = error;
234 crypto_done(crp);
235 return (0);
236 }
237
238 static device_method_t blake2_methods[] = {
239 DEVMETHOD(device_identify, blake2_identify),
240 DEVMETHOD(device_probe, blake2_probe),
241 DEVMETHOD(device_attach, blake2_attach),
242 DEVMETHOD(device_detach, blake2_detach),
243
244 DEVMETHOD(cryptodev_probesession, blake2_probesession),
245 DEVMETHOD(cryptodev_newsession, blake2_newsession),
246 DEVMETHOD(cryptodev_process, blake2_process),
247
248 DEVMETHOD_END
249 };
250
251 static driver_t blake2_driver = {
252 "blaketwo",
253 blake2_methods,
254 sizeof(struct blake2_softc),
255 };
256 static devclass_t blake2_devclass;
257
258 DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
259 MODULE_VERSION(blake2, 1);
260 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
261
262 static bool
blake2_check_klen(const struct crypto_session_params * csp,unsigned klen)263 blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
264 {
265
266 if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
267 return (klen <= BLAKE2S_KEYBYTES);
268 else
269 return (klen <= BLAKE2B_KEYBYTES);
270 }
271
272 static int
blake2_cipher_setup(struct blake2_session * ses,const struct crypto_session_params * csp)273 blake2_cipher_setup(struct blake2_session *ses,
274 const struct crypto_session_params *csp)
275 {
276 int hashlen;
277
278 CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
279
280 if (!blake2_check_klen(csp, csp->csp_auth_klen))
281 return (EINVAL);
282
283 if (csp->csp_auth_mlen < 0)
284 return (EINVAL);
285
286 switch (csp->csp_auth_alg) {
287 case CRYPTO_BLAKE2S:
288 hashlen = BLAKE2S_OUTBYTES;
289 break;
290 case CRYPTO_BLAKE2B:
291 hashlen = BLAKE2B_OUTBYTES;
292 break;
293 default:
294 return (EINVAL);
295 }
296
297 if (csp->csp_auth_mlen > hashlen)
298 return (EINVAL);
299
300 if (csp->csp_auth_mlen == 0)
301 ses->mlen = hashlen;
302 else
303 ses->mlen = csp->csp_auth_mlen;
304 return (0);
305 }
306
307 static int
blake2b_applicator(void * state,const void * buf,u_int len)308 blake2b_applicator(void *state, const void *buf, u_int len)
309 {
310 int rc;
311
312 rc = blake2b_update(state, buf, len);
313 if (rc != 0)
314 return (EINVAL);
315 return (0);
316 }
317
318 static int
blake2s_applicator(void * state,const void * buf,u_int len)319 blake2s_applicator(void *state, const void *buf, u_int len)
320 {
321 int rc;
322
323 rc = blake2s_update(state, buf, len);
324 if (rc != 0)
325 return (EINVAL);
326 return (0);
327 }
328
329 static int
blake2_cipher_process(struct blake2_session * ses,struct cryptop * crp)330 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
331 {
332 union {
333 blake2b_state sb;
334 blake2s_state ss;
335 } bctx;
336 char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
337 const struct crypto_session_params *csp;
338 struct fpu_kern_ctx *ctx;
339 const void *key;
340 int ctxidx;
341 bool kt;
342 int error, rc;
343 unsigned klen;
344
345 ctx = NULL;
346 ctxidx = 0;
347 error = EINVAL;
348
349 kt = is_fpu_kern_thread(0);
350 if (!kt) {
351 ACQUIRE_CTX(ctxidx, ctx);
352 fpu_kern_enter(curthread, ctx,
353 FPU_KERN_NORMAL | FPU_KERN_KTHR);
354 }
355
356 csp = crypto_get_params(crp->crp_session);
357 if (crp->crp_auth_key != NULL)
358 key = crp->crp_auth_key;
359 else
360 key = csp->csp_auth_key;
361 klen = csp->csp_auth_klen;
362 switch (csp->csp_auth_alg) {
363 case CRYPTO_BLAKE2B:
364 if (klen > 0)
365 rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
366 else
367 rc = blake2b_init(&bctx.sb, ses->mlen);
368 if (rc != 0)
369 goto out;
370 error = crypto_apply(crp, crp->crp_payload_start,
371 crp->crp_payload_length, blake2b_applicator, &bctx.sb);
372 if (error != 0)
373 goto out;
374 rc = blake2b_final(&bctx.sb, res, ses->mlen);
375 if (rc != 0) {
376 error = EINVAL;
377 goto out;
378 }
379 break;
380 case CRYPTO_BLAKE2S:
381 if (klen > 0)
382 rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
383 else
384 rc = blake2s_init(&bctx.ss, ses->mlen);
385 if (rc != 0)
386 goto out;
387 error = crypto_apply(crp, crp->crp_payload_start,
388 crp->crp_payload_length, blake2s_applicator, &bctx.ss);
389 if (error != 0)
390 goto out;
391 rc = blake2s_final(&bctx.ss, res, ses->mlen);
392 if (rc != 0) {
393 error = EINVAL;
394 goto out;
395 }
396 break;
397 default:
398 panic("unreachable");
399 }
400
401 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
402 crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
403 if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
404 return (EBADMSG);
405 } else
406 crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
407
408 out:
409 if (!kt) {
410 fpu_kern_leave(curthread, ctx);
411 RELEASE_CTX(ctxidx, ctx);
412 }
413 return (error);
414 }
415