1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020 Netflix, Inc
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer,
11 * without modification.
12 * 2. Redistributions in binary form must reproduce at minimum a disclaimer
13 * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
14 * redistribution must be conditioned upon including a substantially
15 * similar Disclaimer requirement for further binary redistribution.
16 *
17 * NO WARRANTY
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
21 * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
23 * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
26 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
28 * THE POSSIBILITY OF SUCH DAMAGES.
29 */
30
31 /*
32 * A driver for the OpenCrypto framework which uses assembly routines
33 * from OpenSSL.
34 */
35
36 #include <sys/cdefs.h>
37 #include <sys/types.h>
38 #include <sys/bus.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/module.h>
42
43 #include <machine/fpu.h>
44
45 #include <opencrypto/cryptodev.h>
46 #include <opencrypto/xform_auth.h>
47
48 #include <crypto/openssl/ossl.h>
49 #include <crypto/openssl/ossl_chacha.h>
50 #include <crypto/openssl/ossl_cipher.h>
51
52 #include "cryptodev_if.h"
53
54 static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto");
55
56 static void
ossl_identify(driver_t * driver,device_t parent)57 ossl_identify(driver_t *driver, device_t parent)
58 {
59
60 if (device_find_child(parent, "ossl", -1) == NULL)
61 BUS_ADD_CHILD(parent, 10, "ossl", -1);
62 }
63
64 static int
ossl_probe(device_t dev)65 ossl_probe(device_t dev)
66 {
67
68 device_set_desc(dev, "OpenSSL crypto");
69 return (BUS_PROBE_DEFAULT);
70 }
71
72 static int
ossl_attach(device_t dev)73 ossl_attach(device_t dev)
74 {
75 struct ossl_softc *sc;
76
77 sc = device_get_softc(dev);
78
79 sc->has_aes = sc->has_aes_gcm = false;
80
81 ossl_cpuid(sc);
82 sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session),
83 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC |
84 CRYPTOCAP_F_ACCEL_SOFTWARE);
85 if (sc->sc_cid < 0) {
86 device_printf(dev, "failed to allocate crypto driver id\n");
87 return (ENXIO);
88 }
89
90 return (0);
91 }
92
93 static int
ossl_detach(device_t dev)94 ossl_detach(device_t dev)
95 {
96 struct ossl_softc *sc;
97
98 sc = device_get_softc(dev);
99
100 crypto_unregister_all(sc->sc_cid);
101
102 return (0);
103 }
104
105 static struct auth_hash *
ossl_lookup_hash(const struct crypto_session_params * csp)106 ossl_lookup_hash(const struct crypto_session_params *csp)
107 {
108
109 switch (csp->csp_auth_alg) {
110 case CRYPTO_SHA1:
111 case CRYPTO_SHA1_HMAC:
112 return (&ossl_hash_sha1);
113 case CRYPTO_SHA2_224:
114 case CRYPTO_SHA2_224_HMAC:
115 return (&ossl_hash_sha224);
116 case CRYPTO_SHA2_256:
117 case CRYPTO_SHA2_256_HMAC:
118 return (&ossl_hash_sha256);
119 case CRYPTO_SHA2_384:
120 case CRYPTO_SHA2_384_HMAC:
121 return (&ossl_hash_sha384);
122 case CRYPTO_SHA2_512:
123 case CRYPTO_SHA2_512_HMAC:
124 return (&ossl_hash_sha512);
125 case CRYPTO_POLY1305:
126 return (&ossl_hash_poly1305);
127 default:
128 return (NULL);
129 }
130 }
131
132 static struct ossl_cipher*
ossl_lookup_cipher(const struct crypto_session_params * csp)133 ossl_lookup_cipher(const struct crypto_session_params *csp)
134 {
135
136 switch (csp->csp_cipher_alg) {
137 case CRYPTO_AES_CBC:
138 switch (csp->csp_cipher_klen * 8) {
139 case 128:
140 case 192:
141 case 256:
142 break;
143 default:
144 return (NULL);
145 }
146 return (&ossl_cipher_aes_cbc);
147 case CRYPTO_AES_NIST_GCM_16:
148 switch (csp->csp_cipher_klen * 8) {
149 case 128:
150 case 192:
151 case 256:
152 break;
153 default:
154 return (NULL);
155 }
156 return (&ossl_cipher_aes_gcm);
157 case CRYPTO_CHACHA20:
158 if (csp->csp_cipher_klen != CHACHA_KEY_SIZE)
159 return (NULL);
160 return (&ossl_cipher_chacha20);
161 default:
162 return (NULL);
163 }
164 }
165
166 static int
ossl_probesession(device_t dev,const struct crypto_session_params * csp)167 ossl_probesession(device_t dev, const struct crypto_session_params *csp)
168 {
169 struct ossl_softc *sc = device_get_softc(dev);
170
171 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
172 0)
173 return (EINVAL);
174 switch (csp->csp_mode) {
175 case CSP_MODE_DIGEST:
176 if (ossl_lookup_hash(csp) == NULL)
177 return (EINVAL);
178 break;
179 case CSP_MODE_CIPHER:
180 if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes)
181 return (EINVAL);
182 if (ossl_lookup_cipher(csp) == NULL)
183 return (EINVAL);
184 break;
185 case CSP_MODE_ETA:
186 if (!sc->has_aes ||
187 csp->csp_cipher_alg == CRYPTO_CHACHA20 ||
188 ossl_lookup_hash(csp) == NULL ||
189 ossl_lookup_cipher(csp) == NULL)
190 return (EINVAL);
191 break;
192 case CSP_MODE_AEAD:
193 switch (csp->csp_cipher_alg) {
194 case CRYPTO_CHACHA20_POLY1305:
195 break;
196 case CRYPTO_AES_NIST_GCM_16:
197 if (!sc->has_aes_gcm || ossl_lookup_cipher(csp) == NULL)
198 return (EINVAL);
199 if (csp->csp_ivlen != AES_GCM_IV_LEN)
200 return (EINVAL);
201 if (csp->csp_auth_mlen != 0 &&
202 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
203 return (EINVAL);
204 break;
205 default:
206 return (EINVAL);
207 }
208 break;
209 default:
210 return (EINVAL);
211 }
212
213 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
214 }
215
216 static void
ossl_newsession_hash(struct ossl_session * s,const struct crypto_session_params * csp)217 ossl_newsession_hash(struct ossl_session *s,
218 const struct crypto_session_params *csp)
219 {
220 struct auth_hash *axf;
221
222 axf = ossl_lookup_hash(csp);
223 s->hash.axf = axf;
224 if (csp->csp_auth_mlen == 0)
225 s->hash.mlen = axf->hashsize;
226 else
227 s->hash.mlen = csp->csp_auth_mlen;
228
229 if (csp->csp_auth_klen == 0) {
230 axf->Init(&s->hash.ictx);
231 } else {
232 if (csp->csp_auth_key != NULL) {
233 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
234 if (axf->Setkey != NULL) {
235 axf->Init(&s->hash.ictx);
236 axf->Setkey(&s->hash.ictx, csp->csp_auth_key,
237 csp->csp_auth_klen);
238 } else {
239 hmac_init_ipad(axf, csp->csp_auth_key,
240 csp->csp_auth_klen, &s->hash.ictx);
241 hmac_init_opad(axf, csp->csp_auth_key,
242 csp->csp_auth_klen, &s->hash.octx);
243 }
244 fpu_kern_leave(curthread, NULL);
245 }
246 }
247 }
248
249 static int
ossl_newsession_cipher(struct ossl_session * s,const struct crypto_session_params * csp)250 ossl_newsession_cipher(struct ossl_session *s,
251 const struct crypto_session_params *csp)
252 {
253 struct ossl_cipher *cipher;
254 int error = 0;
255
256 cipher = ossl_lookup_cipher(csp);
257 if (cipher == NULL)
258 return (EINVAL);
259
260 s->cipher.cipher = cipher;
261
262 if (csp->csp_cipher_key == NULL)
263 return (0);
264
265 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
266 if (cipher->set_encrypt_key != NULL) {
267 error = cipher->set_encrypt_key(csp->csp_cipher_key,
268 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx);
269 if (error != 0) {
270 fpu_kern_leave(curthread, NULL);
271 return (error);
272 }
273 }
274 if (cipher->set_decrypt_key != NULL)
275 error = cipher->set_decrypt_key(csp->csp_cipher_key,
276 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx);
277 fpu_kern_leave(curthread, NULL);
278
279 return (error);
280 }
281
282 static int
ossl_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)283 ossl_newsession(device_t dev, crypto_session_t cses,
284 const struct crypto_session_params *csp)
285 {
286 struct ossl_session *s;
287 int error = 0;
288
289 s = crypto_get_driver_session(cses);
290 switch (csp->csp_mode) {
291 case CSP_MODE_DIGEST:
292 ossl_newsession_hash(s, csp);
293 break;
294 case CSP_MODE_CIPHER:
295 error = ossl_newsession_cipher(s, csp);
296 break;
297 case CSP_MODE_ETA:
298 ossl_newsession_hash(s, csp);
299 error = ossl_newsession_cipher(s, csp);
300 break;
301 case CSP_MODE_AEAD:
302 if (csp->csp_cipher_alg != CRYPTO_CHACHA20_POLY1305)
303 error = ossl_newsession_cipher(s, csp);
304 break;
305 default:
306 __assert_unreachable();
307 }
308
309 return (error);
310 }
311
312 static int
ossl_process_hash(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)313 ossl_process_hash(struct ossl_session *s, struct cryptop *crp,
314 const struct crypto_session_params *csp)
315 {
316 struct ossl_hash_context ctx;
317 char digest[HASH_MAX_LEN];
318 struct auth_hash *axf;
319 int error;
320
321 axf = s->hash.axf;
322
323 if (crp->crp_auth_key == NULL) {
324 ctx = s->hash.ictx;
325 } else {
326 if (axf->Setkey != NULL) {
327 axf->Init(&ctx);
328 axf->Setkey(&ctx, crp->crp_auth_key,
329 csp->csp_auth_klen);
330 } else {
331 hmac_init_ipad(axf, crp->crp_auth_key,
332 csp->csp_auth_klen, &ctx);
333 }
334 }
335
336 if (crp->crp_aad != NULL)
337 error = axf->Update(&ctx, crp->crp_aad, crp->crp_aad_length);
338 else
339 error = crypto_apply(crp, crp->crp_aad_start,
340 crp->crp_aad_length, axf->Update, &ctx);
341 if (error)
342 goto out;
343
344 error = crypto_apply(crp, crp->crp_payload_start,
345 crp->crp_payload_length, axf->Update, &ctx);
346 if (error)
347 goto out;
348
349 axf->Final(digest, &ctx);
350
351 if (csp->csp_auth_klen != 0 && axf->Setkey == NULL) {
352 if (crp->crp_auth_key == NULL)
353 ctx = s->hash.octx;
354 else
355 hmac_init_opad(axf, crp->crp_auth_key,
356 csp->csp_auth_klen, &ctx);
357 axf->Update(&ctx, digest, axf->hashsize);
358 axf->Final(digest, &ctx);
359 }
360
361 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
362 char digest2[HASH_MAX_LEN];
363
364 crypto_copydata(crp, crp->crp_digest_start, s->hash.mlen,
365 digest2);
366 if (timingsafe_bcmp(digest, digest2, s->hash.mlen) != 0)
367 error = EBADMSG;
368 explicit_bzero(digest2, sizeof(digest2));
369 } else {
370 crypto_copyback(crp, crp->crp_digest_start, s->hash.mlen,
371 digest);
372 }
373 explicit_bzero(digest, sizeof(digest));
374
375 out:
376 explicit_bzero(&ctx, sizeof(ctx));
377 return (error);
378 }
379
380 static int
ossl_process_cipher(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)381 ossl_process_cipher(struct ossl_session *s, struct cryptop *crp,
382 const struct crypto_session_params *csp)
383 {
384 return (s->cipher.cipher->process(&s->cipher, crp, csp));
385 }
386
387 static int
ossl_process_eta(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)388 ossl_process_eta(struct ossl_session *s, struct cryptop *crp,
389 const struct crypto_session_params *csp)
390 {
391 int error;
392
393 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
394 error = s->cipher.cipher->process(&s->cipher, crp, csp);
395 if (error == 0)
396 error = ossl_process_hash(s, crp, csp);
397 } else {
398 error = ossl_process_hash(s, crp, csp);
399 if (error == 0)
400 error = s->cipher.cipher->process(&s->cipher, crp, csp);
401 }
402
403 return (error);
404 }
405
406 static int
ossl_process_aead(struct ossl_session * s,struct cryptop * crp,const struct crypto_session_params * csp)407 ossl_process_aead(struct ossl_session *s, struct cryptop *crp,
408 const struct crypto_session_params *csp)
409 {
410 if (csp->csp_cipher_alg == CRYPTO_CHACHA20_POLY1305) {
411 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
412 return (ossl_chacha20_poly1305_encrypt(crp, csp));
413 else
414 return (ossl_chacha20_poly1305_decrypt(crp, csp));
415 } else {
416 return (s->cipher.cipher->process(&s->cipher, crp, csp));
417 }
418 }
419
420 static int
ossl_process(device_t dev,struct cryptop * crp,int hint)421 ossl_process(device_t dev, struct cryptop *crp, int hint)
422 {
423 const struct crypto_session_params *csp;
424 struct ossl_session *s;
425 int error;
426 bool fpu_entered;
427
428 s = crypto_get_driver_session(crp->crp_session);
429 csp = crypto_get_params(crp->crp_session);
430
431 if (is_fpu_kern_thread(0)) {
432 fpu_entered = false;
433 } else {
434 fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);
435 fpu_entered = true;
436 }
437
438 switch (csp->csp_mode) {
439 case CSP_MODE_DIGEST:
440 error = ossl_process_hash(s, crp, csp);
441 break;
442 case CSP_MODE_CIPHER:
443 error = ossl_process_cipher(s, crp, csp);
444 break;
445 case CSP_MODE_ETA:
446 error = ossl_process_eta(s, crp, csp);
447 break;
448 case CSP_MODE_AEAD:
449 error = ossl_process_aead(s, crp, csp);
450 break;
451 default:
452 __assert_unreachable();
453 }
454
455 if (fpu_entered)
456 fpu_kern_leave(curthread, NULL);
457
458 crp->crp_etype = error;
459 crypto_done(crp);
460
461 return (0);
462 }
463
464 static device_method_t ossl_methods[] = {
465 DEVMETHOD(device_identify, ossl_identify),
466 DEVMETHOD(device_probe, ossl_probe),
467 DEVMETHOD(device_attach, ossl_attach),
468 DEVMETHOD(device_detach, ossl_detach),
469
470 DEVMETHOD(cryptodev_probesession, ossl_probesession),
471 DEVMETHOD(cryptodev_newsession, ossl_newsession),
472 DEVMETHOD(cryptodev_process, ossl_process),
473
474 DEVMETHOD_END
475 };
476
477 static driver_t ossl_driver = {
478 "ossl",
479 ossl_methods,
480 sizeof(struct ossl_softc)
481 };
482
483 DRIVER_MODULE(ossl, nexus, ossl_driver, NULL, NULL);
484 MODULE_VERSION(ossl, 1);
485 MODULE_DEPEND(ossl, crypto, 1, 1, 1);
486