1 /*-
2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <[email protected]>
3 * Copyright (c) 2010 Konstantin Belousov <[email protected]>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
6 * All rights reserved.
7 *
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
11 *
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37 /*
38 * This is based on the aesni code.
39 */
40
41 #include <sys/cdefs.h>
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/bus.h>
46 #include <sys/endian.h>
47 #include <sys/malloc.h>
48 #include <sys/mbuf.h>
49 #include <sys/module.h>
50 #include <sys/queue.h>
51 #include <sys/smp.h>
52 #include <sys/uio.h>
53
54 #include <machine/vfp.h>
55
56 #include <opencrypto/cryptodev.h>
57 #include <opencrypto/gmac.h>
58 #include <cryptodev_if.h>
59 #include <crypto/armv8/armv8_crypto.h>
60 #include <crypto/rijndael/rijndael.h>
61
62 struct armv8_crypto_softc {
63 int32_t cid;
64 bool has_pmul;
65 };
66
67 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
68 struct cryptop *);
69
70 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
71
72 static void
armv8_crypto_identify(driver_t * drv,device_t parent)73 armv8_crypto_identify(driver_t *drv, device_t parent)
74 {
75
76 /* NB: order 10 is so we get attached after h/w devices */
77 if (device_find_child(parent, "armv8crypto", -1) == NULL &&
78 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
79 panic("ARMv8 crypto: could not attach");
80 }
81
82 static int
armv8_crypto_probe(device_t dev)83 armv8_crypto_probe(device_t dev)
84 {
85 uint64_t reg;
86 int ret = ENXIO;
87
88 reg = READ_SPECIALREG(id_aa64isar0_el1);
89
90 switch (ID_AA64ISAR0_AES_VAL(reg)) {
91 case ID_AA64ISAR0_AES_BASE:
92 ret = 0;
93 device_set_desc(dev, "AES-CBC,AES-XTS");
94 break;
95 case ID_AA64ISAR0_AES_PMULL:
96 ret = 0;
97 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
98 break;
99 default:
100 break;
101 case ID_AA64ISAR0_AES_NONE:
102 device_printf(dev, "CPU lacks AES instructions\n");
103 break;
104 }
105
106 /* TODO: Check more fields as we support more features */
107
108 return (ret);
109 }
110
111 static int
armv8_crypto_attach(device_t dev)112 armv8_crypto_attach(device_t dev)
113 {
114 struct armv8_crypto_softc *sc;
115 uint64_t reg;
116
117 sc = device_get_softc(dev);
118
119 reg = READ_SPECIALREG(id_aa64isar0_el1);
120
121 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
122 sc->has_pmul = true;
123
124 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
125 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
126 if (sc->cid < 0) {
127 device_printf(dev, "Could not get crypto driver id.\n");
128 return (ENOMEM);
129 }
130
131 return (0);
132 }
133
134 static int
armv8_crypto_detach(device_t dev)135 armv8_crypto_detach(device_t dev)
136 {
137 struct armv8_crypto_softc *sc;
138
139 sc = device_get_softc(dev);
140
141 crypto_unregister_all(sc->cid);
142
143 return (0);
144 }
145
146 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
147
148 static int
armv8_crypto_probesession(device_t dev,const struct crypto_session_params * csp)149 armv8_crypto_probesession(device_t dev,
150 const struct crypto_session_params *csp)
151 {
152 struct armv8_crypto_softc *sc;
153
154 sc = device_get_softc(dev);
155
156 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
157 return (EINVAL);
158
159 switch (csp->csp_mode) {
160 case CSP_MODE_AEAD:
161 switch (csp->csp_cipher_alg) {
162 case CRYPTO_AES_NIST_GCM_16:
163 if (!sc->has_pmul)
164 return (EINVAL);
165 if (csp->csp_auth_mlen != 0 &&
166 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
167 return (EINVAL);
168 switch (csp->csp_cipher_klen * 8) {
169 case 128:
170 case 192:
171 case 256:
172 break;
173 default:
174 return (EINVAL);
175 }
176 break;
177 default:
178 return (EINVAL);
179 }
180 break;
181 case CSP_MODE_CIPHER:
182 switch (csp->csp_cipher_alg) {
183 case CRYPTO_AES_CBC:
184 if (csp->csp_ivlen != AES_BLOCK_LEN)
185 return (EINVAL);
186 switch (csp->csp_cipher_klen * 8) {
187 case 128:
188 case 192:
189 case 256:
190 break;
191 default:
192 return (EINVAL);
193 }
194 break;
195 case CRYPTO_AES_XTS:
196 if (csp->csp_ivlen != AES_XTS_IV_LEN)
197 return (EINVAL);
198 switch (csp->csp_cipher_klen * 8) {
199 case 256:
200 case 512:
201 break;
202 default:
203 return (EINVAL);
204 }
205 break;
206 default:
207 return (EINVAL);
208 }
209 break;
210 default:
211 return (EINVAL);
212 }
213 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
214 }
215
216 static int
armv8_crypto_cipher_setup(struct armv8_crypto_session * ses,const struct crypto_session_params * csp,const uint8_t * key,int keylen)217 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
218 const struct crypto_session_params *csp, const uint8_t *key, int keylen)
219 {
220 __uint128_val_t H;
221
222 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
223 keylen /= 2;
224
225 switch (keylen * 8) {
226 case 128:
227 case 192:
228 case 256:
229 break;
230 default:
231 return (EINVAL);
232 }
233
234 fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
235
236 aes_v8_set_encrypt_key(key,
237 keylen * 8, &ses->enc_schedule);
238
239 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
240 (csp->csp_cipher_alg == CRYPTO_AES_CBC))
241 aes_v8_set_decrypt_key(key,
242 keylen * 8, &ses->dec_schedule);
243
244 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
245 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
246
247 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
248 memset(H.c, 0, sizeof(H.c));
249 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
250 H.u[0] = bswap64(H.u[0]);
251 H.u[1] = bswap64(H.u[1]);
252 gcm_init_v8(ses->Htable, H.u);
253 }
254
255 fpu_kern_leave(curthread, NULL);
256
257 return (0);
258 }
259
260 static int
armv8_crypto_newsession(device_t dev,crypto_session_t cses,const struct crypto_session_params * csp)261 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
262 const struct crypto_session_params *csp)
263 {
264 struct armv8_crypto_session *ses;
265 int error;
266
267 ses = crypto_get_driver_session(cses);
268 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
269 csp->csp_cipher_klen);
270 return (error);
271 }
272
273 static int
armv8_crypto_process(device_t dev,struct cryptop * crp,int hint __unused)274 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
275 {
276 struct armv8_crypto_session *ses;
277
278 ses = crypto_get_driver_session(crp->crp_session);
279 crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
280 crypto_done(crp);
281 return (0);
282 }
283
284 static uint8_t *
armv8_crypto_cipher_alloc(struct cryptop * crp,int start,int length,int * allocated)285 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
286 {
287 uint8_t *addr;
288
289 addr = crypto_contiguous_subsegment(crp, start, length);
290 if (addr != NULL) {
291 *allocated = 0;
292 return (addr);
293 }
294 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
295 if (addr != NULL) {
296 *allocated = 1;
297 crypto_copydata(crp, start, length, addr);
298 } else
299 *allocated = 0;
300 return (addr);
301 }
302
303 static int
armv8_crypto_cipher_process(struct armv8_crypto_session * ses,struct cryptop * crp)304 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
305 struct cryptop *crp)
306 {
307 struct crypto_buffer_cursor fromc, toc;
308 const struct crypto_session_params *csp;
309 uint8_t *authbuf;
310 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
311 int authallocated;
312 int encflag;
313 int error;
314
315 csp = crypto_get_params(crp->crp_session);
316 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
317
318 authallocated = 0;
319 authbuf = NULL;
320
321 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
322 if (crp->crp_aad != NULL)
323 authbuf = crp->crp_aad;
324 else
325 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
326 crp->crp_aad_length, &authallocated);
327 if (authbuf == NULL)
328 return (ENOMEM);
329 }
330 crypto_cursor_init(&fromc, &crp->crp_buf);
331 crypto_cursor_advance(&fromc, crp->crp_payload_start);
332 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
333 crypto_cursor_init(&toc, &crp->crp_obuf);
334 crypto_cursor_advance(&toc, crp->crp_payload_output_start);
335 } else {
336 crypto_cursor_copy(&fromc, &toc);
337 }
338
339 if (crp->crp_cipher_key != NULL) {
340 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
341 csp->csp_cipher_klen);
342 }
343
344 crypto_read_iv(crp, iv);
345
346 fpu_kern_enter(curthread, NULL, FPU_KERN_NORMAL | FPU_KERN_NOCTX);
347
348 error = 0;
349 switch (csp->csp_cipher_alg) {
350 case CRYPTO_AES_CBC:
351 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
352 error = EINVAL;
353 break;
354 }
355 if (encflag)
356 armv8_aes_encrypt_cbc(&ses->enc_schedule,
357 crp->crp_payload_length, &fromc, &toc, iv);
358 else
359 armv8_aes_decrypt_cbc(&ses->dec_schedule,
360 crp->crp_payload_length, &fromc, &toc, iv);
361 break;
362 case CRYPTO_AES_XTS:
363 if (encflag)
364 armv8_aes_encrypt_xts(&ses->enc_schedule,
365 &ses->xts_schedule.aes_key, crp->crp_payload_length,
366 &fromc, &toc, iv);
367 else
368 armv8_aes_decrypt_xts(&ses->dec_schedule,
369 &ses->xts_schedule.aes_key, crp->crp_payload_length,
370 &fromc, &toc, iv);
371 break;
372 case CRYPTO_AES_NIST_GCM_16:
373 if (encflag) {
374 memset(tag, 0, sizeof(tag));
375 armv8_aes_encrypt_gcm(&ses->enc_schedule,
376 crp->crp_payload_length, &fromc, &toc,
377 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
378 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
379 tag);
380 } else {
381 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
382 tag);
383 error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
384 crp->crp_payload_length, &fromc, &toc,
385 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
386 }
387 break;
388 }
389
390 fpu_kern_leave(curthread, NULL);
391
392 if (authallocated)
393 zfree(authbuf, M_ARMV8_CRYPTO);
394 explicit_bzero(iv, sizeof(iv));
395 explicit_bzero(tag, sizeof(tag));
396
397 return (error);
398 }
399
400 static device_method_t armv8_crypto_methods[] = {
401 DEVMETHOD(device_identify, armv8_crypto_identify),
402 DEVMETHOD(device_probe, armv8_crypto_probe),
403 DEVMETHOD(device_attach, armv8_crypto_attach),
404 DEVMETHOD(device_detach, armv8_crypto_detach),
405
406 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
407 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
408 DEVMETHOD(cryptodev_process, armv8_crypto_process),
409
410 DEVMETHOD_END,
411 };
412
413 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
414 sizeof(struct armv8_crypto_softc));
415
416 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, 0, 0);
417