1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Netflix Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/counter.h>
34 #include <sys/endian.h>
35 #include <sys/kernel.h>
36 #include <sys/ktls.h>
37 #include <sys/lock.h>
38 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/sysctl.h>
42 #include <sys/uio.h>
43 #include <opencrypto/cryptodev.h>
44
45 struct ocf_session {
46 crypto_session_t sid;
47 crypto_session_t mac_sid;
48 int mac_len;
49 struct mtx lock;
50 bool implicit_iv;
51
52 /* Only used for TLS 1.0 with the implicit IV. */
53 #ifdef INVARIANTS
54 bool in_progress;
55 uint64_t next_seqno;
56 #endif
57 char iv[AES_BLOCK_LEN];
58 };
59
60 struct ocf_operation {
61 struct ocf_session *os;
62 bool done;
63 };
64
65 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
66
67 SYSCTL_DECL(_kern_ipc_tls);
68 SYSCTL_DECL(_kern_ipc_tls_stats);
69
70 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf,
71 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
72 "Kernel TLS offload via OCF stats");
73
74 static COUNTER_U64_DEFINE_EARLY(ocf_tls10_cbc_encrypts);
75 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls10_cbc_encrypts,
76 CTLFLAG_RD, &ocf_tls10_cbc_encrypts,
77 "Total number of OCF TLS 1.0 CBC encryption operations");
78
79 static COUNTER_U64_DEFINE_EARLY(ocf_tls11_cbc_encrypts);
80 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls11_cbc_encrypts,
81 CTLFLAG_RD, &ocf_tls11_cbc_encrypts,
82 "Total number of OCF TLS 1.1/1.2 CBC encryption operations");
83
84 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_decrypts);
85 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_decrypts,
86 CTLFLAG_RD, &ocf_tls12_gcm_decrypts,
87 "Total number of OCF TLS 1.2 GCM decryption operations");
88
89 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_gcm_encrypts);
90 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_encrypts,
91 CTLFLAG_RD, &ocf_tls12_gcm_encrypts,
92 "Total number of OCF TLS 1.2 GCM encryption operations");
93
94 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_decrypts);
95 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_decrypts,
96 CTLFLAG_RD, &ocf_tls12_chacha20_decrypts,
97 "Total number of OCF TLS 1.2 Chacha20-Poly1305 decryption operations");
98
99 static COUNTER_U64_DEFINE_EARLY(ocf_tls12_chacha20_encrypts);
100 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_chacha20_encrypts,
101 CTLFLAG_RD, &ocf_tls12_chacha20_encrypts,
102 "Total number of OCF TLS 1.2 Chacha20-Poly1305 encryption operations");
103
104 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_gcm_encrypts);
105 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_encrypts,
106 CTLFLAG_RD, &ocf_tls13_gcm_encrypts,
107 "Total number of OCF TLS 1.3 GCM encryption operations");
108
109 static COUNTER_U64_DEFINE_EARLY(ocf_tls13_chacha20_encrypts);
110 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_chacha20_encrypts,
111 CTLFLAG_RD, &ocf_tls13_chacha20_encrypts,
112 "Total number of OCF TLS 1.3 Chacha20-Poly1305 encryption operations");
113
114 static COUNTER_U64_DEFINE_EARLY(ocf_inplace);
115 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
116 CTLFLAG_RD, &ocf_inplace,
117 "Total number of OCF in-place operations");
118
119 static COUNTER_U64_DEFINE_EARLY(ocf_separate_output);
120 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
121 CTLFLAG_RD, &ocf_separate_output,
122 "Total number of OCF operations with a separate output buffer");
123
124 static COUNTER_U64_DEFINE_EARLY(ocf_retries);
125 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
126 &ocf_retries,
127 "Number of OCF encryption operation retries");
128
129 static int
ktls_ocf_callback(struct cryptop * crp)130 ktls_ocf_callback(struct cryptop *crp)
131 {
132 struct ocf_operation *oo;
133
134 oo = crp->crp_opaque;
135 mtx_lock(&oo->os->lock);
136 oo->done = true;
137 mtx_unlock(&oo->os->lock);
138 wakeup(oo);
139 return (0);
140 }
141
142 static int
ktls_ocf_dispatch(struct ocf_session * os,struct cryptop * crp)143 ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp)
144 {
145 struct ocf_operation oo;
146 int error;
147
148 oo.os = os;
149 oo.done = false;
150
151 crp->crp_opaque = &oo;
152 crp->crp_callback = ktls_ocf_callback;
153 for (;;) {
154 error = crypto_dispatch(crp);
155 if (error)
156 break;
157
158 mtx_lock(&os->lock);
159 while (!oo.done)
160 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
161 mtx_unlock(&os->lock);
162
163 if (crp->crp_etype != EAGAIN) {
164 error = crp->crp_etype;
165 break;
166 }
167
168 crp->crp_etype = 0;
169 crp->crp_flags &= ~CRYPTO_F_DONE;
170 oo.done = false;
171 counter_u64_add(ocf_retries, 1);
172 }
173 return (error);
174 }
175
176 static int
ktls_ocf_tls_cbc_encrypt(struct ktls_session * tls,const struct tls_record_layer * hdr,uint8_t * trailer,struct iovec * iniov,struct iovec * outiov,int iovcnt,uint64_t seqno,uint8_t record_type __unused)177 ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
178 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
179 struct iovec *outiov, int iovcnt, uint64_t seqno,
180 uint8_t record_type __unused)
181 {
182 struct uio uio, out_uio;
183 struct tls_mac_data ad;
184 struct cryptop crp;
185 struct ocf_session *os;
186 struct iovec iov[iovcnt + 2];
187 struct iovec out_iov[iovcnt + 1];
188 int i, error;
189 uint16_t tls_comp_len;
190 uint8_t pad;
191 bool inplace;
192
193 os = tls->cipher;
194
195 #ifdef INVARIANTS
196 if (os->implicit_iv) {
197 mtx_lock(&os->lock);
198 KASSERT(!os->in_progress,
199 ("concurrent implicit IV encryptions"));
200 if (os->next_seqno != seqno) {
201 printf("KTLS CBC: TLS records out of order. "
202 "Expected %ju, got %ju\n",
203 (uintmax_t)os->next_seqno, (uintmax_t)seqno);
204 mtx_unlock(&os->lock);
205 return (EINVAL);
206 }
207 os->in_progress = true;
208 mtx_unlock(&os->lock);
209 }
210 #endif
211
212 /*
213 * Compute the payload length.
214 *
215 * XXX: This could be easily computed O(1) from the mbuf
216 * fields, but we don't have those accessible here. Can
217 * at least compute inplace as well while we are here.
218 */
219 tls_comp_len = 0;
220 inplace = true;
221 for (i = 0; i < iovcnt; i++) {
222 tls_comp_len += iniov[i].iov_len;
223 if (iniov[i].iov_base != outiov[i].iov_base)
224 inplace = false;
225 }
226
227 /* Initialize the AAD. */
228 ad.seq = htobe64(seqno);
229 ad.type = hdr->tls_type;
230 ad.tls_vmajor = hdr->tls_vmajor;
231 ad.tls_vminor = hdr->tls_vminor;
232 ad.tls_length = htons(tls_comp_len);
233
234 /* First, compute the MAC. */
235 iov[0].iov_base = &ad;
236 iov[0].iov_len = sizeof(ad);
237 memcpy(&iov[1], iniov, sizeof(*iniov) * iovcnt);
238 iov[iovcnt + 1].iov_base = trailer;
239 iov[iovcnt + 1].iov_len = os->mac_len;
240 uio.uio_iov = iov;
241 uio.uio_iovcnt = iovcnt + 2;
242 uio.uio_offset = 0;
243 uio.uio_segflg = UIO_SYSSPACE;
244 uio.uio_td = curthread;
245 uio.uio_resid = sizeof(ad) + tls_comp_len + os->mac_len;
246
247 crypto_initreq(&crp, os->mac_sid);
248 crp.crp_payload_start = 0;
249 crp.crp_payload_length = sizeof(ad) + tls_comp_len;
250 crp.crp_digest_start = crp.crp_payload_length;
251 crp.crp_op = CRYPTO_OP_COMPUTE_DIGEST;
252 crp.crp_flags = CRYPTO_F_CBIMM;
253 crypto_use_uio(&crp, &uio);
254 error = ktls_ocf_dispatch(os, &crp);
255
256 crypto_destroyreq(&crp);
257 if (error) {
258 #ifdef INVARIANTS
259 if (os->implicit_iv) {
260 mtx_lock(&os->lock);
261 os->in_progress = false;
262 mtx_unlock(&os->lock);
263 }
264 #endif
265 return (error);
266 }
267
268 /* Second, add the padding. */
269 pad = (unsigned)(AES_BLOCK_LEN - (tls_comp_len + os->mac_len + 1)) %
270 AES_BLOCK_LEN;
271 for (i = 0; i < pad + 1; i++)
272 trailer[os->mac_len + i] = pad;
273
274 /* Finally, encrypt the record. */
275
276 /*
277 * Don't recopy the input iovec, instead just adjust the
278 * trailer length and skip over the AAD vector in the uio.
279 */
280 iov[iovcnt + 1].iov_len += pad + 1;
281 uio.uio_iov = iov + 1;
282 uio.uio_iovcnt = iovcnt + 1;
283 uio.uio_resid = tls_comp_len + iov[iovcnt + 1].iov_len;
284 KASSERT(uio.uio_resid % AES_BLOCK_LEN == 0,
285 ("invalid encryption size"));
286
287 crypto_initreq(&crp, os->sid);
288 crp.crp_payload_start = 0;
289 crp.crp_payload_length = uio.uio_resid;
290 crp.crp_op = CRYPTO_OP_ENCRYPT;
291 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
292 if (os->implicit_iv)
293 memcpy(crp.crp_iv, os->iv, AES_BLOCK_LEN);
294 else
295 memcpy(crp.crp_iv, hdr + 1, AES_BLOCK_LEN);
296 crypto_use_uio(&crp, &uio);
297 if (!inplace) {
298 memcpy(out_iov, outiov, sizeof(*iniov) * iovcnt);
299 out_iov[iovcnt] = iov[iovcnt + 1];
300 out_uio.uio_iov = out_iov;
301 out_uio.uio_iovcnt = iovcnt + 1;
302 out_uio.uio_offset = 0;
303 out_uio.uio_segflg = UIO_SYSSPACE;
304 out_uio.uio_td = curthread;
305 out_uio.uio_resid = uio.uio_resid;
306 crypto_use_output_uio(&crp, &out_uio);
307 }
308
309 if (os->implicit_iv)
310 counter_u64_add(ocf_tls10_cbc_encrypts, 1);
311 else
312 counter_u64_add(ocf_tls11_cbc_encrypts, 1);
313 if (inplace)
314 counter_u64_add(ocf_inplace, 1);
315 else
316 counter_u64_add(ocf_separate_output, 1);
317 error = ktls_ocf_dispatch(os, &crp);
318
319 crypto_destroyreq(&crp);
320
321 if (os->implicit_iv) {
322 KASSERT(os->mac_len + pad + 1 >= AES_BLOCK_LEN,
323 ("trailer too short to read IV"));
324 memcpy(os->iv, trailer + os->mac_len + pad + 1 - AES_BLOCK_LEN,
325 AES_BLOCK_LEN);
326 #ifdef INVARIANTS
327 mtx_lock(&os->lock);
328 os->next_seqno = seqno + 1;
329 os->in_progress = false;
330 mtx_unlock(&os->lock);
331 #endif
332 }
333 return (error);
334 }
335
336 static int
ktls_ocf_tls12_aead_encrypt(struct ktls_session * tls,const struct tls_record_layer * hdr,uint8_t * trailer,struct iovec * iniov,struct iovec * outiov,int iovcnt,uint64_t seqno,uint8_t record_type __unused)337 ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
338 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
339 struct iovec *outiov, int iovcnt, uint64_t seqno,
340 uint8_t record_type __unused)
341 {
342 struct uio uio, out_uio, *tag_uio;
343 struct tls_aead_data ad;
344 struct cryptop crp;
345 struct ocf_session *os;
346 struct iovec iov[iovcnt + 1];
347 int i, error;
348 uint16_t tls_comp_len;
349 bool inplace;
350
351 os = tls->cipher;
352
353 uio.uio_iov = iniov;
354 uio.uio_iovcnt = iovcnt;
355 uio.uio_offset = 0;
356 uio.uio_segflg = UIO_SYSSPACE;
357 uio.uio_td = curthread;
358
359 out_uio.uio_iov = outiov;
360 out_uio.uio_iovcnt = iovcnt;
361 out_uio.uio_offset = 0;
362 out_uio.uio_segflg = UIO_SYSSPACE;
363 out_uio.uio_td = curthread;
364
365 crypto_initreq(&crp, os->sid);
366
367 /* Setup the IV. */
368 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
369 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
370 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
371 sizeof(uint64_t));
372 } else {
373 /*
374 * Chacha20-Poly1305 constructs the IV for TLS 1.2
375 * identically to constructing the IV for AEAD in TLS
376 * 1.3.
377 */
378 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
379 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
380 }
381
382 /* Setup the AAD. */
383 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
384 tls_comp_len = ntohs(hdr->tls_length) -
385 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
386 else
387 tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
388 ad.seq = htobe64(seqno);
389 ad.type = hdr->tls_type;
390 ad.tls_vmajor = hdr->tls_vmajor;
391 ad.tls_vminor = hdr->tls_vminor;
392 ad.tls_length = htons(tls_comp_len);
393 crp.crp_aad = &ad;
394 crp.crp_aad_length = sizeof(ad);
395
396 /* Compute payload length and determine if encryption is in place. */
397 inplace = true;
398 crp.crp_payload_start = 0;
399 for (i = 0; i < iovcnt; i++) {
400 if (iniov[i].iov_base != outiov[i].iov_base)
401 inplace = false;
402 crp.crp_payload_length += iniov[i].iov_len;
403 }
404 uio.uio_resid = crp.crp_payload_length;
405 out_uio.uio_resid = crp.crp_payload_length;
406
407 if (inplace)
408 tag_uio = &uio;
409 else
410 tag_uio = &out_uio;
411
412 /* Duplicate iovec and append vector for tag. */
413 memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
414 iov[iovcnt].iov_base = trailer;
415 iov[iovcnt].iov_len = tls->params.tls_tlen;
416 tag_uio->uio_iov = iov;
417 tag_uio->uio_iovcnt++;
418 crp.crp_digest_start = tag_uio->uio_resid;
419 tag_uio->uio_resid += tls->params.tls_tlen;
420
421 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
422 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
423 crypto_use_uio(&crp, &uio);
424 if (!inplace)
425 crypto_use_output_uio(&crp, &out_uio);
426
427 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
428 counter_u64_add(ocf_tls12_gcm_encrypts, 1);
429 else
430 counter_u64_add(ocf_tls12_chacha20_encrypts, 1);
431 if (inplace)
432 counter_u64_add(ocf_inplace, 1);
433 else
434 counter_u64_add(ocf_separate_output, 1);
435 error = ktls_ocf_dispatch(os, &crp);
436
437 crypto_destroyreq(&crp);
438 return (error);
439 }
440
441 static int
ktls_ocf_tls12_aead_decrypt(struct ktls_session * tls,const struct tls_record_layer * hdr,struct mbuf * m,uint64_t seqno,int * trailer_len)442 ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,
443 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
444 int *trailer_len)
445 {
446 struct tls_aead_data ad;
447 struct cryptop crp;
448 struct ocf_session *os;
449 struct ocf_operation oo;
450 int error;
451 uint16_t tls_comp_len;
452
453 os = tls->cipher;
454
455 oo.os = os;
456 oo.done = false;
457
458 crypto_initreq(&crp, os->sid);
459
460 /* Setup the IV. */
461 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
462 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
463 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1,
464 sizeof(uint64_t));
465 } else {
466 /*
467 * Chacha20-Poly1305 constructs the IV for TLS 1.2
468 * identically to constructing the IV for AEAD in TLS
469 * 1.3.
470 */
471 memcpy(crp.crp_iv, tls->params.iv, tls->params.iv_len);
472 *(uint64_t *)(crp.crp_iv + 4) ^= htobe64(seqno);
473 }
474
475 /* Setup the AAD. */
476 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
477 tls_comp_len = ntohs(hdr->tls_length) -
478 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
479 else
480 tls_comp_len = ntohs(hdr->tls_length) - POLY1305_HASH_LEN;
481 ad.seq = htobe64(seqno);
482 ad.type = hdr->tls_type;
483 ad.tls_vmajor = hdr->tls_vmajor;
484 ad.tls_vminor = hdr->tls_vminor;
485 ad.tls_length = htons(tls_comp_len);
486 crp.crp_aad = &ad;
487 crp.crp_aad_length = sizeof(ad);
488
489 crp.crp_payload_start = tls->params.tls_hlen;
490 crp.crp_payload_length = tls_comp_len;
491 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
492
493 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
494 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
495 crypto_use_mbuf(&crp, m);
496
497 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
498 counter_u64_add(ocf_tls12_gcm_decrypts, 1);
499 else
500 counter_u64_add(ocf_tls12_chacha20_decrypts, 1);
501 error = ktls_ocf_dispatch(os, &crp);
502
503 crypto_destroyreq(&crp);
504 *trailer_len = tls->params.tls_tlen;
505 return (error);
506 }
507
508 static int
ktls_ocf_tls13_aead_encrypt(struct ktls_session * tls,const struct tls_record_layer * hdr,uint8_t * trailer,struct iovec * iniov,struct iovec * outiov,int iovcnt,uint64_t seqno,uint8_t record_type)509 ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
510 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
511 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
512 {
513 struct uio uio, out_uio;
514 struct tls_aead_data_13 ad;
515 char nonce[12];
516 struct cryptop crp;
517 struct ocf_session *os;
518 struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1];
519 int i, error;
520 bool inplace;
521
522 os = tls->cipher;
523
524 crypto_initreq(&crp, os->sid);
525
526 /* Setup the nonce. */
527 memcpy(nonce, tls->params.iv, tls->params.iv_len);
528 *(uint64_t *)(nonce + 4) ^= htobe64(seqno);
529
530 /* Setup the AAD. */
531 ad.type = hdr->tls_type;
532 ad.tls_vmajor = hdr->tls_vmajor;
533 ad.tls_vminor = hdr->tls_vminor;
534 ad.tls_length = hdr->tls_length;
535 crp.crp_aad = &ad;
536 crp.crp_aad_length = sizeof(ad);
537
538 /* Compute payload length and determine if encryption is in place. */
539 inplace = true;
540 crp.crp_payload_start = 0;
541 for (i = 0; i < iovcnt; i++) {
542 if (iniov[i].iov_base != outiov[i].iov_base)
543 inplace = false;
544 crp.crp_payload_length += iniov[i].iov_len;
545 }
546
547 /* Store the record type as the first byte of the trailer. */
548 trailer[0] = record_type;
549 crp.crp_payload_length++;
550 crp.crp_digest_start = crp.crp_payload_length;
551
552 /*
553 * Duplicate the input iov to append the trailer. Always
554 * include the full trailer as input to get the record_type
555 * even if only the first byte is used.
556 */
557 memcpy(iov, iniov, iovcnt * sizeof(*iov));
558 iov[iovcnt].iov_base = trailer;
559 iov[iovcnt].iov_len = tls->params.tls_tlen;
560 uio.uio_iov = iov;
561 uio.uio_iovcnt = iovcnt + 1;
562 uio.uio_offset = 0;
563 uio.uio_resid = crp.crp_payload_length + tls->params.tls_tlen - 1;
564 uio.uio_segflg = UIO_SYSSPACE;
565 uio.uio_td = curthread;
566 crypto_use_uio(&crp, &uio);
567
568 if (!inplace) {
569 /* Duplicate the output iov to append the trailer. */
570 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
571 out_iov[iovcnt] = iov[iovcnt];
572
573 out_uio.uio_iov = out_iov;
574 out_uio.uio_iovcnt = iovcnt + 1;
575 out_uio.uio_offset = 0;
576 out_uio.uio_resid = crp.crp_payload_length +
577 tls->params.tls_tlen - 1;
578 out_uio.uio_segflg = UIO_SYSSPACE;
579 out_uio.uio_td = curthread;
580 crypto_use_output_uio(&crp, &out_uio);
581 }
582
583 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
584 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
585
586 memcpy(crp.crp_iv, nonce, sizeof(nonce));
587
588 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
589 counter_u64_add(ocf_tls13_gcm_encrypts, 1);
590 else
591 counter_u64_add(ocf_tls13_chacha20_encrypts, 1);
592 if (inplace)
593 counter_u64_add(ocf_inplace, 1);
594 else
595 counter_u64_add(ocf_separate_output, 1);
596 error = ktls_ocf_dispatch(os, &crp);
597
598 crypto_destroyreq(&crp);
599 return (error);
600 }
601
602 static void
ktls_ocf_free(struct ktls_session * tls)603 ktls_ocf_free(struct ktls_session *tls)
604 {
605 struct ocf_session *os;
606
607 os = tls->cipher;
608 crypto_freesession(os->sid);
609 mtx_destroy(&os->lock);
610 zfree(os, M_KTLS_OCF);
611 }
612
613 static int
ktls_ocf_try(struct socket * so,struct ktls_session * tls,int direction)614 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
615 {
616 struct crypto_session_params csp, mac_csp;
617 struct ocf_session *os;
618 int error, mac_len;
619
620 memset(&csp, 0, sizeof(csp));
621 memset(&mac_csp, 0, sizeof(mac_csp));
622 mac_csp.csp_mode = CSP_MODE_NONE;
623 mac_len = 0;
624
625 switch (tls->params.cipher_algorithm) {
626 case CRYPTO_AES_NIST_GCM_16:
627 switch (tls->params.cipher_key_len) {
628 case 128 / 8:
629 case 256 / 8:
630 break;
631 default:
632 return (EINVAL);
633 }
634
635 /* Only TLS 1.2 and 1.3 are supported. */
636 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
637 tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
638 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
639 return (EPROTONOSUPPORT);
640
641 /* TLS 1.3 is not yet supported for receive. */
642 if (direction == KTLS_RX &&
643 tls->params.tls_vminor == TLS_MINOR_VER_THREE)
644 return (EPROTONOSUPPORT);
645
646 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
647 csp.csp_mode = CSP_MODE_AEAD;
648 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
649 csp.csp_cipher_key = tls->params.cipher_key;
650 csp.csp_cipher_klen = tls->params.cipher_key_len;
651 csp.csp_ivlen = AES_GCM_IV_LEN;
652 break;
653 case CRYPTO_AES_CBC:
654 switch (tls->params.cipher_key_len) {
655 case 128 / 8:
656 case 256 / 8:
657 break;
658 default:
659 return (EINVAL);
660 }
661
662 switch (tls->params.auth_algorithm) {
663 case CRYPTO_SHA1_HMAC:
664 mac_len = SHA1_HASH_LEN;
665 break;
666 case CRYPTO_SHA2_256_HMAC:
667 mac_len = SHA2_256_HASH_LEN;
668 break;
669 case CRYPTO_SHA2_384_HMAC:
670 mac_len = SHA2_384_HASH_LEN;
671 break;
672 default:
673 return (EINVAL);
674 }
675
676 /* Only TLS 1.0-1.2 are supported. */
677 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
678 tls->params.tls_vminor < TLS_MINOR_VER_ZERO ||
679 tls->params.tls_vminor > TLS_MINOR_VER_TWO)
680 return (EPROTONOSUPPORT);
681
682 /* AES-CBC is not supported for receive. */
683 if (direction == KTLS_RX)
684 return (EPROTONOSUPPORT);
685
686 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
687 csp.csp_mode = CSP_MODE_CIPHER;
688 csp.csp_cipher_alg = CRYPTO_AES_CBC;
689 csp.csp_cipher_key = tls->params.cipher_key;
690 csp.csp_cipher_klen = tls->params.cipher_key_len;
691 csp.csp_ivlen = AES_BLOCK_LEN;
692
693 mac_csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
694 mac_csp.csp_mode = CSP_MODE_DIGEST;
695 mac_csp.csp_auth_alg = tls->params.auth_algorithm;
696 mac_csp.csp_auth_key = tls->params.auth_key;
697 mac_csp.csp_auth_klen = tls->params.auth_key_len;
698 break;
699 case CRYPTO_CHACHA20_POLY1305:
700 switch (tls->params.cipher_key_len) {
701 case 256 / 8:
702 break;
703 default:
704 return (EINVAL);
705 }
706
707 /* Only TLS 1.2 and 1.3 are supported. */
708 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
709 tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
710 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
711 return (EPROTONOSUPPORT);
712
713 /* TLS 1.3 is not yet supported for receive. */
714 if (direction == KTLS_RX &&
715 tls->params.tls_vminor == TLS_MINOR_VER_THREE)
716 return (EPROTONOSUPPORT);
717
718 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
719 csp.csp_mode = CSP_MODE_AEAD;
720 csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
721 csp.csp_cipher_key = tls->params.cipher_key;
722 csp.csp_cipher_klen = tls->params.cipher_key_len;
723 csp.csp_ivlen = CHACHA20_POLY1305_IV_LEN;
724 break;
725 default:
726 return (EPROTONOSUPPORT);
727 }
728
729 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
730 if (os == NULL)
731 return (ENOMEM);
732
733 error = crypto_newsession(&os->sid, &csp,
734 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
735 if (error) {
736 free(os, M_KTLS_OCF);
737 return (error);
738 }
739
740 if (mac_csp.csp_mode != CSP_MODE_NONE) {
741 error = crypto_newsession(&os->mac_sid, &mac_csp,
742 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
743 if (error) {
744 crypto_freesession(os->sid);
745 free(os, M_KTLS_OCF);
746 return (error);
747 }
748 os->mac_len = mac_len;
749 }
750
751 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
752 tls->cipher = os;
753 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 ||
754 tls->params.cipher_algorithm == CRYPTO_CHACHA20_POLY1305) {
755 if (direction == KTLS_TX) {
756 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
757 tls->sw_encrypt = ktls_ocf_tls13_aead_encrypt;
758 else
759 tls->sw_encrypt = ktls_ocf_tls12_aead_encrypt;
760 } else {
761 tls->sw_decrypt = ktls_ocf_tls12_aead_decrypt;
762 }
763 } else {
764 tls->sw_encrypt = ktls_ocf_tls_cbc_encrypt;
765 if (tls->params.tls_vminor == TLS_MINOR_VER_ZERO) {
766 os->implicit_iv = true;
767 memcpy(os->iv, tls->params.iv, AES_BLOCK_LEN);
768 #ifdef INVARIANTS
769 os->next_seqno = tls->next_seqno;
770 #endif
771 }
772 }
773 tls->free = ktls_ocf_free;
774 return (0);
775 }
776
777 struct ktls_crypto_backend ocf_backend = {
778 .name = "OCF",
779 .prio = 5,
780 .api_version = KTLS_API_VERSION,
781 .try = ktls_ocf_try,
782 };
783
784 static int
ktls_ocf_modevent(module_t mod,int what,void * arg)785 ktls_ocf_modevent(module_t mod, int what, void *arg)
786 {
787 switch (what) {
788 case MOD_LOAD:
789 return (ktls_crypto_backend_register(&ocf_backend));
790 case MOD_UNLOAD:
791 return (ktls_crypto_backend_deregister(&ocf_backend));
792 default:
793 return (EOPNOTSUPP);
794 }
795 }
796
797 static moduledata_t ktls_ocf_moduledata = {
798 "ktls_ocf",
799 ktls_ocf_modevent,
800 NULL
801 };
802
803 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);
804