1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2017-2018 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: John Baldwin <[email protected]>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_inet.h"
31 #include "opt_kern_tls.h"
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 #include <sys/param.h>
37 #include <sys/ktr.h>
38 #ifdef KERN_TLS
39 #include <sys/ktls.h>
40 #endif
41 #include <sys/sglist.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/systm.h>
45 #include <netinet/in.h>
46 #include <netinet/in_pcb.h>
47 #include <netinet/tcp_var.h>
48 #include <netinet/toecore.h>
49 #ifdef KERN_TLS
50 #include <opencrypto/cryptodev.h>
51 #include <opencrypto/xform.h>
52 #endif
53
54 #ifdef TCP_OFFLOAD
55 #include "common/common.h"
56 #include "common/t4_tcb.h"
57 #include "crypto/t4_crypto.h"
58 #include "tom/t4_tom_l2t.h"
59 #include "tom/t4_tom.h"
60
61 /*
62 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while
63 * the mbuf is in the ulp_pdu_reclaimq.
64 */
65 #define tls_tcp_seq PH_loc.thirtytwo[0]
66
67 static void
t4_set_tls_tcb_field(struct toepcb * toep,uint16_t word,uint64_t mask,uint64_t val)68 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
69 uint64_t val)
70 {
71 struct adapter *sc = td_adapter(toep->td);
72
73 t4_set_tcb_field(sc, &toep->ofld_txq->wrq, toep, word, mask, val, 0, 0);
74 }
75
76 /* TLS and DTLS common routines */
77 bool
can_tls_offload(struct adapter * sc)78 can_tls_offload(struct adapter *sc)
79 {
80
81 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS);
82 }
83
84 int
tls_tx_key(struct toepcb * toep)85 tls_tx_key(struct toepcb *toep)
86 {
87 struct tls_ofld_info *tls_ofld = &toep->tls;
88
89 return (tls_ofld->tx_key_addr >= 0);
90 }
91
92 int
tls_rx_key(struct toepcb * toep)93 tls_rx_key(struct toepcb *toep)
94 {
95 struct tls_ofld_info *tls_ofld = &toep->tls;
96
97 return (tls_ofld->rx_key_addr >= 0);
98 }
99
100 static int
key_size(struct toepcb * toep)101 key_size(struct toepcb *toep)
102 {
103 struct tls_ofld_info *tls_ofld = &toep->tls;
104
105 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ?
106 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE);
107 }
108
109 /* Set TLS Key-Id in TCB */
110 static void
t4_set_tls_keyid(struct toepcb * toep,unsigned int key_id)111 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id)
112 {
113
114 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG,
115 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG),
116 V_TCB_RX_TLS_KEY_TAG(key_id));
117 }
118
119 /* Clear TF_RX_QUIESCE to re-enable receive. */
120 static void
t4_clear_rx_quiesce(struct toepcb * toep)121 t4_clear_rx_quiesce(struct toepcb *toep)
122 {
123
124 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
125 }
126
127 static void
tls_clr_ofld_mode(struct toepcb * toep)128 tls_clr_ofld_mode(struct toepcb *toep)
129 {
130
131 tls_stop_handshake_timer(toep);
132
133 KASSERT(toep->tls.rx_key_addr == -1,
134 ("%s: tid %d has RX key", __func__, toep->tid));
135
136 /* Switch to plain TOE mode. */
137 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
138 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)),
139 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(0)));
140 t4_set_tls_tcb_field(toep, W_TCB_ULP_TYPE,
141 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE), V_TCB_ULP_TYPE(ULP_MODE_NONE));
142 t4_clear_rx_quiesce(toep);
143
144 toep->flags &= ~(TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED);
145 toep->params.ulp_mode = ULP_MODE_NONE;
146 }
147
148 static void
tls_clr_quiesce(struct toepcb * toep)149 tls_clr_quiesce(struct toepcb *toep)
150 {
151
152 tls_stop_handshake_timer(toep);
153 t4_clear_rx_quiesce(toep);
154 }
155
156 /*
157 * Calculate the TLS data expansion size
158 */
159 static int
tls_expansion_size(struct toepcb * toep,int data_len,int full_pdus_only,unsigned short * pdus_per_ulp)160 tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only,
161 unsigned short *pdus_per_ulp)
162 {
163 struct tls_ofld_info *tls_ofld = &toep->tls;
164 struct tls_scmd *scmd = &tls_ofld->scmd0;
165 int expn_size = 0, frag_count = 0, pad_per_pdu = 0,
166 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0;
167 int exp_per_pdu = 0;
168 int hdr_len = TLS_HEADER_LENGTH;
169
170 do {
171 max_frag_size = tls_ofld->k_ctx.frag_size;
172 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) ==
173 SCMD_CIPH_MODE_AES_GCM) {
174 frag_count = (data_len / max_frag_size);
175 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
176 hdr_len;
177 expn_size = frag_count * exp_per_pdu;
178 if (full_pdus_only) {
179 *pdus_per_ulp = data_len / (exp_per_pdu +
180 max_frag_size);
181 if (*pdus_per_ulp > 32)
182 *pdus_per_ulp = 32;
183 else if(!*pdus_per_ulp)
184 *pdus_per_ulp = 1;
185 expn_size = (*pdus_per_ulp) * exp_per_pdu;
186 break;
187 }
188 if ((last_frag_size = data_len % max_frag_size) > 0) {
189 frag_count += 1;
190 expn_size += exp_per_pdu;
191 }
192 break;
193 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) !=
194 SCMD_CIPH_MODE_NOP) {
195 /* Calculate the number of fragments we can make */
196 frag_count = (data_len / max_frag_size);
197 if (frag_count > 0) {
198 pad_per_pdu = (((howmany((max_frag_size +
199 tls_ofld->mac_length),
200 CIPHER_BLOCK_SIZE)) *
201 CIPHER_BLOCK_SIZE) -
202 (max_frag_size +
203 tls_ofld->mac_length));
204 if (!pad_per_pdu)
205 pad_per_pdu = CIPHER_BLOCK_SIZE;
206 exp_per_pdu = pad_per_pdu +
207 tls_ofld->mac_length +
208 hdr_len + CIPHER_BLOCK_SIZE;
209 expn_size = frag_count * exp_per_pdu;
210 }
211 if (full_pdus_only) {
212 *pdus_per_ulp = data_len / (exp_per_pdu +
213 max_frag_size);
214 if (*pdus_per_ulp > 32)
215 *pdus_per_ulp = 32;
216 else if (!*pdus_per_ulp)
217 *pdus_per_ulp = 1;
218 expn_size = (*pdus_per_ulp) * exp_per_pdu;
219 break;
220 }
221 /* Consider the last fragment */
222 if ((last_frag_size = data_len % max_frag_size) > 0) {
223 pad_last_pdu = (((howmany((last_frag_size +
224 tls_ofld->mac_length),
225 CIPHER_BLOCK_SIZE)) *
226 CIPHER_BLOCK_SIZE) -
227 (last_frag_size +
228 tls_ofld->mac_length));
229 if (!pad_last_pdu)
230 pad_last_pdu = CIPHER_BLOCK_SIZE;
231 expn_size += (pad_last_pdu +
232 tls_ofld->mac_length + hdr_len +
233 CIPHER_BLOCK_SIZE);
234 }
235 }
236 } while (0);
237
238 return (expn_size);
239 }
240
241 /* Copy Key to WR */
242 static void
tls_copy_tx_key(struct toepcb * toep,void * dst)243 tls_copy_tx_key(struct toepcb *toep, void *dst)
244 {
245 struct tls_ofld_info *tls_ofld = &toep->tls;
246 struct ulptx_sc_memrd *sc_memrd;
247 struct ulptx_idata *sc;
248
249 if (tls_ofld->k_ctx.tx_key_info_size <= 0)
250 return;
251
252 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) {
253 sc = dst;
254 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
255 sc->len = htobe32(0);
256 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
257 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
258 V_ULP_TX_SC_MORE(1) |
259 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4));
260 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5);
261 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) {
262 memcpy(dst, &tls_ofld->k_ctx.tx,
263 tls_ofld->k_ctx.tx_key_info_size);
264 }
265 }
266
267 /* TLS/DTLS content type for CPL SFO */
268 static inline unsigned char
tls_content_type(unsigned char content_type)269 tls_content_type(unsigned char content_type)
270 {
271 /*
272 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and
273 * default to "CUSTOM" for all other types including
274 * heartbeat?
275 */
276 switch (content_type) {
277 case CONTENT_TYPE_CCS:
278 return CPL_TX_TLS_SFO_TYPE_CCS;
279 case CONTENT_TYPE_ALERT:
280 return CPL_TX_TLS_SFO_TYPE_ALERT;
281 case CONTENT_TYPE_HANDSHAKE:
282 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
283 case CONTENT_TYPE_HEARTBEAT:
284 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
285 }
286 return CPL_TX_TLS_SFO_TYPE_DATA;
287 }
288
289 static unsigned char
get_cipher_key_size(unsigned int ck_size)290 get_cipher_key_size(unsigned int ck_size)
291 {
292 switch (ck_size) {
293 case AES_NOP: /* NOP */
294 return 15;
295 case AES_128: /* AES128 */
296 return CH_CK_SIZE_128;
297 case AES_192: /* AES192 */
298 return CH_CK_SIZE_192;
299 case AES_256: /* AES256 */
300 return CH_CK_SIZE_256;
301 default:
302 return CH_CK_SIZE_256;
303 }
304 }
305
306 static unsigned char
get_mac_key_size(unsigned int mk_size)307 get_mac_key_size(unsigned int mk_size)
308 {
309 switch (mk_size) {
310 case SHA_NOP: /* NOP */
311 return CH_MK_SIZE_128;
312 case SHA_GHASH: /* GHASH */
313 case SHA_512: /* SHA512 */
314 return CH_MK_SIZE_512;
315 case SHA_224: /* SHA2-224 */
316 return CH_MK_SIZE_192;
317 case SHA_256: /* SHA2-256*/
318 return CH_MK_SIZE_256;
319 case SHA_384: /* SHA384 */
320 return CH_MK_SIZE_512;
321 case SHA1: /* SHA1 */
322 default:
323 return CH_MK_SIZE_160;
324 }
325 }
326
327 static unsigned int
get_proto_ver(int proto_ver)328 get_proto_ver(int proto_ver)
329 {
330 switch (proto_ver) {
331 case TLS1_2_VERSION:
332 return TLS_1_2_VERSION;
333 case TLS1_1_VERSION:
334 return TLS_1_1_VERSION;
335 case DTLS1_2_VERSION:
336 return DTLS_1_2_VERSION;
337 default:
338 return TLS_VERSION_MAX;
339 }
340 }
341
342 static void
tls_rxkey_flit1(struct tls_keyctx * kwr,struct tls_key_context * kctx)343 tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx)
344 {
345
346 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
347 kwr->u.rxhdr.ivinsert_to_authinsrt =
348 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
349 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
350 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
351 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
352 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
353 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
354 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
355 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
356 kwr->u.rxhdr.ivpresent_to_rxmk_size &=
357 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1));
358 kwr->u.rxhdr.authmode_to_rxvalid &=
359 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1));
360 } else {
361 kwr->u.rxhdr.ivinsert_to_authinsrt =
362 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
363 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
364 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
365 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
366 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
367 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
368 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
369 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
370 }
371 }
372
373 /* Rx key */
374 static void
prepare_rxkey_wr(struct tls_keyctx * kwr,struct tls_key_context * kctx)375 prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
376 {
377 unsigned int ck_size = kctx->cipher_secret_size;
378 unsigned int mk_size = kctx->mac_secret_size;
379 int proto_ver = kctx->proto_ver;
380
381 kwr->u.rxhdr.flitcnt_hmacctrl =
382 ((kctx->rx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
383
384 kwr->u.rxhdr.protover_ciphmode =
385 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) |
386 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode);
387
388 kwr->u.rxhdr.authmode_to_rxvalid =
389 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) |
390 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) |
391 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
392 V_TLS_KEYCTX_TX_WR_RXVALID(1);
393
394 kwr->u.rxhdr.ivpresent_to_rxmk_size =
395 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
396 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) |
397 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) |
398 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size));
399
400 tls_rxkey_flit1(kwr, kctx);
401
402 /* No key reversal for GCM */
403 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) {
404 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key,
405 (kctx->cipher_secret_size << 3));
406 memcpy(kwr->keys.edkey + kctx->cipher_secret_size,
407 kctx->rx.key + kctx->cipher_secret_size,
408 (IPAD_SIZE + OPAD_SIZE));
409 } else {
410 memcpy(kwr->keys.edkey, kctx->rx.key,
411 (kctx->rx_key_info_size - SALT_SIZE));
412 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE);
413 }
414 }
415
416 /* Tx key */
417 static void
prepare_txkey_wr(struct tls_keyctx * kwr,struct tls_key_context * kctx)418 prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
419 {
420 unsigned int ck_size = kctx->cipher_secret_size;
421 unsigned int mk_size = kctx->mac_secret_size;
422
423 kwr->u.txhdr.ctxlen =
424 (kctx->tx_key_info_size >> 4);
425 kwr->u.txhdr.dualck_to_txvalid =
426 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) |
427 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
428 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) |
429 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) |
430 V_TLS_KEYCTX_TX_WR_TXVALID(1);
431
432 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE);
433 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
434 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE);
435 kwr->u.txhdr.dualck_to_txvalid &=
436 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1));
437 }
438 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid);
439 }
440
441 /* TLS Key memory management */
442 static int
get_new_keyid(struct toepcb * toep)443 get_new_keyid(struct toepcb *toep)
444 {
445 struct adapter *sc = td_adapter(toep->td);
446 vmem_addr_t addr;
447
448 if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
449 &addr) != 0)
450 return (-1);
451
452 return (addr);
453 }
454
455 static void
free_keyid(struct toepcb * toep,int keyid)456 free_keyid(struct toepcb *toep, int keyid)
457 {
458 struct adapter *sc = td_adapter(toep->td);
459
460 vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
461 }
462
463 static void
clear_tls_keyid(struct toepcb * toep)464 clear_tls_keyid(struct toepcb *toep)
465 {
466 struct tls_ofld_info *tls_ofld = &toep->tls;
467
468 if (tls_ofld->rx_key_addr >= 0) {
469 free_keyid(toep, tls_ofld->rx_key_addr);
470 tls_ofld->rx_key_addr = -1;
471 }
472 if (tls_ofld->tx_key_addr >= 0) {
473 free_keyid(toep, tls_ofld->tx_key_addr);
474 tls_ofld->tx_key_addr = -1;
475 }
476 }
477
478 static int
get_keyid(struct tls_ofld_info * tls_ofld,unsigned int ops)479 get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops)
480 {
481 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr :
482 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1));
483 }
484
485 static int
get_tp_plen_max(struct tls_ofld_info * tls_ofld)486 get_tp_plen_max(struct tls_ofld_info *tls_ofld)
487 {
488 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448;
489
490 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX);
491 }
492
493 /* Send request to get the key-id */
494 static int
tls_program_key_id(struct toepcb * toep,struct tls_key_context * k_ctx)495 tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx)
496 {
497 struct tls_ofld_info *tls_ofld = &toep->tls;
498 struct adapter *sc = td_adapter(toep->td);
499 struct ofld_tx_sdesc *txsd;
500 int kwrlen, kctxlen, keyid, len;
501 struct wrqe *wr;
502 struct tls_key_req *kwr;
503 struct tls_keyctx *kctx;
504
505 kwrlen = sizeof(*kwr);
506 kctxlen = roundup2(sizeof(*kctx), 32);
507 len = roundup2(kwrlen + kctxlen, 16);
508
509 if (toep->txsd_avail == 0)
510 return (EAGAIN);
511
512 /* Dont initialize key for re-neg */
513 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) {
514 if ((keyid = get_new_keyid(toep)) < 0) {
515 return (ENOSPC);
516 }
517 } else {
518 keyid = get_keyid(tls_ofld, k_ctx->l_p_key);
519 }
520
521 wr = alloc_wrqe(len, &toep->ofld_txq->wrq);
522 if (wr == NULL) {
523 free_keyid(toep, keyid);
524 return (ENOMEM);
525 }
526 kwr = wrtod(wr);
527 memset(kwr, 0, kwrlen);
528
529 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL |
530 F_FW_WR_ATOMIC);
531 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) |
532 V_FW_WR_FLOWID(toep->tid));
533 kwr->protocol = get_proto_ver(k_ctx->proto_ver);
534 kwr->mfs = htons(k_ctx->frag_size);
535 kwr->reneg_to_write_rx = k_ctx->l_p_key;
536
537 /* master command */
538 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
539 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
540 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
541 kwr->len16 = htobe32((toep->tid << 8) |
542 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
543 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
544
545 /* sub command */
546 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
547 kwr->sc_len = htobe32(kctxlen);
548
549 kctx = (struct tls_keyctx *)(kwr + 1);
550 memset(kctx, 0, kctxlen);
551
552 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
553 tls_ofld->tx_key_addr = keyid;
554 prepare_txkey_wr(kctx, k_ctx);
555 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
556 tls_ofld->rx_key_addr = keyid;
557 prepare_rxkey_wr(kctx, k_ctx);
558 }
559
560 txsd = &toep->txsd[toep->txsd_pidx];
561 txsd->tx_credits = DIV_ROUND_UP(len, 16);
562 txsd->plen = 0;
563 toep->tx_credits -= txsd->tx_credits;
564 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
565 toep->txsd_pidx = 0;
566 toep->txsd_avail--;
567
568 t4_wrq_tx(sc, wr);
569
570 return (0);
571 }
572
573 /* Store a key received from SSL in DDR. */
574 static int
program_key_context(struct tcpcb * tp,struct toepcb * toep,struct tls_key_context * uk_ctx)575 program_key_context(struct tcpcb *tp, struct toepcb *toep,
576 struct tls_key_context *uk_ctx)
577 {
578 struct adapter *sc = td_adapter(toep->td);
579 struct tls_ofld_info *tls_ofld = &toep->tls;
580 struct tls_key_context *k_ctx;
581 int error, key_offset;
582
583 if (tp->t_state != TCPS_ESTABLISHED) {
584 /*
585 * XXX: Matches Linux driver, but not sure this is a
586 * very appropriate error.
587 */
588 return (ENOENT);
589 }
590
591 /* Stop timer on handshake completion */
592 tls_stop_handshake_timer(toep);
593
594 toep->flags &= ~TPF_FORCE_CREDITS;
595
596 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid,
597 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" :
598 "KEY_WRITE_TX", uk_ctx->proto_ver);
599
600 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX &&
601 ulp_mode(toep) != ULP_MODE_TLS)
602 return (EOPNOTSUPP);
603
604 /* Don't copy the 'tx' and 'rx' fields. */
605 k_ctx = &tls_ofld->k_ctx;
606 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key,
607 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key));
608
609 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */
610 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) {
611 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
612 tls_ofld->rx_key_addr = -1;
613 t4_clear_rx_quiesce(toep);
614 } else {
615 tls_ofld->tx_key_addr = -1;
616 }
617 return (0);
618 }
619
620 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
621 k_ctx->iv_size = 4;
622 k_ctx->mac_first = 0;
623 k_ctx->hmac_ctrl = 0;
624 } else {
625 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
626 k_ctx->mac_first = 1;
627 }
628
629 tls_ofld->scmd0.seqno_numivs =
630 (V_SCMD_SEQ_NO_CTRL(3) |
631 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
632 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
633 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
634 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
635 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
636 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
637 V_SCMD_IV_SIZE(k_ctx->iv_size));
638
639 tls_ofld->scmd0.ivgen_hdrlen =
640 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
641 V_SCMD_KEY_CTX_INLINE(0) |
642 V_SCMD_TLS_FRAG_ENABLE(1));
643
644 tls_ofld->mac_length = k_ctx->mac_secret_size;
645
646 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
647 k_ctx->rx = uk_ctx->rx;
648 /* Dont initialize key for re-neg */
649 if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
650 tls_ofld->rx_key_addr = -1;
651 } else {
652 k_ctx->tx = uk_ctx->tx;
653 /* Dont initialize key for re-neg */
654 if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
655 tls_ofld->tx_key_addr = -1;
656 }
657
658 /* Flush pending data before new Tx key becomes active */
659 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
660 struct sockbuf *sb;
661
662 /* XXX: This might not drain everything. */
663 t4_push_frames(sc, toep, 0);
664 sb = &toep->inp->inp_socket->so_snd;
665 SOCKBUF_LOCK(sb);
666
667 /* XXX: This asserts that everything has been pushed. */
668 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL);
669 sb->sb_sndptr = NULL;
670 tls_ofld->sb_off = sbavail(sb);
671 SOCKBUF_UNLOCK(sb);
672 tls_ofld->tx_seq_no = 0;
673 }
674
675 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) ||
676 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) {
677
678 /*
679 * XXX: The userland library sets tx_key_info_size, not
680 * rx_key_info_size.
681 */
682 k_ctx->rx_key_info_size = k_ctx->tx_key_info_size;
683
684 error = tls_program_key_id(toep, k_ctx);
685 if (error) {
686 /* XXX: Only clear quiesce for KEY_WRITE_RX? */
687 t4_clear_rx_quiesce(toep);
688 return (error);
689 }
690 }
691
692 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
693 /*
694 * RX key tags are an index into the key portion of MA
695 * memory stored as an offset from the base address in
696 * units of 64 bytes.
697 */
698 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start;
699 t4_set_tls_keyid(toep, key_offset / 64);
700 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
701 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
702 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
703 V_TF_TLS_CONTROL(1) |
704 V_TF_TLS_ACTIVE(1) |
705 V_TF_TLS_ENABLE(1))));
706 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
707 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
708 V_TCB_TLS_SEQ(0));
709 t4_clear_rx_quiesce(toep);
710
711 toep->flags |= TPF_TLS_RECEIVE;
712 } else {
713 unsigned short pdus_per_ulp;
714
715 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE)
716 tls_ofld->tx_key_addr = 1;
717
718 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld);
719 tls_ofld->expn_per_ulp = tls_expansion_size(toep,
720 tls_ofld->fcplenmax, 1, &pdus_per_ulp);
721 tls_ofld->pdus_per_ulp = pdus_per_ulp;
722 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp *
723 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) +
724 tls_ofld->k_ctx.frag_size);
725 }
726
727 return (0);
728 }
729
730 /*
731 * In some cases a client connection can hang without sending the
732 * ServerHelloDone message from the NIC to the host. Send a dummy
733 * RX_DATA_ACK with RX_MODULATE to unstick the connection.
734 */
735 static void
tls_send_handshake_ack(void * arg)736 tls_send_handshake_ack(void *arg)
737 {
738 struct toepcb *toep = arg;
739 struct tls_ofld_info *tls_ofld = &toep->tls;
740 struct adapter *sc = td_adapter(toep->td);
741
742 /* Bail without rescheduling if the connection has closed. */
743 if ((toep->flags & (TPF_FIN_SENT | TPF_ABORT_SHUTDOWN)) != 0)
744 return;
745
746 /*
747 * If this connection has timed out without receiving more
748 * data, downgrade to plain TOE mode and don't re-arm the
749 * timer.
750 */
751 if (sc->tt.tls_rx_timeout != 0) {
752 struct inpcb *inp;
753 struct tcpcb *tp;
754
755 inp = toep->inp;
756 tp = intotcpcb(inp);
757 if ((ticks - tp->t_rcvtime) >= sc->tt.tls_rx_timeout) {
758 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
759 toep->tid);
760 tls_clr_ofld_mode(toep);
761 return;
762 }
763 }
764
765 /*
766 * XXX: Does not have the t4_get_tcb() checks to refine the
767 * workaround.
768 */
769 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);
770
771 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
772 send_rx_modulate(sc, toep);
773 }
774
775 static void
tls_start_handshake_timer(struct toepcb * toep)776 tls_start_handshake_timer(struct toepcb *toep)
777 {
778 struct tls_ofld_info *tls_ofld = &toep->tls;
779
780 INP_WLOCK_ASSERT(toep->inp);
781 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
782 tls_send_handshake_ack, toep);
783 }
784
785 void
tls_stop_handshake_timer(struct toepcb * toep)786 tls_stop_handshake_timer(struct toepcb *toep)
787 {
788 struct tls_ofld_info *tls_ofld = &toep->tls;
789
790 INP_WLOCK_ASSERT(toep->inp);
791 callout_stop(&tls_ofld->handshake_timer);
792 }
793
794 int
t4_ctloutput_tls(struct socket * so,struct sockopt * sopt)795 t4_ctloutput_tls(struct socket *so, struct sockopt *sopt)
796 {
797 struct tls_key_context uk_ctx;
798 struct inpcb *inp;
799 struct tcpcb *tp;
800 struct toepcb *toep;
801 int error, optval;
802
803 error = 0;
804 if (sopt->sopt_dir == SOPT_SET &&
805 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) {
806 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx),
807 sizeof(uk_ctx));
808 if (error)
809 return (error);
810 }
811
812 inp = sotoinpcb(so);
813 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
814 INP_WLOCK(inp);
815 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
816 INP_WUNLOCK(inp);
817 return (ECONNRESET);
818 }
819 tp = intotcpcb(inp);
820 toep = tp->t_toe;
821 switch (sopt->sopt_dir) {
822 case SOPT_SET:
823 switch (sopt->sopt_name) {
824 case TCP_TLSOM_SET_TLS_CONTEXT:
825 if (toep->tls.mode == TLS_MODE_KTLS)
826 error = EINVAL;
827 else {
828 error = program_key_context(tp, toep, &uk_ctx);
829 if (error == 0)
830 toep->tls.mode = TLS_MODE_TLSOM;
831 }
832 INP_WUNLOCK(inp);
833 break;
834 case TCP_TLSOM_CLR_TLS_TOM:
835 if (toep->tls.mode == TLS_MODE_KTLS)
836 error = EINVAL;
837 else if (ulp_mode(toep) == ULP_MODE_TLS) {
838 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM",
839 __func__, toep->tid);
840 tls_clr_ofld_mode(toep);
841 } else
842 error = EOPNOTSUPP;
843 INP_WUNLOCK(inp);
844 break;
845 case TCP_TLSOM_CLR_QUIES:
846 if (toep->tls.mode == TLS_MODE_KTLS)
847 error = EINVAL;
848 else if (ulp_mode(toep) == ULP_MODE_TLS) {
849 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES",
850 __func__, toep->tid);
851 tls_clr_quiesce(toep);
852 } else
853 error = EOPNOTSUPP;
854 INP_WUNLOCK(inp);
855 break;
856 default:
857 INP_WUNLOCK(inp);
858 error = EOPNOTSUPP;
859 break;
860 }
861 break;
862 case SOPT_GET:
863 switch (sopt->sopt_name) {
864 case TCP_TLSOM_GET_TLS_TOM:
865 /*
866 * TLS TX is permitted on any TOE socket, but
867 * TLS RX requires a TLS ULP mode.
868 */
869 optval = TLS_TOM_NONE;
870 if (can_tls_offload(td_adapter(toep->td)) &&
871 toep->tls.mode != TLS_MODE_KTLS) {
872 switch (ulp_mode(toep)) {
873 case ULP_MODE_NONE:
874 case ULP_MODE_TCPDDP:
875 optval = TLS_TOM_TXONLY;
876 break;
877 case ULP_MODE_TLS:
878 optval = TLS_TOM_BOTH;
879 break;
880 }
881 }
882 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d",
883 __func__, toep->tid, optval);
884 INP_WUNLOCK(inp);
885 error = sooptcopyout(sopt, &optval, sizeof(optval));
886 break;
887 default:
888 INP_WUNLOCK(inp);
889 error = EOPNOTSUPP;
890 break;
891 }
892 break;
893 }
894 return (error);
895 }
896
897 #ifdef KERN_TLS
898 static void
init_ktls_key_context(struct ktls_session * tls,struct tls_key_context * k_ctx,int direction)899 init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx,
900 int direction)
901 {
902 struct auth_hash *axf;
903 u_int key_info_size, mac_key_size;
904 char *hash, *key;
905
906 k_ctx->l_p_key = V_KEY_GET_LOC(direction == KTLS_TX ? KEY_WRITE_TX :
907 KEY_WRITE_RX);
908 k_ctx->proto_ver = tls->params.tls_vmajor << 8 | tls->params.tls_vminor;
909 k_ctx->cipher_secret_size = tls->params.cipher_key_len;
910 key_info_size = sizeof(struct tx_keyctx_hdr) +
911 k_ctx->cipher_secret_size;
912 if (direction == KTLS_TX)
913 key = k_ctx->tx.key;
914 else
915 key = k_ctx->rx.key;
916 memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len);
917 hash = key + tls->params.cipher_key_len;
918 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
919 k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH;
920 k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM;
921 k_ctx->iv_size = 4;
922 k_ctx->mac_first = 0;
923 k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP;
924 key_info_size += GMAC_BLOCK_LEN;
925 k_ctx->mac_secret_size = 0;
926 if (direction == KTLS_TX)
927 memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
928 else
929 memcpy(k_ctx->rx.salt, tls->params.iv, SALT_SIZE);
930 t4_init_gmac_hash(tls->params.cipher_key,
931 tls->params.cipher_key_len, hash);
932 } else {
933 switch (tls->params.auth_algorithm) {
934 case CRYPTO_SHA1_HMAC:
935 axf = &auth_hash_hmac_sha1;
936 mac_key_size = SHA1_HASH_LEN;
937 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA1;
938 break;
939 case CRYPTO_SHA2_256_HMAC:
940 axf = &auth_hash_hmac_sha2_256;
941 mac_key_size = SHA2_256_HASH_LEN;
942 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA256;
943 break;
944 case CRYPTO_SHA2_384_HMAC:
945 axf = &auth_hash_hmac_sha2_384;
946 mac_key_size = SHA2_512_HASH_LEN;
947 k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA512_384;
948 break;
949 default:
950 panic("bad auth mode");
951 }
952 k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_CBC;
953 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
954 k_ctx->mac_first = 1;
955 k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
956 key_info_size += roundup2(mac_key_size, 16) * 2;
957 k_ctx->mac_secret_size = mac_key_size;
958 t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
959 tls->params.auth_key_len, hash);
960 }
961
962 if (direction == KTLS_TX)
963 k_ctx->tx_key_info_size = key_info_size;
964 else
965 k_ctx->rx_key_info_size = key_info_size;
966 k_ctx->frag_size = tls->params.max_frame_len;
967 k_ctx->iv_ctrl = 1;
968 }
969
970 int
tls_alloc_ktls(struct toepcb * toep,struct ktls_session * tls,int direction)971 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls, int direction)
972 {
973 struct adapter *sc = td_adapter(toep->td);
974 struct tls_key_context *k_ctx;
975 int error, key_offset;
976
977 if (toep->tls.mode == TLS_MODE_TLSOM)
978 return (EINVAL);
979 if (!can_tls_offload(td_adapter(toep->td)))
980 return (EINVAL);
981 switch (ulp_mode(toep)) {
982 case ULP_MODE_TLS:
983 break;
984 case ULP_MODE_NONE:
985 case ULP_MODE_TCPDDP:
986 if (direction != KTLS_TX)
987 return (EINVAL);
988 break;
989 default:
990 return (EINVAL);
991 }
992
993 switch (tls->params.cipher_algorithm) {
994 case CRYPTO_AES_CBC:
995 /* XXX: Explicitly ignore any provided IV. */
996 switch (tls->params.cipher_key_len) {
997 case 128 / 8:
998 case 192 / 8:
999 case 256 / 8:
1000 break;
1001 default:
1002 error = EINVAL;
1003 goto clr_ofld;
1004 }
1005 switch (tls->params.auth_algorithm) {
1006 case CRYPTO_SHA1_HMAC:
1007 case CRYPTO_SHA2_256_HMAC:
1008 case CRYPTO_SHA2_384_HMAC:
1009 break;
1010 default:
1011 error = EPROTONOSUPPORT;
1012 goto clr_ofld;
1013 }
1014 break;
1015 case CRYPTO_AES_NIST_GCM_16:
1016 if (tls->params.iv_len != SALT_SIZE) {
1017 error = EINVAL;
1018 goto clr_ofld;
1019 }
1020 switch (tls->params.cipher_key_len) {
1021 case 128 / 8:
1022 case 192 / 8:
1023 case 256 / 8:
1024 break;
1025 default:
1026 error = EINVAL;
1027 goto clr_ofld;
1028 }
1029 break;
1030 default:
1031 error = EPROTONOSUPPORT;
1032 goto clr_ofld;
1033 }
1034
1035 /* Only TLS 1.1 and TLS 1.2 are currently supported. */
1036 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
1037 tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
1038 tls->params.tls_vminor > TLS_MINOR_VER_TWO) {
1039 error = EPROTONOSUPPORT;
1040 goto clr_ofld;
1041 }
1042
1043 /* Bail if we already have a key. */
1044 if (direction == KTLS_TX) {
1045 if (toep->tls.tx_key_addr != -1)
1046 return (EOPNOTSUPP);
1047 } else {
1048 if (toep->tls.rx_key_addr != -1)
1049 return (EOPNOTSUPP);
1050 }
1051
1052 /*
1053 * XXX: This assumes no key renegotation. If KTLS ever supports
1054 * that we will want to allocate TLS sessions dynamically rather
1055 * than as a static member of toep.
1056 */
1057 k_ctx = &toep->tls.k_ctx;
1058 init_ktls_key_context(tls, k_ctx, direction);
1059
1060 error = tls_program_key_id(toep, k_ctx);
1061 if (error) {
1062 if (direction == KTLS_RX)
1063 goto clr_ofld;
1064 return (error);
1065 }
1066
1067 if (direction == KTLS_TX) {
1068 toep->tls.scmd0.seqno_numivs =
1069 (V_SCMD_SEQ_NO_CTRL(3) |
1070 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
1071 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
1072 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
1073 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
1074 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
1075 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
1076 V_SCMD_IV_SIZE(k_ctx->iv_size));
1077
1078 toep->tls.scmd0.ivgen_hdrlen =
1079 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
1080 V_SCMD_KEY_CTX_INLINE(0) |
1081 V_SCMD_TLS_FRAG_ENABLE(1));
1082
1083 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
1084 toep->tls.iv_len = 8;
1085 else
1086 toep->tls.iv_len = AES_BLOCK_LEN;
1087
1088 toep->tls.mac_length = k_ctx->mac_secret_size;
1089
1090 toep->tls.fcplenmax = get_tp_plen_max(&toep->tls);
1091 toep->tls.expn_per_ulp = tls->params.tls_hlen +
1092 tls->params.tls_tlen;
1093 toep->tls.pdus_per_ulp = 1;
1094 toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
1095 toep->tls.k_ctx.frag_size;
1096 } else {
1097 /* Stop timer on handshake completion */
1098 tls_stop_handshake_timer(toep);
1099
1100 toep->flags &= ~TPF_FORCE_CREDITS;
1101 toep->flags |= TPF_TLS_RECEIVE;
1102
1103 /*
1104 * RX key tags are an index into the key portion of MA
1105 * memory stored as an offset from the base address in
1106 * units of 64 bytes.
1107 */
1108 key_offset = toep->tls.rx_key_addr - sc->vres.key.start;
1109 t4_set_tls_keyid(toep, key_offset / 64);
1110 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
1111 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
1112 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
1113 V_TF_TLS_CONTROL(1) |
1114 V_TF_TLS_ACTIVE(1) |
1115 V_TF_TLS_ENABLE(1))));
1116 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
1117 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
1118 V_TCB_TLS_SEQ(0));
1119 t4_clear_rx_quiesce(toep);
1120 }
1121
1122 toep->tls.mode = TLS_MODE_KTLS;
1123
1124 return (0);
1125
1126 clr_ofld:
1127 if (ulp_mode(toep) == ULP_MODE_TLS) {
1128 CTR2(KTR_CXGBE, "%s: tid %d clr_ofld_mode", __func__,
1129 toep->tid);
1130 tls_clr_ofld_mode(toep);
1131 }
1132 return (error);
1133 }
1134 #endif
1135
1136 void
tls_init_toep(struct toepcb * toep)1137 tls_init_toep(struct toepcb *toep)
1138 {
1139 struct tls_ofld_info *tls_ofld = &toep->tls;
1140
1141 tls_ofld->mode = TLS_MODE_OFF;
1142 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR;
1143 tls_ofld->rx_key_addr = -1;
1144 tls_ofld->tx_key_addr = -1;
1145 }
1146
1147 void
tls_establish(struct toepcb * toep)1148 tls_establish(struct toepcb *toep)
1149 {
1150
1151 /*
1152 * Enable PDU extraction.
1153 *
1154 * XXX: Supposedly this should be done by the firmware when
1155 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
1156 * in practice this seems to be required.
1157 */
1158 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
1159 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
1160 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
1161
1162 toep->flags |= TPF_FORCE_CREDITS | TPF_TLS_ESTABLISHED;
1163
1164 callout_init_rw(&toep->tls.handshake_timer, &toep->inp->inp_lock, 0);
1165 tls_start_handshake_timer(toep);
1166 }
1167
1168 void
tls_detach(struct toepcb * toep)1169 tls_detach(struct toepcb *toep)
1170 {
1171
1172 if (toep->flags & TPF_TLS_ESTABLISHED) {
1173 tls_stop_handshake_timer(toep);
1174 toep->flags &= ~TPF_TLS_ESTABLISHED;
1175 }
1176 }
1177
1178 void
tls_uninit_toep(struct toepcb * toep)1179 tls_uninit_toep(struct toepcb *toep)
1180 {
1181
1182 MPASS((toep->flags & TPF_TLS_ESTABLISHED) == 0);
1183 clear_tls_keyid(toep);
1184 }
1185
1186 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
1187 #define MIN_OFLD_TLSTX_CREDITS(toep) \
1188 (howmany(sizeof(struct fw_tlstx_data_wr) + \
1189 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \
1190 CIPHER_BLOCK_SIZE + 1, 16))
1191
1192 static inline u_int
max_imm_tls_space(int tx_credits)1193 max_imm_tls_space(int tx_credits)
1194 {
1195 const int n = 2; /* Use only up to 2 desc for imm. data WR */
1196 int space;
1197
1198 KASSERT(tx_credits >= 0 &&
1199 tx_credits <= MAX_OFLD_TX_CREDITS,
1200 ("%s: %d credits", __func__, tx_credits));
1201
1202 if (tx_credits >= (n * EQ_ESIZE) / 16)
1203 space = (n * EQ_ESIZE);
1204 else
1205 space = tx_credits * 16;
1206 return (space);
1207 }
1208
1209 static int
count_mbuf_segs(struct mbuf * m,int skip,int len,int * max_nsegs_1mbufp)1210 count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp)
1211 {
1212 int max_nsegs_1mbuf, n, nsegs;
1213
1214 while (skip >= m->m_len) {
1215 skip -= m->m_len;
1216 m = m->m_next;
1217 }
1218
1219 nsegs = 0;
1220 max_nsegs_1mbuf = 0;
1221 while (len > 0) {
1222 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip);
1223 if (n > max_nsegs_1mbuf)
1224 max_nsegs_1mbuf = n;
1225 nsegs += n;
1226 len -= m->m_len - skip;
1227 skip = 0;
1228 m = m->m_next;
1229 }
1230 *max_nsegs_1mbufp = max_nsegs_1mbuf;
1231 return (nsegs);
1232 }
1233
1234 static void
write_tlstx_wr(struct fw_tlstx_data_wr * txwr,struct toepcb * toep,unsigned int immdlen,unsigned int plen,unsigned int expn,unsigned int pdus,uint8_t credits,int shove,int imm_ivs)1235 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
1236 unsigned int immdlen, unsigned int plen, unsigned int expn,
1237 unsigned int pdus, uint8_t credits, int shove, int imm_ivs)
1238 {
1239 struct tls_ofld_info *tls_ofld = &toep->tls;
1240 unsigned int len = plen + expn;
1241
1242 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) |
1243 V_FW_TLSTX_DATA_WR_COMPL(1) |
1244 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen));
1245 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
1246 V_FW_TLSTX_DATA_WR_LEN16(credits));
1247 txwr->plen = htobe32(len);
1248 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) |
1249 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove));
1250 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) |
1251 V_FW_TLSTX_DATA_WR_EXP(expn) |
1252 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) |
1253 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) |
1254 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4));
1255 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size);
1256 txwr->adjustedplen_pkd = htobe16(
1257 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen));
1258 txwr->expinplenmax_pkd = htobe16(
1259 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp));
1260 txwr->pdusinplenmax_pkd =
1261 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp);
1262 }
1263
1264 static void
write_tlstx_cpl(struct cpl_tx_tls_sfo * cpl,struct toepcb * toep,struct tls_hdr * tls_hdr,unsigned int plen,unsigned int pdus)1265 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
1266 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus)
1267 {
1268 struct tls_ofld_info *tls_ofld = &toep->tls;
1269 int data_type, seglen;
1270
1271 if (plen < tls_ofld->k_ctx.frag_size)
1272 seglen = plen;
1273 else
1274 seglen = tls_ofld->k_ctx.frag_size;
1275 data_type = tls_content_type(tls_hdr->type);
1276 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) |
1277 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) |
1278 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen));
1279 cpl->pld_len = htobe32(plen);
1280 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT)
1281 cpl->type_protover = htobe32(
1282 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type));
1283 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs |
1284 V_SCMD_NUM_IVS(pdus));
1285 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen);
1286 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no);
1287 tls_ofld->tx_seq_no += pdus;
1288 }
1289
1290 /*
1291 * Similar to write_tx_sgl() except that it accepts an optional
1292 * trailer buffer for IVs.
1293 */
1294 static void
write_tlstx_sgl(void * dst,struct mbuf * start,int skip,int plen,void * iv_buffer,int iv_len,int nsegs,int n)1295 write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen,
1296 void *iv_buffer, int iv_len, int nsegs, int n)
1297 {
1298 struct mbuf *m;
1299 struct ulptx_sgl *usgl = dst;
1300 int i, j, rc;
1301 struct sglist sg;
1302 struct sglist_seg segs[n];
1303
1304 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
1305
1306 sglist_init(&sg, n, segs);
1307 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1308 V_ULPTX_NSGE(nsegs));
1309
1310 for (m = start; skip >= m->m_len; m = m->m_next)
1311 skip -= m->m_len;
1312
1313 i = -1;
1314 for (m = start; plen > 0; m = m->m_next) {
1315 rc = sglist_append(&sg, mtod(m, char *) + skip,
1316 m->m_len - skip);
1317 if (__predict_false(rc != 0))
1318 panic("%s: sglist_append %d", __func__, rc);
1319 plen -= m->m_len - skip;
1320 skip = 0;
1321
1322 for (j = 0; j < sg.sg_nseg; i++, j++) {
1323 if (i < 0) {
1324 usgl->len0 = htobe32(segs[j].ss_len);
1325 usgl->addr0 = htobe64(segs[j].ss_paddr);
1326 } else {
1327 usgl->sge[i / 2].len[i & 1] =
1328 htobe32(segs[j].ss_len);
1329 usgl->sge[i / 2].addr[i & 1] =
1330 htobe64(segs[j].ss_paddr);
1331 }
1332 #ifdef INVARIANTS
1333 nsegs--;
1334 #endif
1335 }
1336 sglist_reset(&sg);
1337 }
1338 if (iv_buffer != NULL) {
1339 rc = sglist_append(&sg, iv_buffer, iv_len);
1340 if (__predict_false(rc != 0))
1341 panic("%s: sglist_append %d", __func__, rc);
1342
1343 for (j = 0; j < sg.sg_nseg; i++, j++) {
1344 if (i < 0) {
1345 usgl->len0 = htobe32(segs[j].ss_len);
1346 usgl->addr0 = htobe64(segs[j].ss_paddr);
1347 } else {
1348 usgl->sge[i / 2].len[i & 1] =
1349 htobe32(segs[j].ss_len);
1350 usgl->sge[i / 2].addr[i & 1] =
1351 htobe64(segs[j].ss_paddr);
1352 }
1353 #ifdef INVARIANTS
1354 nsegs--;
1355 #endif
1356 }
1357 }
1358 if (i & 1)
1359 usgl->sge[i / 2].len[1] = htobe32(0);
1360 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p",
1361 __func__, nsegs, start, iv_buffer));
1362 }
1363
1364 /*
1365 * Similar to t4_push_frames() but handles TLS sockets when TLS offload
1366 * is enabled. Rather than transmitting bulk data, the socket buffer
1367 * contains TLS records. The work request requires a full TLS record,
1368 * so batch mbufs up until a full TLS record is seen. This requires
1369 * reading the TLS header out of the start of each record to determine
1370 * its length.
1371 */
1372 void
t4_push_tls_records(struct adapter * sc,struct toepcb * toep,int drop)1373 t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
1374 {
1375 struct tls_hdr thdr;
1376 struct mbuf *sndptr;
1377 struct fw_tlstx_data_wr *txwr;
1378 struct cpl_tx_tls_sfo *cpl;
1379 struct wrqe *wr;
1380 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len;
1381 u_int expn_size, iv_len, pdus, sndptroff;
1382 struct tls_ofld_info *tls_ofld = &toep->tls;
1383 struct inpcb *inp = toep->inp;
1384 struct tcpcb *tp = intotcpcb(inp);
1385 struct socket *so = inp->inp_socket;
1386 struct sockbuf *sb = &so->so_snd;
1387 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup;
1388 struct ofld_tx_sdesc *txsd;
1389 bool imm_ivs, imm_payload;
1390 void *iv_buffer, *iv_dst, *buf;
1391
1392 INP_WLOCK_ASSERT(inp);
1393 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1394 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
1395
1396 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
1397 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
1398 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
1399 KASSERT(tls_tx_key(toep),
1400 ("%s: TX key not set for toep %p", __func__, toep));
1401
1402 #ifdef VERBOSE_TRACES
1403 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
1404 __func__, toep->tid, toep->flags, tp->t_flags);
1405 #endif
1406 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
1407 return;
1408
1409 #ifdef RATELIMIT
1410 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
1411 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
1412 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
1413 }
1414 #endif
1415
1416 /*
1417 * This function doesn't resume by itself. Someone else must clear the
1418 * flag and call this function.
1419 */
1420 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
1421 KASSERT(drop == 0,
1422 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
1423 return;
1424 }
1425
1426 txsd = &toep->txsd[toep->txsd_pidx];
1427 for (;;) {
1428 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
1429 space = max_imm_tls_space(tx_credits);
1430 wr_len = sizeof(struct fw_tlstx_data_wr) +
1431 sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
1432 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) {
1433 #ifdef VERBOSE_TRACES
1434 CTR5(KTR_CXGBE,
1435 "%s: tid %d tx_credits %d min_wr %d space %d",
1436 __func__, toep->tid, tx_credits, wr_len +
1437 CIPHER_BLOCK_SIZE + 1, space);
1438 #endif
1439 return;
1440 }
1441
1442 SOCKBUF_LOCK(sb);
1443 sowwakeup = drop;
1444 if (drop) {
1445 sbdrop_locked(sb, drop);
1446 MPASS(tls_ofld->sb_off >= drop);
1447 tls_ofld->sb_off -= drop;
1448 drop = 0;
1449 }
1450
1451 /*
1452 * Send a FIN if requested, but only if there's no
1453 * more data to send.
1454 */
1455 if (sbavail(sb) == tls_ofld->sb_off &&
1456 toep->flags & TPF_SEND_FIN) {
1457 if (sowwakeup)
1458 sowwakeup_locked(so);
1459 else
1460 SOCKBUF_UNLOCK(sb);
1461 SOCKBUF_UNLOCK_ASSERT(sb);
1462 t4_close_conn(sc, toep);
1463 return;
1464 }
1465
1466 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) {
1467 /*
1468 * A full TLS header is not yet queued, stop
1469 * for now until more data is added to the
1470 * socket buffer. However, if the connection
1471 * has been closed, we will never get the rest
1472 * of the header so just discard the partial
1473 * header and close the connection.
1474 */
1475 #ifdef VERBOSE_TRACES
1476 CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s",
1477 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
1478 toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN");
1479 #endif
1480 if (sowwakeup)
1481 sowwakeup_locked(so);
1482 else
1483 SOCKBUF_UNLOCK(sb);
1484 SOCKBUF_UNLOCK_ASSERT(sb);
1485 if (toep->flags & TPF_SEND_FIN)
1486 t4_close_conn(sc, toep);
1487 return;
1488 }
1489
1490 /* Read the header of the next TLS record. */
1491 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff);
1492 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr);
1493 tls_size = htons(thdr.length);
1494 plen = TLS_HEADER_LENGTH + tls_size;
1495 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size);
1496 iv_len = pdus * CIPHER_BLOCK_SIZE;
1497
1498 if (sbavail(sb) < tls_ofld->sb_off + plen) {
1499 /*
1500 * The full TLS record is not yet queued, stop
1501 * for now until more data is added to the
1502 * socket buffer. However, if the connection
1503 * has been closed, we will never get the rest
1504 * of the record so just discard the partial
1505 * record and close the connection.
1506 */
1507 #ifdef VERBOSE_TRACES
1508 CTR6(KTR_CXGBE,
1509 "%s: tid %d sbavail %d sb_off %d plen %d%s",
1510 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
1511 plen, toep->flags & TPF_SEND_FIN ? "" :
1512 " SEND_FIN");
1513 #endif
1514 if (sowwakeup)
1515 sowwakeup_locked(so);
1516 else
1517 SOCKBUF_UNLOCK(sb);
1518 SOCKBUF_UNLOCK_ASSERT(sb);
1519 if (toep->flags & TPF_SEND_FIN)
1520 t4_close_conn(sc, toep);
1521 return;
1522 }
1523
1524 /* Shove if there is no additional data pending. */
1525 shove = (sbavail(sb) == tls_ofld->sb_off + plen) &&
1526 !(tp->t_flags & TF_MORETOCOME);
1527
1528 if (sb->sb_flags & SB_AUTOSIZE &&
1529 V_tcp_do_autosndbuf &&
1530 sb->sb_hiwat < V_tcp_autosndbuf_max &&
1531 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
1532 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
1533 V_tcp_autosndbuf_max);
1534
1535 if (!sbreserve_locked(sb, newsize, so, NULL))
1536 sb->sb_flags &= ~SB_AUTOSIZE;
1537 else
1538 sowwakeup = 1; /* room available */
1539 }
1540 if (sowwakeup)
1541 sowwakeup_locked(so);
1542 else
1543 SOCKBUF_UNLOCK(sb);
1544 SOCKBUF_UNLOCK_ASSERT(sb);
1545
1546 if (__predict_false(toep->flags & TPF_FIN_SENT))
1547 panic("%s: excess tx.", __func__);
1548
1549 /* Determine whether to use immediate vs SGL. */
1550 imm_payload = false;
1551 imm_ivs = false;
1552 if (wr_len + iv_len <= space) {
1553 imm_ivs = true;
1554 wr_len += iv_len;
1555 if (wr_len + tls_size <= space) {
1556 wr_len += tls_size;
1557 imm_payload = true;
1558 }
1559 }
1560
1561 /* Allocate space for IVs if needed. */
1562 if (!imm_ivs) {
1563 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT);
1564 if (iv_buffer == NULL) {
1565 /*
1566 * XXX: How to restart this?
1567 */
1568 if (sowwakeup)
1569 sowwakeup_locked(so);
1570 else
1571 SOCKBUF_UNLOCK(sb);
1572 SOCKBUF_UNLOCK_ASSERT(sb);
1573 CTR3(KTR_CXGBE,
1574 "%s: tid %d failed to alloc IV space len %d",
1575 __func__, toep->tid, iv_len);
1576 return;
1577 }
1578 } else
1579 iv_buffer = NULL;
1580
1581 /* Determine size of SGL. */
1582 nsegs = 0;
1583 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
1584 if (!imm_payload) {
1585 nsegs = count_mbuf_segs(sndptr, sndptroff +
1586 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf);
1587 if (!imm_ivs) {
1588 int n = sglist_count(iv_buffer, iv_len);
1589 nsegs += n;
1590 if (n > max_nsegs_1mbuf)
1591 max_nsegs_1mbuf = n;
1592 }
1593
1594 /* Account for SGL in work request length. */
1595 wr_len += sizeof(struct ulptx_sgl) +
1596 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1597 }
1598
1599 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
1600 if (wr == NULL) {
1601 /* XXX: how will we recover from this? */
1602 toep->flags |= TPF_TX_SUSPENDED;
1603 return;
1604 }
1605
1606 #ifdef VERBOSE_TRACES
1607 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d",
1608 __func__, toep->tid, thdr.type, tls_size, pdus);
1609 #endif
1610 txwr = wrtod(wr);
1611 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
1612 memset(txwr, 0, roundup2(wr_len, 16));
1613 credits = howmany(wr_len, 16);
1614 expn_size = tls_expansion_size(toep, tls_size, 0, NULL);
1615 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0,
1616 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0);
1617 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus);
1618 tls_copy_tx_key(toep, cpl + 1);
1619
1620 /* Generate random IVs */
1621 buf = (char *)(cpl + 1) + key_size(toep);
1622 if (imm_ivs) {
1623 MPASS(iv_buffer == NULL);
1624 iv_dst = buf;
1625 buf = (char *)iv_dst + iv_len;
1626 } else
1627 iv_dst = iv_buffer;
1628 arc4rand(iv_dst, iv_len, 0);
1629
1630 if (imm_payload) {
1631 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH,
1632 tls_size, buf);
1633 } else {
1634 write_tlstx_sgl(buf, sndptr,
1635 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer,
1636 iv_len, nsegs, max_nsegs_1mbuf);
1637 }
1638
1639 KASSERT(toep->tx_credits >= credits,
1640 ("%s: not enough credits", __func__));
1641
1642 toep->tx_credits -= credits;
1643
1644 tp->snd_nxt += plen;
1645 tp->snd_max += plen;
1646
1647 SOCKBUF_LOCK(sb);
1648 sbsndptr_adv(sb, sb->sb_sndptr, plen);
1649 tls_ofld->sb_off += plen;
1650 SOCKBUF_UNLOCK(sb);
1651
1652 toep->flags |= TPF_TX_DATA_SENT;
1653 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
1654 toep->flags |= TPF_TX_SUSPENDED;
1655
1656 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1657 txsd->plen = plen;
1658 txsd->tx_credits = credits;
1659 txsd->iv_buffer = iv_buffer;
1660 txsd++;
1661 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1662 toep->txsd_pidx = 0;
1663 txsd = &toep->txsd[0];
1664 }
1665 toep->txsd_avail--;
1666
1667 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
1668 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, plen);
1669
1670 t4_l2t_send(sc, wr, toep->l2te);
1671 }
1672 }
1673
1674 #ifdef KERN_TLS
1675 static int
count_ext_pgs_segs(struct mbuf * m)1676 count_ext_pgs_segs(struct mbuf *m)
1677 {
1678 vm_paddr_t nextpa;
1679 u_int i, nsegs;
1680
1681 MPASS(m->m_epg_npgs > 0);
1682 nsegs = 1;
1683 nextpa = m->m_epg_pa[0] + PAGE_SIZE;
1684 for (i = 1; i < m->m_epg_npgs; i++) {
1685 if (nextpa != m->m_epg_pa[i])
1686 nsegs++;
1687 nextpa = m->m_epg_pa[i] + PAGE_SIZE;
1688 }
1689 return (nsegs);
1690 }
1691
1692 static void
write_ktlstx_sgl(void * dst,struct mbuf * m,int nsegs)1693 write_ktlstx_sgl(void *dst, struct mbuf *m, int nsegs)
1694 {
1695 struct ulptx_sgl *usgl = dst;
1696 vm_paddr_t pa;
1697 uint32_t len;
1698 int i, j;
1699
1700 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
1701
1702 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1703 V_ULPTX_NSGE(nsegs));
1704
1705 /* Figure out the first S/G length. */
1706 pa = m->m_epg_pa[0] + m->m_epg_1st_off;
1707 usgl->addr0 = htobe64(pa);
1708 len = m_epg_pagelen(m, 0, m->m_epg_1st_off);
1709 pa += len;
1710 for (i = 1; i < m->m_epg_npgs; i++) {
1711 if (m->m_epg_pa[i] != pa)
1712 break;
1713 len += m_epg_pagelen(m, i, 0);
1714 pa += m_epg_pagelen(m, i, 0);
1715 }
1716 usgl->len0 = htobe32(len);
1717 #ifdef INVARIANTS
1718 nsegs--;
1719 #endif
1720
1721 j = -1;
1722 for (; i < m->m_epg_npgs; i++) {
1723 if (j == -1 || m->m_epg_pa[i] != pa) {
1724 if (j >= 0)
1725 usgl->sge[j / 2].len[j & 1] = htobe32(len);
1726 j++;
1727 #ifdef INVARIANTS
1728 nsegs--;
1729 #endif
1730 pa = m->m_epg_pa[i];
1731 usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
1732 len = m_epg_pagelen(m, i, 0);
1733 pa += len;
1734 } else {
1735 len += m_epg_pagelen(m, i, 0);
1736 pa += m_epg_pagelen(m, i, 0);
1737 }
1738 }
1739 if (j >= 0) {
1740 usgl->sge[j / 2].len[j & 1] = htobe32(len);
1741
1742 if ((j & 1) == 0)
1743 usgl->sge[j / 2].len[1] = htobe32(0);
1744 }
1745 KASSERT(nsegs == 0, ("%s: nsegs %d, m %p", __func__, nsegs, m));
1746 }
1747
1748 /*
1749 * Similar to t4_push_frames() but handles sockets that contain TLS
1750 * record mbufs. Unlike TLSOM, each mbuf is a complete TLS record and
1751 * corresponds to a single work request.
1752 */
1753 void
t4_push_ktls(struct adapter * sc,struct toepcb * toep,int drop)1754 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
1755 {
1756 struct tls_hdr *thdr;
1757 struct fw_tlstx_data_wr *txwr;
1758 struct cpl_tx_tls_sfo *cpl;
1759 struct wrqe *wr;
1760 struct mbuf *m;
1761 u_int nsegs, credits, wr_len;
1762 u_int expn_size;
1763 struct inpcb *inp = toep->inp;
1764 struct tcpcb *tp = intotcpcb(inp);
1765 struct socket *so = inp->inp_socket;
1766 struct sockbuf *sb = &so->so_snd;
1767 int tls_size, tx_credits, shove, sowwakeup;
1768 struct ofld_tx_sdesc *txsd;
1769 char *buf;
1770
1771 INP_WLOCK_ASSERT(inp);
1772 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1773 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
1774
1775 KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
1776 ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
1777 ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
1778 KASSERT(tls_tx_key(toep),
1779 ("%s: TX key not set for toep %p", __func__, toep));
1780
1781 #ifdef VERBOSE_TRACES
1782 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
1783 __func__, toep->tid, toep->flags, tp->t_flags);
1784 #endif
1785 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
1786 return;
1787
1788 #ifdef RATELIMIT
1789 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
1790 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
1791 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
1792 }
1793 #endif
1794
1795 /*
1796 * This function doesn't resume by itself. Someone else must clear the
1797 * flag and call this function.
1798 */
1799 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
1800 KASSERT(drop == 0,
1801 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
1802 return;
1803 }
1804
1805 txsd = &toep->txsd[toep->txsd_pidx];
1806 for (;;) {
1807 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
1808
1809 SOCKBUF_LOCK(sb);
1810 sowwakeup = drop;
1811 if (drop) {
1812 sbdrop_locked(sb, drop);
1813 drop = 0;
1814 }
1815
1816 m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb;
1817
1818 /*
1819 * Send a FIN if requested, but only if there's no
1820 * more data to send.
1821 */
1822 if (m == NULL && toep->flags & TPF_SEND_FIN) {
1823 if (sowwakeup)
1824 sowwakeup_locked(so);
1825 else
1826 SOCKBUF_UNLOCK(sb);
1827 SOCKBUF_UNLOCK_ASSERT(sb);
1828 t4_close_conn(sc, toep);
1829 return;
1830 }
1831
1832 /*
1833 * If there is no ready data to send, wait until more
1834 * data arrives.
1835 */
1836 if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) {
1837 if (sowwakeup)
1838 sowwakeup_locked(so);
1839 else
1840 SOCKBUF_UNLOCK(sb);
1841 SOCKBUF_UNLOCK_ASSERT(sb);
1842 #ifdef VERBOSE_TRACES
1843 CTR2(KTR_CXGBE, "%s: tid %d no ready data to send",
1844 __func__, toep->tid);
1845 #endif
1846 return;
1847 }
1848
1849 KASSERT(m->m_flags & M_EXTPG, ("%s: mbuf %p is not NOMAP",
1850 __func__, m));
1851 KASSERT(m->m_epg_tls != NULL,
1852 ("%s: mbuf %p doesn't have TLS session", __func__, m));
1853
1854 /* Calculate WR length. */
1855 wr_len = sizeof(struct fw_tlstx_data_wr) +
1856 sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
1857
1858 /* Explicit IVs for AES-CBC and AES-GCM are <= 16. */
1859 MPASS(toep->tls.iv_len <= AES_BLOCK_LEN);
1860 wr_len += AES_BLOCK_LEN;
1861
1862 /* Account for SGL in work request length. */
1863 nsegs = count_ext_pgs_segs(m);
1864 wr_len += sizeof(struct ulptx_sgl) +
1865 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1866
1867 /* Not enough credits for this work request. */
1868 if (howmany(wr_len, 16) > tx_credits) {
1869 if (sowwakeup)
1870 sowwakeup_locked(so);
1871 else
1872 SOCKBUF_UNLOCK(sb);
1873 SOCKBUF_UNLOCK_ASSERT(sb);
1874 #ifdef VERBOSE_TRACES
1875 CTR5(KTR_CXGBE,
1876 "%s: tid %d mbuf %p requires %d credits, but only %d available",
1877 __func__, toep->tid, m, howmany(wr_len, 16),
1878 tx_credits);
1879 #endif
1880 toep->flags |= TPF_TX_SUSPENDED;
1881 return;
1882 }
1883
1884 /* Shove if there is no additional data pending. */
1885 shove = ((m->m_next == NULL ||
1886 (m->m_next->m_flags & M_NOTAVAIL) != 0)) &&
1887 (tp->t_flags & TF_MORETOCOME) == 0;
1888
1889 if (sb->sb_flags & SB_AUTOSIZE &&
1890 V_tcp_do_autosndbuf &&
1891 sb->sb_hiwat < V_tcp_autosndbuf_max &&
1892 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
1893 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
1894 V_tcp_autosndbuf_max);
1895
1896 if (!sbreserve_locked(sb, newsize, so, NULL))
1897 sb->sb_flags &= ~SB_AUTOSIZE;
1898 else
1899 sowwakeup = 1; /* room available */
1900 }
1901 if (sowwakeup)
1902 sowwakeup_locked(so);
1903 else
1904 SOCKBUF_UNLOCK(sb);
1905 SOCKBUF_UNLOCK_ASSERT(sb);
1906
1907 if (__predict_false(toep->flags & TPF_FIN_SENT))
1908 panic("%s: excess tx.", __func__);
1909
1910 wr = alloc_wrqe(roundup2(wr_len, 16), &toep->ofld_txq->wrq);
1911 if (wr == NULL) {
1912 /* XXX: how will we recover from this? */
1913 toep->flags |= TPF_TX_SUSPENDED;
1914 return;
1915 }
1916
1917 thdr = (struct tls_hdr *)&m->m_epg_hdr;
1918 #ifdef VERBOSE_TRACES
1919 CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x",
1920 __func__, toep->tid, m->m_epg_seqno, thdr->type,
1921 m->m_len);
1922 #endif
1923 txwr = wrtod(wr);
1924 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
1925 memset(txwr, 0, roundup2(wr_len, 16));
1926 credits = howmany(wr_len, 16);
1927 expn_size = m->m_epg_hdrlen +
1928 m->m_epg_trllen;
1929 tls_size = m->m_len - expn_size;
1930 write_tlstx_wr(txwr, toep, 0,
1931 tls_size, expn_size, 1, credits, shove, 1);
1932 toep->tls.tx_seq_no = m->m_epg_seqno;
1933 write_tlstx_cpl(cpl, toep, thdr, tls_size, 1);
1934 tls_copy_tx_key(toep, cpl + 1);
1935
1936 /* Copy IV. */
1937 buf = (char *)(cpl + 1) + key_size(toep);
1938 memcpy(buf, thdr + 1, toep->tls.iv_len);
1939 buf += AES_BLOCK_LEN;
1940
1941 write_ktlstx_sgl(buf, m, nsegs);
1942
1943 KASSERT(toep->tx_credits >= credits,
1944 ("%s: not enough credits", __func__));
1945
1946 toep->tx_credits -= credits;
1947
1948 tp->snd_nxt += m->m_len;
1949 tp->snd_max += m->m_len;
1950
1951 SOCKBUF_LOCK(sb);
1952 sb->sb_sndptr = m;
1953 SOCKBUF_UNLOCK(sb);
1954
1955 toep->flags |= TPF_TX_DATA_SENT;
1956 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
1957 toep->flags |= TPF_TX_SUSPENDED;
1958
1959 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1960 txsd->plen = m->m_len;
1961 txsd->tx_credits = credits;
1962 txsd++;
1963 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1964 toep->txsd_pidx = 0;
1965 txsd = &toep->txsd[0];
1966 }
1967 toep->txsd_avail--;
1968
1969 counter_u64_add(toep->ofld_txq->tx_toe_tls_records, 1);
1970 counter_u64_add(toep->ofld_txq->tx_toe_tls_octets, m->m_len);
1971
1972 t4_l2t_send(sc, wr, toep->l2te);
1973 }
1974 }
1975 #endif
1976
1977 /*
1978 * For TLS data we place received mbufs received via CPL_TLS_DATA into
1979 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is
1980 * received, the completed PDUs are placed into the socket receive
1981 * buffer.
1982 *
1983 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs.
1984 */
1985 static int
do_tls_data(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)1986 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1987 {
1988 struct adapter *sc = iq->adapter;
1989 const struct cpl_tls_data *cpl = mtod(m, const void *);
1990 unsigned int tid = GET_TID(cpl);
1991 struct toepcb *toep = lookup_tid(sc, tid);
1992 struct inpcb *inp = toep->inp;
1993 struct tcpcb *tp;
1994 int len;
1995
1996 /* XXX: Should this match do_rx_data instead? */
1997 KASSERT(!(toep->flags & TPF_SYNQE),
1998 ("%s: toep %p claims to be a synq entry", __func__, toep));
1999
2000 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
2001
2002 /* strip off CPL header */
2003 m_adj(m, sizeof(*cpl));
2004 len = m->m_pkthdr.len;
2005
2006 toep->ofld_rxq->rx_toe_tls_octets += len;
2007
2008 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
2009 ("%s: payload length mismatch", __func__));
2010
2011 INP_WLOCK(inp);
2012 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
2013 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
2014 __func__, tid, len, inp->inp_flags);
2015 INP_WUNLOCK(inp);
2016 m_freem(m);
2017 return (0);
2018 }
2019
2020 /* Save TCP sequence number. */
2021 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq);
2022
2023 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
2024 #ifdef INVARIANTS
2025 panic("Failed to queue TLS data packet");
2026 #else
2027 printf("%s: Failed to queue TLS data packet\n", __func__);
2028 INP_WUNLOCK(inp);
2029 m_freem(m);
2030 return (0);
2031 #endif
2032 }
2033
2034 tp = intotcpcb(inp);
2035 tp->t_rcvtime = ticks;
2036
2037 #ifdef VERBOSE_TRACES
2038 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
2039 be32toh(cpl->seq));
2040 #endif
2041
2042 INP_WUNLOCK(inp);
2043 return (0);
2044 }
2045
2046 static int
do_rx_tls_cmp(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)2047 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
2048 {
2049 struct adapter *sc = iq->adapter;
2050 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *);
2051 struct tlsrx_hdr_pkt *tls_hdr_pkt;
2052 unsigned int tid = GET_TID(cpl);
2053 struct toepcb *toep = lookup_tid(sc, tid);
2054 struct inpcb *inp = toep->inp;
2055 struct tcpcb *tp;
2056 struct socket *so;
2057 struct sockbuf *sb;
2058 struct mbuf *tls_data;
2059 #ifdef KERN_TLS
2060 struct tls_get_record *tgr;
2061 struct mbuf *control;
2062 #endif
2063 int pdu_length, rx_credits;
2064 #if defined(KTR) || defined(INVARIANTS)
2065 int len;
2066 #endif
2067
2068 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
2069 KASSERT(!(toep->flags & TPF_SYNQE),
2070 ("%s: toep %p claims to be a synq entry", __func__, toep));
2071
2072 /* strip off CPL header */
2073 m_adj(m, sizeof(*cpl));
2074 #if defined(KTR) || defined(INVARIANTS)
2075 len = m->m_pkthdr.len;
2076 #endif
2077
2078 toep->ofld_rxq->rx_toe_tls_records++;
2079
2080 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
2081 ("%s: payload length mismatch", __func__));
2082
2083 INP_WLOCK(inp);
2084 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
2085 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
2086 __func__, tid, len, inp->inp_flags);
2087 INP_WUNLOCK(inp);
2088 m_freem(m);
2089 return (0);
2090 }
2091
2092 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
2093
2094 so = inp_inpcbtosocket(inp);
2095 tp = intotcpcb(inp);
2096
2097 #ifdef VERBOSE_TRACES
2098 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
2099 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt);
2100 #endif
2101
2102 tp->rcv_nxt += pdu_length;
2103 KASSERT(tp->rcv_wnd >= pdu_length,
2104 ("%s: negative window size", __func__));
2105 tp->rcv_wnd -= pdu_length;
2106
2107 /* XXX: Not sure what to do about urgent data. */
2108
2109 /*
2110 * The payload of this CPL is the TLS header followed by
2111 * additional fields.
2112 */
2113 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt),
2114 ("%s: payload too small", __func__));
2115 tls_hdr_pkt = mtod(m, void *);
2116
2117 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
2118 if (tls_data != NULL) {
2119 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
2120 ("%s: sequence mismatch", __func__));
2121 }
2122
2123 #ifdef KERN_TLS
2124 if (toep->tls.mode == TLS_MODE_KTLS) {
2125 /* Report decryption errors as EBADMSG. */
2126 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) !=
2127 0) {
2128 m_freem(m);
2129 m_freem(tls_data);
2130
2131 CURVNET_SET(toep->vnet);
2132 so->so_error = EBADMSG;
2133 sorwakeup(so);
2134
2135 INP_WUNLOCK(inp);
2136 CURVNET_RESTORE();
2137
2138 return (0);
2139 }
2140
2141 /* Allocate the control message mbuf. */
2142 control = sbcreatecontrol(NULL, sizeof(*tgr), TLS_GET_RECORD,
2143 IPPROTO_TCP);
2144 if (control == NULL) {
2145 m_freem(m);
2146 m_freem(tls_data);
2147
2148 CURVNET_SET(toep->vnet);
2149 so->so_error = ENOBUFS;
2150 sorwakeup(so);
2151
2152 INP_WUNLOCK(inp);
2153 CURVNET_RESTORE();
2154
2155 return (0);
2156 }
2157
2158 tgr = (struct tls_get_record *)
2159 CMSG_DATA(mtod(control, struct cmsghdr *));
2160 tgr->tls_type = tls_hdr_pkt->type;
2161 tgr->tls_vmajor = be16toh(tls_hdr_pkt->version) >> 8;
2162 tgr->tls_vminor = be16toh(tls_hdr_pkt->version) & 0xff;
2163
2164 m_freem(m);
2165
2166 if (tls_data != NULL) {
2167 m_last(tls_data)->m_flags |= M_EOR;
2168 tgr->tls_length = htobe16(tls_data->m_pkthdr.len);
2169 } else
2170 tgr->tls_length = 0;
2171 m = tls_data;
2172 } else
2173 #endif
2174 {
2175 /*
2176 * Only the TLS header is sent to OpenSSL, so report
2177 * errors by altering the record type.
2178 */
2179 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) !=
2180 0)
2181 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
2182
2183 /* Trim this CPL's mbuf to only include the TLS header. */
2184 KASSERT(m->m_len == len && m->m_next == NULL,
2185 ("%s: CPL spans multiple mbufs", __func__));
2186 m->m_len = TLS_HEADER_LENGTH;
2187 m->m_pkthdr.len = TLS_HEADER_LENGTH;
2188
2189 if (tls_data != NULL) {
2190 /*
2191 * Update the TLS header length to be the length of
2192 * the payload data.
2193 */
2194 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
2195
2196 m->m_next = tls_data;
2197 m->m_pkthdr.len += tls_data->m_len;
2198 }
2199
2200 #ifdef KERN_TLS
2201 control = NULL;
2202 #endif
2203 }
2204
2205 sb = &so->so_rcv;
2206 SOCKBUF_LOCK(sb);
2207
2208 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
2209 struct epoch_tracker et;
2210
2211 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
2212 __func__, tid, pdu_length);
2213 m_freem(m);
2214 #ifdef KERN_TLS
2215 m_freem(control);
2216 #endif
2217 SOCKBUF_UNLOCK(sb);
2218 INP_WUNLOCK(inp);
2219
2220 CURVNET_SET(toep->vnet);
2221 NET_EPOCH_ENTER(et);
2222 INP_WLOCK(inp);
2223 tp = tcp_drop(tp, ECONNRESET);
2224 if (tp)
2225 INP_WUNLOCK(inp);
2226 NET_EPOCH_EXIT(et);
2227 CURVNET_RESTORE();
2228
2229 return (0);
2230 }
2231
2232 /*
2233 * Not all of the bytes on the wire are included in the socket buffer
2234 * (e.g. the MAC of the TLS record). However, those bytes are included
2235 * in the TCP sequence space.
2236 */
2237
2238 /* receive buffer autosize */
2239 MPASS(toep->vnet == so->so_vnet);
2240 CURVNET_SET(toep->vnet);
2241 if (sb->sb_flags & SB_AUTOSIZE &&
2242 V_tcp_do_autorcvbuf &&
2243 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
2244 m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) {
2245 unsigned int hiwat = sb->sb_hiwat;
2246 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
2247 V_tcp_autorcvbuf_max);
2248
2249 if (!sbreserve_locked(sb, newsize, so, NULL))
2250 sb->sb_flags &= ~SB_AUTOSIZE;
2251 }
2252
2253 #ifdef KERN_TLS
2254 if (control != NULL)
2255 sbappendcontrol_locked(sb, m, control, 0);
2256 else
2257 #endif
2258 sbappendstream_locked(sb, m, 0);
2259 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
2260 #ifdef VERBOSE_TRACES
2261 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u",
2262 __func__, tid, rx_credits, tp->rcv_wnd);
2263 #endif
2264 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
2265 rx_credits = send_rx_credits(sc, toep, rx_credits);
2266 tp->rcv_wnd += rx_credits;
2267 tp->rcv_adv += rx_credits;
2268 }
2269
2270 sorwakeup_locked(so);
2271 SOCKBUF_UNLOCK_ASSERT(sb);
2272
2273 INP_WUNLOCK(inp);
2274 CURVNET_RESTORE();
2275 return (0);
2276 }
2277
2278 void
do_rx_data_tls(const struct cpl_rx_data * cpl,struct toepcb * toep,struct mbuf * m)2279 do_rx_data_tls(const struct cpl_rx_data *cpl, struct toepcb *toep,
2280 struct mbuf *m)
2281 {
2282 struct inpcb *inp = toep->inp;
2283 struct tls_ofld_info *tls_ofld = &toep->tls;
2284 struct tls_hdr *hdr;
2285 struct tcpcb *tp;
2286 struct socket *so;
2287 struct sockbuf *sb;
2288 int error, len, rx_credits;
2289
2290 len = m->m_pkthdr.len;
2291
2292 INP_WLOCK_ASSERT(inp);
2293
2294 so = inp_inpcbtosocket(inp);
2295 tp = intotcpcb(inp);
2296 sb = &so->so_rcv;
2297 SOCKBUF_LOCK(sb);
2298 CURVNET_SET(toep->vnet);
2299
2300 tp->rcv_nxt += len;
2301 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
2302 tp->rcv_wnd -= len;
2303
2304 /* Do we have a full TLS header? */
2305 if (len < sizeof(*hdr)) {
2306 CTR3(KTR_CXGBE, "%s: tid %u len %d: too short for a TLS header",
2307 __func__, toep->tid, len);
2308 so->so_error = EMSGSIZE;
2309 goto out;
2310 }
2311 hdr = mtod(m, struct tls_hdr *);
2312
2313 /* Is the header valid? */
2314 if (be16toh(hdr->version) != tls_ofld->k_ctx.proto_ver) {
2315 CTR3(KTR_CXGBE, "%s: tid %u invalid version %04x",
2316 __func__, toep->tid, be16toh(hdr->version));
2317 error = EINVAL;
2318 goto report_error;
2319 }
2320 if (be16toh(hdr->length) < sizeof(*hdr)) {
2321 CTR3(KTR_CXGBE, "%s: tid %u invalid length %u",
2322 __func__, toep->tid, be16toh(hdr->length));
2323 error = EBADMSG;
2324 goto report_error;
2325 }
2326
2327 /* Did we get a truncated record? */
2328 if (len < be16toh(hdr->length)) {
2329 CTR4(KTR_CXGBE, "%s: tid %u truncated TLS record (%d vs %u)",
2330 __func__, toep->tid, len, be16toh(hdr->length));
2331
2332 error = EMSGSIZE;
2333 goto report_error;
2334 }
2335
2336 /* Is the header type unknown? */
2337 switch (hdr->type) {
2338 case CONTENT_TYPE_CCS:
2339 case CONTENT_TYPE_ALERT:
2340 case CONTENT_TYPE_APP_DATA:
2341 case CONTENT_TYPE_HANDSHAKE:
2342 break;
2343 default:
2344 CTR3(KTR_CXGBE, "%s: tid %u invalid TLS record type %u",
2345 __func__, toep->tid, hdr->type);
2346 error = EBADMSG;
2347 goto report_error;
2348 }
2349
2350 /*
2351 * Just punt. Although this could fall back to software
2352 * decryption, this case should never really happen.
2353 */
2354 CTR4(KTR_CXGBE, "%s: tid %u dropping TLS record type %u, length %u",
2355 __func__, toep->tid, hdr->type, be16toh(hdr->length));
2356 error = EBADMSG;
2357
2358 report_error:
2359 #ifdef KERN_TLS
2360 if (toep->tls.mode == TLS_MODE_KTLS)
2361 so->so_error = error;
2362 else
2363 #endif
2364 {
2365 /*
2366 * Report errors by sending an empty TLS record
2367 * with an error record type.
2368 */
2369 hdr->type = CONTENT_TYPE_ERROR;
2370
2371 /* Trim this CPL's mbuf to only include the TLS header. */
2372 KASSERT(m->m_len == len && m->m_next == NULL,
2373 ("%s: CPL spans multiple mbufs", __func__));
2374 m->m_len = TLS_HEADER_LENGTH;
2375 m->m_pkthdr.len = TLS_HEADER_LENGTH;
2376
2377 sbappendstream_locked(sb, m, 0);
2378 m = NULL;
2379 }
2380
2381 out:
2382 /*
2383 * This connection is going to die anyway, so probably don't
2384 * need to bother with returning credits.
2385 */
2386 rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
2387 #ifdef VERBOSE_TRACES
2388 CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u",
2389 __func__, toep->tid, rx_credits, tp->rcv_wnd);
2390 #endif
2391 if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
2392 rx_credits = send_rx_credits(toep->vi->adapter, toep,
2393 rx_credits);
2394 tp->rcv_wnd += rx_credits;
2395 tp->rcv_adv += rx_credits;
2396 }
2397
2398 sorwakeup_locked(so);
2399 SOCKBUF_UNLOCK_ASSERT(sb);
2400
2401 INP_WUNLOCK(inp);
2402 CURVNET_RESTORE();
2403
2404 m_freem(m);
2405 }
2406
2407 void
t4_tls_mod_load(void)2408 t4_tls_mod_load(void)
2409 {
2410
2411 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
2412 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
2413 }
2414
2415 void
t4_tls_mod_unload(void)2416 t4_tls_mod_unload(void)
2417 {
2418
2419 t4_register_cpl_handler(CPL_TLS_DATA, NULL);
2420 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
2421 }
2422 #endif /* TCP_OFFLOAD */
2423