1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2018-2019 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: John Baldwin <[email protected]>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "opt_inet.h"
31 #include "opt_inet6.h"
32 #include "opt_kern_tls.h"
33
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36
37 #include <sys/param.h>
38 #include <sys/ktr.h>
39 #include <sys/ktls.h>
40 #include <sys/sglist.h>
41 #include <sys/socket.h>
42 #include <sys/socketvar.h>
43 #include <sys/sockbuf.h>
44 #include <netinet/in.h>
45 #include <netinet/in_pcb.h>
46 #include <netinet/ip.h>
47 #include <netinet/ip6.h>
48 #include <netinet/tcp_var.h>
49 #include <opencrypto/cryptodev.h>
50 #include <opencrypto/xform.h>
51
52 #include "common/common.h"
53 #include "common/t4_regs.h"
54 #include "common/t4_regs_values.h"
55 #include "common/t4_tcb.h"
56 #include "t4_l2t.h"
57 #include "t4_clip.h"
58 #include "t4_mp_ring.h"
59 #include "crypto/t4_crypto.h"
60
61 #if defined(INET) || defined(INET6)
62
63 #define SALT_SIZE 4
64
65 #define GCM_TAG_SIZE 16
66 #define TLS_HEADER_LENGTH 5
67
68 #define TLS_KEY_CONTEXT_SZ roundup2(sizeof(struct tls_keyctx), 32)
69
70 struct tls_scmd {
71 __be32 seqno_numivs;
72 __be32 ivgen_hdrlen;
73 };
74
75 struct tls_key_req {
76 /* FW_ULPTX_WR */
77 __be32 wr_hi;
78 __be32 wr_mid;
79 __be32 ftid;
80 __u8 reneg_to_write_rx;
81 __u8 protocol;
82 __be16 mfs;
83 /* master command */
84 __be32 cmd;
85 __be32 len16; /* command length */
86 __be32 dlen; /* data length in 32-byte units */
87 __be32 kaddr;
88 /* sub-command */
89 __be32 sc_more;
90 __be32 sc_len;
91 }__packed;
92
93 struct tls_keyctx {
94 struct tx_keyctx_hdr {
95 __u8 ctxlen;
96 __u8 r2;
97 __be16 dualck_to_txvalid;
98 __u8 txsalt[4];
99 __be64 r5;
100 } txhdr;
101 struct keys {
102 __u8 edkey[32];
103 __u8 ipad[64];
104 __u8 opad[64];
105 } keys;
106 };
107
108 #define S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 11
109 #define M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT 0x1
110 #define V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
111 ((x) << S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
112 #define G_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(x) \
113 (((x) >> S_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT) & \
114 M_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT)
115 #define F_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT \
116 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1U)
117
118 #define S_TLS_KEYCTX_TX_WR_SALT_PRESENT 10
119 #define M_TLS_KEYCTX_TX_WR_SALT_PRESENT 0x1
120 #define V_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
121 ((x) << S_TLS_KEYCTX_TX_WR_SALT_PRESENT)
122 #define G_TLS_KEYCTX_TX_WR_SALT_PRESENT(x) \
123 (((x) >> S_TLS_KEYCTX_TX_WR_SALT_PRESENT) & \
124 M_TLS_KEYCTX_TX_WR_SALT_PRESENT)
125 #define F_TLS_KEYCTX_TX_WR_SALT_PRESENT \
126 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1U)
127
128 #define S_TLS_KEYCTX_TX_WR_TXCK_SIZE 6
129 #define M_TLS_KEYCTX_TX_WR_TXCK_SIZE 0xf
130 #define V_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
131 ((x) << S_TLS_KEYCTX_TX_WR_TXCK_SIZE)
132 #define G_TLS_KEYCTX_TX_WR_TXCK_SIZE(x) \
133 (((x) >> S_TLS_KEYCTX_TX_WR_TXCK_SIZE) & \
134 M_TLS_KEYCTX_TX_WR_TXCK_SIZE)
135
136 #define S_TLS_KEYCTX_TX_WR_TXMK_SIZE 2
137 #define M_TLS_KEYCTX_TX_WR_TXMK_SIZE 0xf
138 #define V_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
139 ((x) << S_TLS_KEYCTX_TX_WR_TXMK_SIZE)
140 #define G_TLS_KEYCTX_TX_WR_TXMK_SIZE(x) \
141 (((x) >> S_TLS_KEYCTX_TX_WR_TXMK_SIZE) & \
142 M_TLS_KEYCTX_TX_WR_TXMK_SIZE)
143
144 #define S_TLS_KEYCTX_TX_WR_TXVALID 0
145 #define M_TLS_KEYCTX_TX_WR_TXVALID 0x1
146 #define V_TLS_KEYCTX_TX_WR_TXVALID(x) \
147 ((x) << S_TLS_KEYCTX_TX_WR_TXVALID)
148 #define G_TLS_KEYCTX_TX_WR_TXVALID(x) \
149 (((x) >> S_TLS_KEYCTX_TX_WR_TXVALID) & M_TLS_KEYCTX_TX_WR_TXVALID)
150 #define F_TLS_KEYCTX_TX_WR_TXVALID V_TLS_KEYCTX_TX_WR_TXVALID(1U)
151
152 /* Key Context Programming Operation type */
153 #define KEY_WRITE_RX 0x1
154 #define KEY_WRITE_TX 0x2
155 #define KEY_DELETE_RX 0x4
156 #define KEY_DELETE_TX 0x8
157
158 struct tlspcb {
159 struct m_snd_tag com;
160 struct vi_info *vi; /* virtual interface */
161 struct adapter *sc;
162 struct l2t_entry *l2te; /* L2 table entry used by this connection */
163 int tid; /* Connection identifier */
164
165 int tx_key_addr;
166 bool inline_key;
167 bool using_timestamps;
168 unsigned char enc_mode;
169
170 struct tls_scmd scmd0;
171 struct tls_scmd scmd0_short;
172
173 unsigned int tx_key_info_size;
174
175 uint32_t prev_seq;
176 uint32_t prev_ack;
177 uint32_t prev_tsecr;
178 uint16_t prev_win;
179 uint16_t prev_mss;
180
181 /* Only used outside of setup and teardown when using inline keys. */
182 struct tls_keyctx keyctx;
183
184 /* Fields only used during setup and teardown. */
185 struct inpcb *inp; /* backpointer to host stack's PCB */
186 struct sge_txq *txq;
187 struct sge_wrq *ctrlq;
188 struct clip_entry *ce; /* CLIP table entry used by this tid */
189
190 unsigned char auth_mode;
191 unsigned char hmac_ctrl;
192 unsigned char mac_first;
193 unsigned char iv_size;
194
195 unsigned int frag_size;
196 unsigned int cipher_secret_size;
197 int proto_ver;
198
199 bool open_pending;
200 };
201
202 static int ktls_setup_keys(struct tlspcb *tlsp,
203 const struct ktls_session *tls, struct sge_txq *txq);
204
205 static inline struct tlspcb *
mst_to_tls(struct m_snd_tag * t)206 mst_to_tls(struct m_snd_tag *t)
207 {
208 return (__containerof(t, struct tlspcb, com));
209 }
210
211 /* XXX: There are similar versions of these two in tom/t4_tls.c. */
212 static int
get_new_keyid(struct tlspcb * tlsp)213 get_new_keyid(struct tlspcb *tlsp)
214 {
215 vmem_addr_t addr;
216
217 if (vmem_alloc(tlsp->sc->key_map, TLS_KEY_CONTEXT_SZ,
218 M_NOWAIT | M_FIRSTFIT, &addr) != 0)
219 return (-1);
220
221 return (addr);
222 }
223
224 static void
free_keyid(struct tlspcb * tlsp,int keyid)225 free_keyid(struct tlspcb *tlsp, int keyid)
226 {
227
228 CTR3(KTR_CXGBE, "%s: tid %d key addr %#x", __func__, tlsp->tid, keyid);
229 vmem_free(tlsp->sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
230 }
231
232 static struct tlspcb *
alloc_tlspcb(struct ifnet * ifp,struct vi_info * vi,int flags)233 alloc_tlspcb(struct ifnet *ifp, struct vi_info *vi, int flags)
234 {
235 struct port_info *pi = vi->pi;
236 struct adapter *sc = pi->adapter;
237 struct tlspcb *tlsp;
238
239 tlsp = malloc(sizeof(*tlsp), M_CXGBE, M_ZERO | flags);
240 if (tlsp == NULL)
241 return (NULL);
242
243 m_snd_tag_init(&tlsp->com, ifp, IF_SND_TAG_TYPE_TLS);
244 tlsp->vi = vi;
245 tlsp->sc = sc;
246 tlsp->ctrlq = &sc->sge.ctrlq[pi->port_id];
247 tlsp->tid = -1;
248 tlsp->tx_key_addr = -1;
249
250 return (tlsp);
251 }
252
253 static void
init_ktls_key_params(struct tlspcb * tlsp,const struct ktls_session * tls)254 init_ktls_key_params(struct tlspcb *tlsp, const struct ktls_session *tls)
255 {
256 int mac_key_size;
257
258 if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
259 tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_1;
260 else
261 tlsp->proto_ver = SCMD_PROTO_VERSION_TLS_1_2;
262 tlsp->cipher_secret_size = tls->params.cipher_key_len;
263 tlsp->tx_key_info_size = sizeof(struct tx_keyctx_hdr) +
264 tlsp->cipher_secret_size;
265 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
266 tlsp->auth_mode = SCMD_AUTH_MODE_GHASH;
267 tlsp->enc_mode = SCMD_CIPH_MODE_AES_GCM;
268 tlsp->iv_size = 4;
269 tlsp->mac_first = 0;
270 tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NOP;
271 tlsp->tx_key_info_size += GMAC_BLOCK_LEN;
272 } else {
273 switch (tls->params.auth_algorithm) {
274 case CRYPTO_SHA1_HMAC:
275 mac_key_size = roundup2(SHA1_HASH_LEN, 16);
276 tlsp->auth_mode = SCMD_AUTH_MODE_SHA1;
277 break;
278 case CRYPTO_SHA2_256_HMAC:
279 mac_key_size = SHA2_256_HASH_LEN;
280 tlsp->auth_mode = SCMD_AUTH_MODE_SHA256;
281 break;
282 case CRYPTO_SHA2_384_HMAC:
283 mac_key_size = SHA2_512_HASH_LEN;
284 tlsp->auth_mode = SCMD_AUTH_MODE_SHA512_384;
285 break;
286 }
287 tlsp->enc_mode = SCMD_CIPH_MODE_AES_CBC;
288 tlsp->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
289 tlsp->mac_first = 1;
290 tlsp->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
291 tlsp->tx_key_info_size += mac_key_size * 2;
292 }
293
294 tlsp->frag_size = tls->params.max_frame_len;
295 }
296
297 static int
ktls_act_open_cpl_size(bool isipv6)298 ktls_act_open_cpl_size(bool isipv6)
299 {
300
301 if (isipv6)
302 return (sizeof(struct cpl_t6_act_open_req6));
303 else
304 return (sizeof(struct cpl_t6_act_open_req));
305 }
306
307 static void
mk_ktls_act_open_req(struct adapter * sc,struct vi_info * vi,struct inpcb * inp,struct tlspcb * tlsp,int atid,void * dst)308 mk_ktls_act_open_req(struct adapter *sc, struct vi_info *vi, struct inpcb *inp,
309 struct tlspcb *tlsp, int atid, void *dst)
310 {
311 struct tcpcb *tp = intotcpcb(inp);
312 struct cpl_t6_act_open_req *cpl6;
313 struct cpl_act_open_req *cpl;
314 uint64_t options;
315 int qid_atid;
316
317 cpl6 = dst;
318 cpl = (struct cpl_act_open_req *)cpl6;
319 INIT_TP_WR(cpl6, 0);
320 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
321 V_TID_COOKIE(CPL_COOKIE_KERN_TLS);
322 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
323 qid_atid));
324 inp_4tuple_get(inp, &cpl->local_ip, &cpl->local_port,
325 &cpl->peer_ip, &cpl->peer_port);
326
327 options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE);
328 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan);
329 options |= F_NON_OFFLOAD;
330 cpl->opt0 = htobe64(options);
331
332 options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]);
333 if (tp->t_flags & TF_REQ_TSTMP)
334 options |= F_TSTAMPS_EN;
335 cpl->opt2 = htobe32(options);
336 }
337
338 static void
mk_ktls_act_open_req6(struct adapter * sc,struct vi_info * vi,struct inpcb * inp,struct tlspcb * tlsp,int atid,void * dst)339 mk_ktls_act_open_req6(struct adapter *sc, struct vi_info *vi,
340 struct inpcb *inp, struct tlspcb *tlsp, int atid, void *dst)
341 {
342 struct tcpcb *tp = intotcpcb(inp);
343 struct cpl_t6_act_open_req6 *cpl6;
344 struct cpl_act_open_req6 *cpl;
345 uint64_t options;
346 int qid_atid;
347
348 cpl6 = dst;
349 cpl = (struct cpl_act_open_req6 *)cpl6;
350 INIT_TP_WR(cpl6, 0);
351 qid_atid = V_TID_QID(sc->sge.fwq.abs_id) | V_TID_TID(atid) |
352 V_TID_COOKIE(CPL_COOKIE_KERN_TLS);
353 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
354 qid_atid));
355 cpl->local_port = inp->inp_lport;
356 cpl->local_ip_hi = *(uint64_t *)&inp->in6p_laddr.s6_addr[0];
357 cpl->local_ip_lo = *(uint64_t *)&inp->in6p_laddr.s6_addr[8];
358 cpl->peer_port = inp->inp_fport;
359 cpl->peer_ip_hi = *(uint64_t *)&inp->in6p_faddr.s6_addr[0];
360 cpl->peer_ip_lo = *(uint64_t *)&inp->in6p_faddr.s6_addr[8];
361
362 options = F_TCAM_BYPASS | V_ULP_MODE(ULP_MODE_NONE);
363 options |= V_SMAC_SEL(vi->smt_idx) | V_TX_CHAN(vi->pi->tx_chan);
364 options |= F_NON_OFFLOAD;
365 cpl->opt0 = htobe64(options);
366
367 options = V_TX_QUEUE(sc->params.tp.tx_modq[vi->pi->tx_chan]);
368 if (tp->t_flags & TF_REQ_TSTMP)
369 options |= F_TSTAMPS_EN;
370 cpl->opt2 = htobe32(options);
371 }
372
373 static int
send_ktls_act_open_req(struct adapter * sc,struct vi_info * vi,struct inpcb * inp,struct tlspcb * tlsp,int atid)374 send_ktls_act_open_req(struct adapter *sc, struct vi_info *vi,
375 struct inpcb *inp, struct tlspcb *tlsp, int atid)
376 {
377 struct wrqe *wr;
378 bool isipv6;
379
380 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
381 if (isipv6) {
382 tlsp->ce = t4_get_clip_entry(sc, &inp->in6p_laddr, true);
383 if (tlsp->ce == NULL)
384 return (ENOENT);
385 }
386
387 wr = alloc_wrqe(ktls_act_open_cpl_size(isipv6), tlsp->ctrlq);
388 if (wr == NULL) {
389 CTR2(KTR_CXGBE, "%s: atid %d failed to alloc WR", __func__,
390 atid);
391 return (ENOMEM);
392 }
393
394 if (isipv6)
395 mk_ktls_act_open_req6(sc, vi, inp, tlsp, atid, wrtod(wr));
396 else
397 mk_ktls_act_open_req(sc, vi, inp, tlsp, atid, wrtod(wr));
398
399 tlsp->open_pending = true;
400 t4_wrq_tx(sc, wr);
401 return (0);
402 }
403
404 static int
ktls_act_open_rpl(struct sge_iq * iq,const struct rss_header * rss,struct mbuf * m)405 ktls_act_open_rpl(struct sge_iq *iq, const struct rss_header *rss,
406 struct mbuf *m)
407 {
408 struct adapter *sc = iq->adapter;
409 const struct cpl_act_open_rpl *cpl = (const void *)(rss + 1);
410 u_int atid = G_TID_TID(G_AOPEN_ATID(be32toh(cpl->atid_status)));
411 u_int status = G_AOPEN_STATUS(be32toh(cpl->atid_status));
412 struct tlspcb *tlsp = lookup_atid(sc, atid);
413 struct inpcb *inp = tlsp->inp;
414
415 CTR3(KTR_CXGBE, "%s: atid %d status %d", __func__, atid, status);
416 free_atid(sc, atid);
417 if (status == 0)
418 tlsp->tid = GET_TID(cpl);
419
420 INP_WLOCK(inp);
421 tlsp->open_pending = false;
422 wakeup(tlsp);
423 INP_WUNLOCK(inp);
424 return (0);
425 }
426
427 /* SET_TCB_FIELD sent as a ULP command looks like this */
428 #define LEN__SET_TCB_FIELD_ULP (sizeof(struct ulp_txpkt) + \
429 sizeof(struct ulptx_idata) + sizeof(struct cpl_set_tcb_field_core))
430
431 _Static_assert((LEN__SET_TCB_FIELD_ULP + sizeof(struct ulptx_idata)) % 16 == 0,
432 "CPL_SET_TCB_FIELD ULP command not 16-byte aligned");
433
434 static void
write_set_tcb_field_ulp(struct tlspcb * tlsp,void * dst,struct sge_txq * txq,uint16_t word,uint64_t mask,uint64_t val)435 write_set_tcb_field_ulp(struct tlspcb *tlsp, void *dst, struct sge_txq *txq,
436 uint16_t word, uint64_t mask, uint64_t val)
437 {
438 struct ulp_txpkt *txpkt;
439 struct ulptx_idata *idata;
440 struct cpl_set_tcb_field_core *cpl;
441
442 /* ULP_TXPKT */
443 txpkt = dst;
444 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
445 V_ULP_TXPKT_DATAMODIFY(0) |
446 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |
447 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1));
448 txpkt->len = htobe32(howmany(LEN__SET_TCB_FIELD_ULP, 16));
449
450 /* ULPTX_IDATA sub-command */
451 idata = (struct ulptx_idata *)(txpkt + 1);
452 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
453 idata->len = htobe32(sizeof(*cpl));
454
455 /* CPL_SET_TCB_FIELD */
456 cpl = (struct cpl_set_tcb_field_core *)(idata + 1);
457 OPCODE_TID(cpl) = htobe32(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tlsp->tid));
458 cpl->reply_ctrl = htobe16(F_NO_REPLY);
459 cpl->word_cookie = htobe16(V_WORD(word));
460 cpl->mask = htobe64(mask);
461 cpl->val = htobe64(val);
462
463 /* ULPTX_NOOP */
464 idata = (struct ulptx_idata *)(cpl + 1);
465 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
466 idata->len = htobe32(0);
467 }
468
469 static int
ktls_set_tcb_fields(struct tlspcb * tlsp,struct tcpcb * tp,struct sge_txq * txq)470 ktls_set_tcb_fields(struct tlspcb *tlsp, struct tcpcb *tp, struct sge_txq *txq)
471 {
472 struct fw_ulptx_wr *wr;
473 struct mbuf *m;
474 char *dst;
475 void *items[1];
476 int error, len;
477
478 len = sizeof(*wr) + 3 * roundup2(LEN__SET_TCB_FIELD_ULP, 16);
479 if (tp->t_flags & TF_REQ_TSTMP)
480 len += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
481 m = alloc_wr_mbuf(len, M_NOWAIT);
482 if (m == NULL) {
483 CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__,
484 tlsp->tid);
485 return (ENOMEM);
486 }
487 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
488 m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
489
490 /* FW_ULPTX_WR */
491 wr = mtod(m, void *);
492 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
493 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
494 V_FW_WR_LEN16(len / 16));
495 wr->cookie = 0;
496 dst = (char *)(wr + 1);
497
498 /* Clear TF_NON_OFFLOAD and set TF_CORE_BYPASS */
499 write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_T_FLAGS,
500 V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1) | V_TF_NON_OFFLOAD(1)),
501 V_TCB_T_FLAGS(V_TF_CORE_BYPASS(1)));
502 dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
503
504 /* Clear the SND_UNA_RAW, SND_NXT_RAW, and SND_MAX_RAW offsets. */
505 write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_UNA_RAW,
506 V_TCB_SND_NXT_RAW(M_TCB_SND_NXT_RAW) |
507 V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW),
508 V_TCB_SND_NXT_RAW(0) | V_TCB_SND_UNA_RAW(0));
509 dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
510
511 write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_SND_MAX_RAW,
512 V_TCB_SND_MAX_RAW(M_TCB_SND_MAX_RAW), V_TCB_SND_MAX_RAW(0));
513 dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
514
515 if (tp->t_flags & TF_REQ_TSTMP) {
516 write_set_tcb_field_ulp(tlsp, dst, txq, W_TCB_TIMESTAMP_OFFSET,
517 V_TCB_TIMESTAMP_OFFSET(M_TCB_TIMESTAMP_OFFSET),
518 V_TCB_TIMESTAMP_OFFSET(tp->ts_offset >> 28));
519 dst += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
520 }
521
522 KASSERT(dst - (char *)wr == len, ("%s: length mismatch", __func__));
523
524 items[0] = m;
525 error = mp_ring_enqueue(txq->r, items, 1, 1);
526 if (error)
527 m_free(m);
528 return (error);
529 }
530
531 int
cxgbe_tls_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** pt)532 cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
533 struct m_snd_tag **pt)
534 {
535 const struct ktls_session *tls;
536 struct tlspcb *tlsp;
537 struct adapter *sc;
538 struct vi_info *vi;
539 struct inpcb *inp;
540 struct tcpcb *tp;
541 struct sge_txq *txq;
542 int atid, error, keyid;
543
544 tls = params->tls.tls;
545
546 /* Only TLS 1.1 and TLS 1.2 are currently supported. */
547 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
548 tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
549 tls->params.tls_vminor > TLS_MINOR_VER_TWO)
550 return (EPROTONOSUPPORT);
551
552 /* Sanity check values in *tls. */
553 switch (tls->params.cipher_algorithm) {
554 case CRYPTO_AES_CBC:
555 /* XXX: Explicitly ignore any provided IV. */
556 switch (tls->params.cipher_key_len) {
557 case 128 / 8:
558 case 192 / 8:
559 case 256 / 8:
560 break;
561 default:
562 return (EINVAL);
563 }
564 switch (tls->params.auth_algorithm) {
565 case CRYPTO_SHA1_HMAC:
566 case CRYPTO_SHA2_256_HMAC:
567 case CRYPTO_SHA2_384_HMAC:
568 break;
569 default:
570 return (EPROTONOSUPPORT);
571 }
572 break;
573 case CRYPTO_AES_NIST_GCM_16:
574 if (tls->params.iv_len != SALT_SIZE)
575 return (EINVAL);
576 switch (tls->params.cipher_key_len) {
577 case 128 / 8:
578 case 192 / 8:
579 case 256 / 8:
580 break;
581 default:
582 return (EINVAL);
583 }
584 break;
585 default:
586 return (EPROTONOSUPPORT);
587 }
588
589 vi = ifp->if_softc;
590 sc = vi->adapter;
591
592 tlsp = alloc_tlspcb(ifp, vi, M_WAITOK);
593
594 atid = alloc_atid(sc, tlsp);
595 if (atid < 0) {
596 error = ENOMEM;
597 goto failed;
598 }
599
600 if (sc->tlst.inline_keys)
601 keyid = -1;
602 else
603 keyid = get_new_keyid(tlsp);
604 if (keyid < 0) {
605 CTR2(KTR_CXGBE, "%s: atid %d using immediate key ctx", __func__,
606 atid);
607 tlsp->inline_key = true;
608 } else {
609 tlsp->tx_key_addr = keyid;
610 CTR3(KTR_CXGBE, "%s: atid %d allocated TX key addr %#x",
611 __func__,
612 atid, tlsp->tx_key_addr);
613 }
614
615 inp = params->tls.inp;
616 INP_RLOCK(inp);
617 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
618 INP_RUNLOCK(inp);
619 error = ECONNRESET;
620 goto failed;
621 }
622 tlsp->inp = inp;
623
624 tp = inp->inp_ppcb;
625 if (tp->t_flags & TF_REQ_TSTMP) {
626 tlsp->using_timestamps = true;
627 if ((tp->ts_offset & 0xfffffff) != 0) {
628 INP_RUNLOCK(inp);
629 error = EINVAL;
630 goto failed;
631 }
632 } else
633 tlsp->using_timestamps = false;
634
635 error = send_ktls_act_open_req(sc, vi, inp, tlsp, atid);
636 if (error) {
637 INP_RUNLOCK(inp);
638 goto failed;
639 }
640
641 /* Wait for reply to active open. */
642 CTR2(KTR_CXGBE, "%s: atid %d sent CPL_ACT_OPEN_REQ", __func__,
643 atid);
644 while (tlsp->open_pending) {
645 /*
646 * XXX: PCATCH? We would then have to discard the PCB
647 * when the completion CPL arrived.
648 */
649 error = rw_sleep(tlsp, &inp->inp_lock, 0, "t6tlsop", 0);
650 }
651
652 atid = -1;
653 if (tlsp->tid < 0) {
654 INP_RUNLOCK(inp);
655 error = ENOMEM;
656 goto failed;
657 }
658
659 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
660 INP_RUNLOCK(inp);
661 error = ECONNRESET;
662 goto failed;
663 }
664
665 txq = &sc->sge.txq[vi->first_txq];
666 if (inp->inp_flowtype != M_HASHTYPE_NONE)
667 txq += ((inp->inp_flowid % (vi->ntxq - vi->rsrv_noflowq)) +
668 vi->rsrv_noflowq);
669 tlsp->txq = txq;
670
671 error = ktls_set_tcb_fields(tlsp, tp, txq);
672 INP_RUNLOCK(inp);
673 if (error)
674 goto failed;
675
676 init_ktls_key_params(tlsp, tls);
677
678 error = ktls_setup_keys(tlsp, tls, txq);
679 if (error)
680 goto failed;
681
682 /* The SCMD fields used when encrypting a full TLS record. */
683 tlsp->scmd0.seqno_numivs = htobe32(V_SCMD_SEQ_NO_CTRL(3) |
684 V_SCMD_PROTO_VERSION(tlsp->proto_ver) |
685 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
686 V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) |
687 V_SCMD_CIPH_MODE(tlsp->enc_mode) |
688 V_SCMD_AUTH_MODE(tlsp->auth_mode) |
689 V_SCMD_HMAC_CTRL(tlsp->hmac_ctrl) |
690 V_SCMD_IV_SIZE(tlsp->iv_size) | V_SCMD_NUM_IVS(1));
691
692 tlsp->scmd0.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
693 V_SCMD_TLS_FRAG_ENABLE(0);
694 if (tlsp->inline_key)
695 tlsp->scmd0.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
696 tlsp->scmd0.ivgen_hdrlen = htobe32(tlsp->scmd0.ivgen_hdrlen);
697
698 /*
699 * The SCMD fields used when encrypting a partial TLS record
700 * (no trailer and possibly a truncated payload).
701 */
702 tlsp->scmd0_short.seqno_numivs = V_SCMD_SEQ_NO_CTRL(0) |
703 V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
704 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
705 V_SCMD_CIPH_AUTH_SEQ_CTRL((tlsp->mac_first == 0)) |
706 V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
707 V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
708 V_SCMD_IV_SIZE(AES_BLOCK_LEN / 2) | V_SCMD_NUM_IVS(0);
709 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
710 tlsp->scmd0_short.seqno_numivs |=
711 V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CTR);
712 else
713 tlsp->scmd0_short.seqno_numivs |=
714 V_SCMD_CIPH_MODE(tlsp->enc_mode);
715 tlsp->scmd0_short.seqno_numivs =
716 htobe32(tlsp->scmd0_short.seqno_numivs);
717
718 tlsp->scmd0_short.ivgen_hdrlen = V_SCMD_IV_GEN_CTRL(0) |
719 V_SCMD_TLS_FRAG_ENABLE(0) |
720 V_SCMD_AADIVDROP(1);
721 if (tlsp->inline_key)
722 tlsp->scmd0_short.ivgen_hdrlen |= V_SCMD_KEY_CTX_INLINE(1);
723
724 TXQ_LOCK(txq);
725 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
726 txq->kern_tls_gcm++;
727 else
728 txq->kern_tls_cbc++;
729 TXQ_UNLOCK(txq);
730 *pt = &tlsp->com;
731 return (0);
732
733 failed:
734 if (atid >= 0)
735 free_atid(sc, atid);
736 m_snd_tag_rele(&tlsp->com);
737 return (error);
738 }
739
740 static int
ktls_setup_keys(struct tlspcb * tlsp,const struct ktls_session * tls,struct sge_txq * txq)741 ktls_setup_keys(struct tlspcb *tlsp, const struct ktls_session *tls,
742 struct sge_txq *txq)
743 {
744 struct auth_hash *axf;
745 int error, keyid, kwrlen, kctxlen, len;
746 struct tls_key_req *kwr;
747 struct tls_keyctx *kctx;
748 void *items[1], *key;
749 struct tx_keyctx_hdr *khdr;
750 unsigned int ck_size, mk_size, partial_digest_len;
751 struct mbuf *m;
752
753 /*
754 * Store the salt and keys in the key context. For
755 * connections with an inline key, this key context is passed
756 * as immediate data in each work request. For connections
757 * storing the key in DDR, a work request is used to store a
758 * copy of the key context in DDR.
759 */
760 kctx = &tlsp->keyctx;
761 khdr = &kctx->txhdr;
762
763 switch (tlsp->cipher_secret_size) {
764 case 128 / 8:
765 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
766 break;
767 case 192 / 8:
768 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
769 break;
770 case 256 / 8:
771 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
772 break;
773 default:
774 panic("bad key size");
775 }
776 axf = NULL;
777 partial_digest_len = 0;
778 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM)
779 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
780 else {
781 switch (tlsp->auth_mode) {
782 case SCMD_AUTH_MODE_SHA1:
783 axf = &auth_hash_hmac_sha1;
784 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
785 partial_digest_len = SHA1_HASH_LEN;
786 break;
787 case SCMD_AUTH_MODE_SHA256:
788 axf = &auth_hash_hmac_sha2_256;
789 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
790 partial_digest_len = SHA2_256_HASH_LEN;
791 break;
792 case SCMD_AUTH_MODE_SHA512_384:
793 axf = &auth_hash_hmac_sha2_384;
794 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
795 partial_digest_len = SHA2_512_HASH_LEN;
796 break;
797 default:
798 panic("bad auth mode");
799 }
800 }
801
802 khdr->ctxlen = (tlsp->tx_key_info_size >> 4);
803 khdr->dualck_to_txvalid = V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
804 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(ck_size) |
805 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(mk_size) |
806 V_TLS_KEYCTX_TX_WR_TXVALID(1);
807 if (tlsp->enc_mode != SCMD_CIPH_MODE_AES_GCM)
808 khdr->dualck_to_txvalid |= V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1);
809 khdr->dualck_to_txvalid = htobe16(khdr->dualck_to_txvalid);
810 key = kctx->keys.edkey;
811 memcpy(key, tls->params.cipher_key, tls->params.cipher_key_len);
812 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
813 memcpy(khdr->txsalt, tls->params.iv, SALT_SIZE);
814 t4_init_gmac_hash(tls->params.cipher_key,
815 tls->params.cipher_key_len,
816 (char *)key + tls->params.cipher_key_len);
817 } else {
818 t4_init_hmac_digest(axf, partial_digest_len,
819 tls->params.auth_key, tls->params.auth_key_len,
820 (char *)key + tls->params.cipher_key_len);
821 }
822
823 if (tlsp->inline_key)
824 return (0);
825
826 keyid = tlsp->tx_key_addr;
827
828 /* Populate key work request. */
829 kwrlen = sizeof(*kwr);
830 kctxlen = roundup2(sizeof(*kctx), 32);
831 len = kwrlen + kctxlen;
832
833 m = alloc_wr_mbuf(len, M_NOWAIT);
834 if (m == NULL) {
835 CTR2(KTR_CXGBE, "%s: tid %d failed to alloc WR mbuf", __func__,
836 tlsp->tid);
837 return (ENOMEM);
838 }
839 m->m_pkthdr.snd_tag = m_snd_tag_ref(&tlsp->com);
840 m->m_pkthdr.csum_flags |= CSUM_SND_TAG;
841 kwr = mtod(m, void *);
842 memset(kwr, 0, len);
843
844 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) |
845 F_FW_WR_ATOMIC);
846 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)));
847 kwr->protocol = tlsp->proto_ver;
848 kwr->mfs = htons(tlsp->frag_size);
849 kwr->reneg_to_write_rx = KEY_WRITE_TX;
850
851 /* master command */
852 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
853 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
854 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
855 kwr->len16 = htobe32((tlsp->tid << 8) |
856 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
857 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
858
859 /* sub command */
860 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
861 kwr->sc_len = htobe32(kctxlen);
862
863 kctx = (struct tls_keyctx *)(kwr + 1);
864 memcpy(kctx, &tlsp->keyctx, sizeof(*kctx));
865
866 /*
867 * Place the key work request in the transmit queue. It
868 * should be sent to the NIC before any TLS packets using this
869 * session.
870 */
871 items[0] = m;
872 error = mp_ring_enqueue(txq->r, items, 1, 1);
873 if (error)
874 m_free(m);
875 else
876 CTR2(KTR_CXGBE, "%s: tid %d sent key WR", __func__, tlsp->tid);
877 return (error);
878 }
879
880 static u_int
ktls_base_wr_size(struct tlspcb * tlsp)881 ktls_base_wr_size(struct tlspcb *tlsp)
882 {
883 u_int wr_len;
884
885 wr_len = sizeof(struct fw_ulptx_wr); // 16
886 wr_len += sizeof(struct ulp_txpkt); // 8
887 wr_len += sizeof(struct ulptx_idata); // 8
888 wr_len += sizeof(struct cpl_tx_sec_pdu);// 32
889 if (tlsp->inline_key)
890 wr_len += tlsp->tx_key_info_size;
891 else {
892 wr_len += sizeof(struct ulptx_sc_memrd);// 8
893 wr_len += sizeof(struct ulptx_idata); // 8
894 }
895 wr_len += sizeof(struct cpl_tx_data); // 16
896 return (wr_len);
897 }
898
899 /* How many bytes of TCP payload to send for a given TLS record. */
900 static u_int
ktls_tcp_payload_length(struct tlspcb * tlsp,struct mbuf * m_tls)901 ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
902 {
903 struct tls_record_layer *hdr;
904 u_int plen, mlen;
905
906 M_ASSERTEXTPG(m_tls);
907 hdr = (void *)m_tls->m_epg_hdr;
908 plen = ntohs(hdr->tls_length);
909
910 /*
911 * What range of the TLS record is the mbuf requesting to be
912 * sent.
913 */
914 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
915
916 /* Always send complete records. */
917 if (mlen == TLS_HEADER_LENGTH + plen)
918 return (mlen);
919
920 /*
921 * If the host stack has asked to send part of the trailer,
922 * trim the length to avoid sending any of the trailer. There
923 * is no way to send a partial trailer currently.
924 */
925 if (mlen > TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen)
926 mlen = TLS_HEADER_LENGTH + plen - m_tls->m_epg_trllen;
927
928
929 /*
930 * For AES-CBC adjust the ciphertext length for the block
931 * size.
932 */
933 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&
934 mlen > TLS_HEADER_LENGTH) {
935 mlen = TLS_HEADER_LENGTH + rounddown(mlen - TLS_HEADER_LENGTH,
936 AES_BLOCK_LEN);
937 }
938
939 #ifdef VERBOSE_TRACES
940 CTR4(KTR_CXGBE, "%s: tid %d short TLS record (%u vs %u)",
941 __func__, tlsp->tid, mlen, TLS_HEADER_LENGTH + plen);
942 #endif
943 return (mlen);
944 }
945
946 /*
947 * For a "short" TLS record, determine the offset into the TLS record
948 * payload to send. This offset does not include the TLS header, but
949 * a non-zero offset implies that a header will not be sent.
950 */
951 static u_int
ktls_payload_offset(struct tlspcb * tlsp,struct mbuf * m_tls)952 ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
953 {
954 struct tls_record_layer *hdr;
955 u_int offset, plen;
956 #ifdef INVARIANTS
957 u_int mlen;
958 #endif
959
960 M_ASSERTEXTPG(m_tls);
961 hdr = (void *)m_tls->m_epg_hdr;
962 plen = ntohs(hdr->tls_length);
963 #ifdef INVARIANTS
964 mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
965 MPASS(mlen < TLS_HEADER_LENGTH + plen);
966 #endif
967 if (mtod(m_tls, vm_offset_t) <= m_tls->m_epg_hdrlen)
968 return (0);
969 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
970 /*
971 * Always send something. This function is only called
972 * if we aren't sending the tag at all, but if the
973 * request starts in the tag then we are in an odd
974 * state where would effectively send nothing. Cap
975 * the offset at the last byte of the record payload
976 * to send the last cipher block.
977 */
978 offset = min(mtod(m_tls, vm_offset_t) - m_tls->m_epg_hdrlen,
979 (plen - TLS_HEADER_LENGTH - m_tls->m_epg_trllen) - 1);
980 return (rounddown(offset, AES_BLOCK_LEN));
981 }
982 return (0);
983 }
984
985 static u_int
ktls_sgl_size(u_int nsegs)986 ktls_sgl_size(u_int nsegs)
987 {
988 u_int wr_len;
989
990 /* First segment is part of ulptx_sgl. */
991 nsegs--;
992
993 wr_len = sizeof(struct ulptx_sgl);
994 wr_len += 8 * ((3 * nsegs) / 2 + (nsegs & 1));
995 return (wr_len);
996 }
997
998 static int
ktls_wr_len(struct tlspcb * tlsp,struct mbuf * m,struct mbuf * m_tls,int * nsegsp)999 ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
1000 int *nsegsp)
1001 {
1002 struct tls_record_layer *hdr;
1003 u_int imm_len, offset, plen, wr_len, tlen;
1004
1005 M_ASSERTEXTPG(m_tls);
1006
1007 /*
1008 * Determine the size of the TLS record payload to send
1009 * excluding header and trailer.
1010 */
1011 tlen = ktls_tcp_payload_length(tlsp, m_tls);
1012 if (tlen <= m_tls->m_epg_hdrlen) {
1013 /*
1014 * For requests that only want to send the TLS header,
1015 * send a tunnelled packet as immediate data.
1016 */
1017 wr_len = sizeof(struct fw_eth_tx_pkt_wr) +
1018 sizeof(struct cpl_tx_pkt_core) +
1019 roundup2(m->m_len + m_tls->m_len, 16);
1020 if (wr_len > SGE_MAX_WR_LEN) {
1021 CTR3(KTR_CXGBE,
1022 "%s: tid %d TLS header-only packet too long (len %d)",
1023 __func__, tlsp->tid, m->m_len + m_tls->m_len);
1024 }
1025
1026 /* This should always be the last TLS record in a chain. */
1027 MPASS(m_tls->m_next == NULL);
1028
1029 /*
1030 * XXX: Set a bogus 'nsegs' value to avoid tripping an
1031 * assertion in mbuf_nsegs() in t4_sge.c.
1032 */
1033 *nsegsp = 1;
1034 return (wr_len);
1035 }
1036
1037 hdr = (void *)m_tls->m_epg_hdr;
1038 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen;
1039 if (tlen < plen) {
1040 plen = tlen;
1041 offset = ktls_payload_offset(tlsp, m_tls);
1042 } else
1043 offset = 0;
1044
1045 /* Calculate the size of the work request. */
1046 wr_len = ktls_base_wr_size(tlsp);
1047
1048 /*
1049 * Full records and short records with an offset of 0 include
1050 * the TLS header as immediate data. Short records include a
1051 * raw AES IV as immediate data.
1052 */
1053 imm_len = 0;
1054 if (offset == 0)
1055 imm_len += m_tls->m_epg_hdrlen;
1056 if (plen == tlen)
1057 imm_len += AES_BLOCK_LEN;
1058 wr_len += roundup2(imm_len, 16);
1059
1060 /* TLS record payload via DSGL. */
1061 *nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen + offset,
1062 plen - (m_tls->m_epg_hdrlen + offset));
1063 wr_len += ktls_sgl_size(*nsegsp);
1064
1065 wr_len = roundup2(wr_len, 16);
1066 return (wr_len);
1067 }
1068
1069 /*
1070 * See if we have any TCP options requiring a dedicated options-only
1071 * packet.
1072 */
1073 static int
ktls_has_tcp_options(struct tcphdr * tcp)1074 ktls_has_tcp_options(struct tcphdr *tcp)
1075 {
1076 u_char *cp;
1077 int cnt, opt, optlen;
1078
1079 cp = (u_char *)(tcp + 1);
1080 cnt = tcp->th_off * 4 - sizeof(struct tcphdr);
1081 for (; cnt > 0; cnt -= optlen, cp += optlen) {
1082 opt = cp[0];
1083 if (opt == TCPOPT_EOL)
1084 break;
1085 if (opt == TCPOPT_NOP)
1086 optlen = 1;
1087 else {
1088 if (cnt < 2)
1089 break;
1090 optlen = cp[1];
1091 if (optlen < 2 || optlen > cnt)
1092 break;
1093 }
1094 switch (opt) {
1095 case TCPOPT_NOP:
1096 case TCPOPT_TIMESTAMP:
1097 break;
1098 default:
1099 return (1);
1100 }
1101 }
1102 return (0);
1103 }
1104
1105 /*
1106 * Find the TCP timestamp option.
1107 */
1108 static void *
ktls_find_tcp_timestamps(struct tcphdr * tcp)1109 ktls_find_tcp_timestamps(struct tcphdr *tcp)
1110 {
1111 u_char *cp;
1112 int cnt, opt, optlen;
1113
1114 cp = (u_char *)(tcp + 1);
1115 cnt = tcp->th_off * 4 - sizeof(struct tcphdr);
1116 for (; cnt > 0; cnt -= optlen, cp += optlen) {
1117 opt = cp[0];
1118 if (opt == TCPOPT_EOL)
1119 break;
1120 if (opt == TCPOPT_NOP)
1121 optlen = 1;
1122 else {
1123 if (cnt < 2)
1124 break;
1125 optlen = cp[1];
1126 if (optlen < 2 || optlen > cnt)
1127 break;
1128 }
1129 if (opt == TCPOPT_TIMESTAMP && optlen == TCPOLEN_TIMESTAMP)
1130 return (cp + 2);
1131 }
1132 return (NULL);
1133 }
1134
1135 int
t6_ktls_parse_pkt(struct mbuf * m,int * nsegsp,int * len16p)1136 t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p)
1137 {
1138 struct tlspcb *tlsp;
1139 struct ether_header *eh;
1140 struct ip *ip;
1141 struct ip6_hdr *ip6;
1142 struct tcphdr *tcp;
1143 struct mbuf *m_tls;
1144 int nsegs;
1145 u_int wr_len, tot_len;
1146
1147 /*
1148 * Locate headers in initial mbuf.
1149 *
1150 * XXX: This assumes all of the headers are in the initial mbuf.
1151 * Could perhaps use m_advance() like parse_pkt() if that turns
1152 * out to not be true.
1153 */
1154 M_ASSERTPKTHDR(m);
1155 MPASS(m->m_pkthdr.snd_tag != NULL);
1156 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
1157
1158 if (m->m_len <= sizeof(*eh) + sizeof(*ip)) {
1159 CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short", __func__,
1160 tlsp->tid);
1161 return (EINVAL);
1162 }
1163 eh = mtod(m, struct ether_header *);
1164 if (ntohs(eh->ether_type) != ETHERTYPE_IP &&
1165 ntohs(eh->ether_type) != ETHERTYPE_IPV6) {
1166 CTR2(KTR_CXGBE, "%s: tid %d mbuf not ETHERTYPE_IP{,V6}",
1167 __func__, tlsp->tid);
1168 return (EINVAL);
1169 }
1170 m->m_pkthdr.l2hlen = sizeof(*eh);
1171
1172 /* XXX: Reject unsupported IP options? */
1173 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1174 ip = (struct ip *)(eh + 1);
1175 if (ip->ip_p != IPPROTO_TCP) {
1176 CTR2(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP",
1177 __func__, tlsp->tid);
1178 return (EINVAL);
1179 }
1180 m->m_pkthdr.l3hlen = ip->ip_hl * 4;
1181 } else {
1182 ip6 = (struct ip6_hdr *)(eh + 1);
1183 if (ip6->ip6_nxt != IPPROTO_TCP) {
1184 CTR3(KTR_CXGBE, "%s: tid %d mbuf not IPPROTO_TCP (%u)",
1185 __func__, tlsp->tid, ip6->ip6_nxt);
1186 return (EINVAL);
1187 }
1188 m->m_pkthdr.l3hlen = sizeof(struct ip6_hdr);
1189 }
1190 if (m->m_len < m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
1191 sizeof(*tcp)) {
1192 CTR2(KTR_CXGBE, "%s: tid %d header mbuf too short (2)",
1193 __func__, tlsp->tid);
1194 return (EINVAL);
1195 }
1196 tcp = (struct tcphdr *)((char *)(eh + 1) + m->m_pkthdr.l3hlen);
1197 m->m_pkthdr.l4hlen = tcp->th_off * 4;
1198
1199 /* Bail if there is TCP payload before the TLS record. */
1200 if (m->m_len != m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen +
1201 m->m_pkthdr.l4hlen) {
1202 CTR6(KTR_CXGBE,
1203 "%s: tid %d header mbuf bad length (%d + %d + %d != %d)",
1204 __func__, tlsp->tid, m->m_pkthdr.l2hlen,
1205 m->m_pkthdr.l3hlen, m->m_pkthdr.l4hlen, m->m_len);
1206 return (EINVAL);
1207 }
1208
1209 /* Assume all headers are in 'm' for now. */
1210 MPASS(m->m_next != NULL);
1211 MPASS(m->m_next->m_flags & M_EXTPG);
1212
1213 tot_len = 0;
1214
1215 /*
1216 * Each of the remaining mbufs in the chain should reference a
1217 * TLS record.
1218 */
1219 *nsegsp = 0;
1220 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
1221 MPASS(m_tls->m_flags & M_EXTPG);
1222
1223 wr_len = ktls_wr_len(tlsp, m, m_tls, &nsegs);
1224 #ifdef VERBOSE_TRACES
1225 CTR4(KTR_CXGBE, "%s: tid %d wr_len %d nsegs %d", __func__,
1226 tlsp->tid, wr_len, nsegs);
1227 #endif
1228 if (wr_len > SGE_MAX_WR_LEN || nsegs > TX_SGL_SEGS)
1229 return (EFBIG);
1230 tot_len += roundup2(wr_len, EQ_ESIZE);
1231
1232 /*
1233 * Store 'nsegs' for the first TLS record in the
1234 * header mbuf's metadata.
1235 */
1236 if (*nsegsp == 0)
1237 *nsegsp = nsegs;
1238 }
1239
1240 MPASS(tot_len != 0);
1241
1242 /*
1243 * See if we have any TCP options or a FIN requiring a
1244 * dedicated packet.
1245 */
1246 if ((tcp->th_flags & TH_FIN) != 0 || ktls_has_tcp_options(tcp)) {
1247 wr_len = sizeof(struct fw_eth_tx_pkt_wr) +
1248 sizeof(struct cpl_tx_pkt_core) + roundup2(m->m_len, 16);
1249 if (wr_len > SGE_MAX_WR_LEN) {
1250 CTR3(KTR_CXGBE,
1251 "%s: tid %d options-only packet too long (len %d)",
1252 __func__, tlsp->tid, m->m_len);
1253 return (EINVAL);
1254 }
1255 tot_len += roundup2(wr_len, EQ_ESIZE);
1256 }
1257
1258 /* Include room for a TP work request to program an L2T entry. */
1259 tot_len += EQ_ESIZE;
1260
1261 /*
1262 * Include room for a ULPTX work request including up to 5
1263 * CPL_SET_TCB_FIELD commands before the first TLS work
1264 * request.
1265 */
1266 wr_len = sizeof(struct fw_ulptx_wr) +
1267 5 * roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1268
1269 /*
1270 * If timestamps are present, reserve 1 more command for
1271 * setting the echoed timestamp.
1272 */
1273 if (tlsp->using_timestamps)
1274 wr_len += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1275
1276 tot_len += roundup2(wr_len, EQ_ESIZE);
1277
1278 *len16p = tot_len / 16;
1279 #ifdef VERBOSE_TRACES
1280 CTR4(KTR_CXGBE, "%s: tid %d len16 %d nsegs %d", __func__,
1281 tlsp->tid, *len16p, *nsegsp);
1282 #endif
1283 return (0);
1284 }
1285
1286 /*
1287 * If the SGL ends on an address that is not 16 byte aligned, this function will
1288 * add a 0 filled flit at the end.
1289 */
1290 static void
write_gl_to_buf(struct sglist * gl,caddr_t to)1291 write_gl_to_buf(struct sglist *gl, caddr_t to)
1292 {
1293 struct sglist_seg *seg;
1294 __be64 *flitp;
1295 struct ulptx_sgl *usgl;
1296 int i, nflits, nsegs;
1297
1298 KASSERT(((uintptr_t)to & 0xf) == 0,
1299 ("%s: SGL must start at a 16 byte boundary: %p", __func__, to));
1300
1301 nsegs = gl->sg_nseg;
1302 MPASS(nsegs > 0);
1303
1304 nflits = (3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1) + 2;
1305 flitp = (__be64 *)to;
1306 seg = &gl->sg_segs[0];
1307 usgl = (void *)flitp;
1308
1309 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1310 V_ULPTX_NSGE(nsegs));
1311 usgl->len0 = htobe32(seg->ss_len);
1312 usgl->addr0 = htobe64(seg->ss_paddr);
1313 seg++;
1314
1315 for (i = 0; i < nsegs - 1; i++, seg++) {
1316 usgl->sge[i / 2].len[i & 1] = htobe32(seg->ss_len);
1317 usgl->sge[i / 2].addr[i & 1] = htobe64(seg->ss_paddr);
1318 }
1319 if (i & 1)
1320 usgl->sge[i / 2].len[1] = htobe32(0);
1321 flitp += nflits;
1322
1323 if (nflits & 1) {
1324 MPASS(((uintptr_t)flitp) & 0xf);
1325 *flitp++ = 0;
1326 }
1327
1328 MPASS((((uintptr_t)flitp) & 0xf) == 0);
1329 }
1330
1331 static inline void
copy_to_txd(struct sge_eq * eq,caddr_t from,caddr_t * to,int len)1332 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
1333 {
1334
1335 MPASS((uintptr_t)(*to) >= (uintptr_t)&eq->desc[0]);
1336 MPASS((uintptr_t)(*to) < (uintptr_t)&eq->desc[eq->sidx]);
1337
1338 if (__predict_true((uintptr_t)(*to) + len <=
1339 (uintptr_t)&eq->desc[eq->sidx])) {
1340 bcopy(from, *to, len);
1341 (*to) += len;
1342 if ((uintptr_t)(*to) == (uintptr_t)&eq->desc[eq->sidx])
1343 (*to) = (caddr_t)eq->desc;
1344 } else {
1345 int portion = (uintptr_t)&eq->desc[eq->sidx] - (uintptr_t)(*to);
1346
1347 bcopy(from, *to, portion);
1348 from += portion;
1349 portion = len - portion; /* remaining */
1350 bcopy(from, (void *)eq->desc, portion);
1351 (*to) = (caddr_t)eq->desc + portion;
1352 }
1353 }
1354
1355 static int
ktls_write_tcp_options(struct sge_txq * txq,void * dst,struct mbuf * m,u_int available,u_int pidx)1356 ktls_write_tcp_options(struct sge_txq *txq, void *dst, struct mbuf *m,
1357 u_int available, u_int pidx)
1358 {
1359 struct tx_sdesc *txsd;
1360 struct fw_eth_tx_pkt_wr *wr;
1361 struct cpl_tx_pkt_core *cpl;
1362 uint32_t ctrl;
1363 uint64_t ctrl1;
1364 int len16, ndesc, pktlen;
1365 struct ether_header *eh;
1366 struct ip *ip, newip;
1367 struct ip6_hdr *ip6, newip6;
1368 struct tcphdr *tcp, newtcp;
1369 caddr_t out;
1370
1371 TXQ_LOCK_ASSERT_OWNED(txq);
1372 M_ASSERTPKTHDR(m);
1373
1374 wr = dst;
1375 pktlen = m->m_len;
1376 ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen;
1377 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16);
1378 ndesc = tx_len16_to_desc(len16);
1379 MPASS(ndesc <= available);
1380
1381 /* Firmware work request header */
1382 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1383 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
1384
1385 ctrl = V_FW_WR_LEN16(len16);
1386 wr->equiq_to_len16 = htobe32(ctrl);
1387 wr->r3 = 0;
1388
1389 cpl = (void *)(wr + 1);
1390
1391 /* CPL header */
1392 cpl->ctrl0 = txq->cpl_ctrl0;
1393 cpl->pack = 0;
1394 cpl->len = htobe16(pktlen);
1395
1396 out = (void *)(cpl + 1);
1397
1398 /* Copy over Ethernet header. */
1399 eh = mtod(m, struct ether_header *);
1400 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1401
1402 /* Fixup length in IP header and copy out. */
1403 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1404 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1405 newip = *ip;
1406 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1407 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1408 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1409 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1410 m->m_pkthdr.l3hlen - sizeof(*ip));
1411 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) |
1412 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1413 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1414 } else {
1415 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1416 newip6 = *ip6;
1417 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
1418 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1419 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1420 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) |
1421 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1422 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1423 }
1424 cpl->ctrl1 = htobe64(ctrl1);
1425 txq->txcsum++;
1426
1427 /* Clear PUSH and FIN in the TCP header if present. */
1428 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1429 newtcp = *tcp;
1430 newtcp.th_flags &= ~(TH_PUSH | TH_FIN);
1431 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1432
1433 /* Copy rest of packet. */
1434 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, pktlen -
1435 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1436 txq->imm_wrs++;
1437
1438 txq->txpkt_wrs++;
1439
1440 txq->kern_tls_options++;
1441
1442 txsd = &txq->sdesc[pidx];
1443 txsd->m = NULL;
1444 txsd->desc_used = ndesc;
1445
1446 return (ndesc);
1447 }
1448
1449 static int
ktls_write_tunnel_packet(struct sge_txq * txq,void * dst,struct mbuf * m,struct mbuf * m_tls,u_int available,tcp_seq tcp_seqno,u_int pidx)1450 ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
1451 struct mbuf *m_tls, u_int available, tcp_seq tcp_seqno, u_int pidx)
1452 {
1453 struct tx_sdesc *txsd;
1454 struct fw_eth_tx_pkt_wr *wr;
1455 struct cpl_tx_pkt_core *cpl;
1456 uint32_t ctrl;
1457 uint64_t ctrl1;
1458 int len16, ndesc, pktlen;
1459 struct ether_header *eh;
1460 struct ip *ip, newip;
1461 struct ip6_hdr *ip6, newip6;
1462 struct tcphdr *tcp, newtcp;
1463 caddr_t out;
1464
1465 TXQ_LOCK_ASSERT_OWNED(txq);
1466 M_ASSERTPKTHDR(m);
1467
1468 /* Locate the template TLS header. */
1469 M_ASSERTEXTPG(m_tls);
1470
1471 /* This should always be the last TLS record in a chain. */
1472 MPASS(m_tls->m_next == NULL);
1473
1474 wr = dst;
1475 pktlen = m->m_len + m_tls->m_len;
1476 ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen;
1477 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16);
1478 ndesc = tx_len16_to_desc(len16);
1479 MPASS(ndesc <= available);
1480
1481 /* Firmware work request header */
1482 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
1483 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
1484
1485 ctrl = V_FW_WR_LEN16(len16);
1486 wr->equiq_to_len16 = htobe32(ctrl);
1487 wr->r3 = 0;
1488
1489 cpl = (void *)(wr + 1);
1490
1491 /* CPL header */
1492 cpl->ctrl0 = txq->cpl_ctrl0;
1493 cpl->pack = 0;
1494 cpl->len = htobe16(pktlen);
1495
1496 out = (void *)(cpl + 1);
1497
1498 /* Copy over Ethernet header. */
1499 eh = mtod(m, struct ether_header *);
1500 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
1501
1502 /* Fixup length in IP header and copy out. */
1503 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
1504 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1505 newip = *ip;
1506 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
1507 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
1508 if (m->m_pkthdr.l3hlen > sizeof(*ip))
1509 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
1510 m->m_pkthdr.l3hlen - sizeof(*ip));
1511 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) |
1512 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1513 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1514 } else {
1515 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
1516 newip6 = *ip6;
1517 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
1518 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
1519 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
1520 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) |
1521 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
1522 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
1523 }
1524 cpl->ctrl1 = htobe64(ctrl1);
1525 txq->txcsum++;
1526
1527 /* Set sequence number in TCP header. */
1528 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
1529 newtcp = *tcp;
1530 newtcp.th_seq = htonl(tcp_seqno + mtod(m_tls, vm_offset_t));
1531 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
1532
1533 /* Copy rest of TCP header. */
1534 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len -
1535 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
1536
1537 /* Copy the subset of the TLS header requested. */
1538 copy_to_txd(&txq->eq, (char *)m_tls->m_epg_hdr +
1539 mtod(m_tls, vm_offset_t), &out, m_tls->m_len);
1540 txq->imm_wrs++;
1541
1542 txq->txpkt_wrs++;
1543
1544 txq->kern_tls_header++;
1545
1546 txsd = &txq->sdesc[pidx];
1547 txsd->m = m;
1548 txsd->desc_used = ndesc;
1549
1550 return (ndesc);
1551 }
1552
1553 _Static_assert(sizeof(struct cpl_set_tcb_field) <= EQ_ESIZE,
1554 "CPL_SET_TCB_FIELD must be smaller than a single TX descriptor");
1555 _Static_assert(W_TCB_SND_UNA_RAW == W_TCB_SND_NXT_RAW,
1556 "SND_NXT_RAW and SND_UNA_RAW are in different words");
1557
1558 static int
ktls_write_tls_wr(struct tlspcb * tlsp,struct sge_txq * txq,void * dst,struct mbuf * m,struct tcphdr * tcp,struct mbuf * m_tls,u_int nsegs,u_int available,tcp_seq tcp_seqno,uint32_t * tsopt,u_int pidx,bool set_l2t_idx)1559 ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
1560 void *dst, struct mbuf *m, struct tcphdr *tcp, struct mbuf *m_tls,
1561 u_int nsegs, u_int available, tcp_seq tcp_seqno, uint32_t *tsopt,
1562 u_int pidx, bool set_l2t_idx)
1563 {
1564 struct sge_eq *eq = &txq->eq;
1565 struct tx_sdesc *txsd;
1566 struct fw_ulptx_wr *wr;
1567 struct ulp_txpkt *txpkt;
1568 struct ulptx_sc_memrd *memrd;
1569 struct ulptx_idata *idata;
1570 struct cpl_tx_sec_pdu *sec_pdu;
1571 struct cpl_tx_data *tx_data;
1572 struct tls_record_layer *hdr;
1573 char *iv, *out;
1574 u_int aad_start, aad_stop;
1575 u_int auth_start, auth_stop, auth_insert;
1576 u_int cipher_start, cipher_stop, iv_offset;
1577 u_int imm_len, mss, ndesc, offset, plen, tlen, twr_len, wr_len;
1578 u_int fields, tx_max_offset, tx_max;
1579 bool first_wr, last_wr, using_scratch;
1580
1581 ndesc = 0;
1582 MPASS(tlsp->txq == txq);
1583
1584 first_wr = (tlsp->prev_seq == 0 && tlsp->prev_ack == 0 &&
1585 tlsp->prev_win == 0);
1586
1587 /*
1588 * Use the per-txq scratch pad if near the end of the ring to
1589 * simplify handling of wrap-around. This uses a simple but
1590 * not quite perfect test of using the scratch buffer if we
1591 * can't fit a maximal work request in without wrapping.
1592 */
1593 using_scratch = (eq->sidx - pidx < SGE_MAX_WR_LEN / EQ_ESIZE);
1594
1595 /* Locate the TLS header. */
1596 M_ASSERTEXTPG(m_tls);
1597 hdr = (void *)m_tls->m_epg_hdr;
1598 plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_epg_trllen;
1599
1600 /* Determine how much of the TLS record to send. */
1601 tlen = ktls_tcp_payload_length(tlsp, m_tls);
1602 if (tlen <= m_tls->m_epg_hdrlen) {
1603 /*
1604 * For requests that only want to send the TLS header,
1605 * send a tunnelled packet as immediate data.
1606 */
1607 #ifdef VERBOSE_TRACES
1608 CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u",
1609 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno);
1610 #endif
1611 return (ktls_write_tunnel_packet(txq, dst, m, m_tls, available,
1612 tcp_seqno, pidx));
1613 }
1614 if (tlen < plen) {
1615 plen = tlen;
1616 offset = ktls_payload_offset(tlsp, m_tls);
1617 #ifdef VERBOSE_TRACES
1618 CTR4(KTR_CXGBE, "%s: tid %d short TLS record %u with offset %u",
1619 __func__, tlsp->tid, (u_int)m_tls->m_epg_seqno, offset);
1620 #endif
1621 if (m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) != 0) {
1622 txq->kern_tls_fin_short++;
1623 #ifdef INVARIANTS
1624 panic("%s: FIN on short TLS record", __func__);
1625 #endif
1626 }
1627 } else
1628 offset = 0;
1629
1630 /*
1631 * This is the last work request for a given TLS mbuf chain if
1632 * it is the last mbuf in the chain and FIN is not set. If
1633 * FIN is set, then ktls_write_tcp_fin() will write out the
1634 * last work request.
1635 */
1636 last_wr = m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) == 0;
1637
1638 /*
1639 * The host stack may ask us to not send part of the start of
1640 * a TLS record. (For example, the stack might have
1641 * previously sent a "short" TLS record and might later send
1642 * down an mbuf that requests to send the remainder of the TLS
1643 * record.) The crypto engine must process a TLS record from
1644 * the beginning if computing a GCM tag or HMAC, so we always
1645 * send the TLS record from the beginning as input to the
1646 * crypto engine and via CPL_TX_DATA to TP. However, TP will
1647 * drop individual packets after they have been chopped up
1648 * into MSS-sized chunks if the entire sequence range of those
1649 * packets is less than SND_UNA. SND_UNA is computed as
1650 * TX_MAX - SND_UNA_RAW. Thus, use the offset stored in
1651 * m_data to set TX_MAX to the first byte in the TCP sequence
1652 * space the host actually wants us to send and set
1653 * SND_UNA_RAW to 0.
1654 *
1655 * If the host sends us back to back requests that span the
1656 * trailer of a single TLS record (first request ends "in" the
1657 * trailer and second request starts at the next byte but
1658 * still "in" the trailer), the initial bytes of the trailer
1659 * that the first request drops will not be retransmitted. If
1660 * the host uses the same requests when retransmitting the
1661 * connection will hang. To handle this, always transmit the
1662 * full trailer for a request that begins "in" the trailer
1663 * (the second request in the example above). This should
1664 * also help to avoid retransmits for the common case.
1665 *
1666 * A similar condition exists when using CBC for back to back
1667 * requests that span a single AES block. The first request
1668 * will be truncated to end at the end of the previous AES
1669 * block. To handle this, always begin transmission at the
1670 * start of the current AES block.
1671 */
1672 tx_max_offset = mtod(m_tls, vm_offset_t);
1673 if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
1674 m_tls->m_epg_trllen) {
1675 /* Always send the full trailer. */
1676 tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
1677 m_tls->m_epg_trllen;
1678 }
1679 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&
1680 tx_max_offset > TLS_HEADER_LENGTH) {
1681 /* Always send all of the first AES block. */
1682 tx_max_offset = TLS_HEADER_LENGTH +
1683 rounddown(tx_max_offset - TLS_HEADER_LENGTH,
1684 AES_BLOCK_LEN);
1685 }
1686 tx_max = tcp_seqno + tx_max_offset;
1687
1688 /*
1689 * Update TCB fields. Reserve space for the FW_ULPTX_WR header
1690 * but don't populate it until we know how many field updates
1691 * are required.
1692 */
1693 if (using_scratch)
1694 wr = (void *)txq->ss;
1695 else
1696 wr = dst;
1697 out = (void *)(wr + 1);
1698 fields = 0;
1699 if (set_l2t_idx) {
1700 KASSERT(nsegs != 0,
1701 ("trying to set L2T_IX for subsequent TLS WR"));
1702 #ifdef VERBOSE_TRACES
1703 CTR3(KTR_CXGBE, "%s: tid %d set L2T_IX to %d", __func__,
1704 tlsp->tid, tlsp->l2te->idx);
1705 #endif
1706 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_L2T_IX,
1707 V_TCB_L2T_IX(M_TCB_L2T_IX), V_TCB_L2T_IX(tlsp->l2te->idx));
1708 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1709 fields++;
1710 }
1711 if (tsopt != NULL && tlsp->prev_tsecr != ntohl(tsopt[1])) {
1712 KASSERT(nsegs != 0,
1713 ("trying to set T_RTSEQ_RECENT for subsequent TLS WR"));
1714 #ifdef VERBOSE_TRACES
1715 CTR2(KTR_CXGBE, "%s: tid %d wrote updated T_RTSEQ_RECENT",
1716 __func__, tlsp->tid);
1717 #endif
1718 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_T_RTSEQ_RECENT,
1719 V_TCB_T_RTSEQ_RECENT(M_TCB_T_RTSEQ_RECENT),
1720 V_TCB_T_RTSEQ_RECENT(ntohl(tsopt[1])));
1721 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1722 fields++;
1723
1724 tlsp->prev_tsecr = ntohl(tsopt[1]);
1725 }
1726
1727 if (first_wr || tlsp->prev_seq != tx_max) {
1728 KASSERT(nsegs != 0,
1729 ("trying to set TX_MAX for subsequent TLS WR"));
1730 #ifdef VERBOSE_TRACES
1731 CTR4(KTR_CXGBE,
1732 "%s: tid %d setting TX_MAX to %u (tcp_seqno %u)",
1733 __func__, tlsp->tid, tx_max, tcp_seqno);
1734 #endif
1735 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_TX_MAX,
1736 V_TCB_TX_MAX(M_TCB_TX_MAX), V_TCB_TX_MAX(tx_max));
1737 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1738 fields++;
1739 }
1740
1741 /*
1742 * If there is data to drop at the beginning of this TLS
1743 * record or if this is a retransmit,
1744 * reset SND_UNA_RAW to 0 so that SND_UNA == TX_MAX.
1745 */
1746 if (tlsp->prev_seq != tx_max || mtod(m_tls, vm_offset_t) != 0) {
1747 KASSERT(nsegs != 0,
1748 ("trying to clear SND_UNA_RAW for subsequent TLS WR"));
1749 #ifdef VERBOSE_TRACES
1750 CTR2(KTR_CXGBE, "%s: tid %d clearing SND_UNA_RAW", __func__,
1751 tlsp->tid);
1752 #endif
1753 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_SND_UNA_RAW,
1754 V_TCB_SND_UNA_RAW(M_TCB_SND_UNA_RAW),
1755 V_TCB_SND_UNA_RAW(0));
1756 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1757 fields++;
1758 }
1759
1760 /*
1761 * Store the expected sequence number of the next byte after
1762 * this record.
1763 */
1764 tlsp->prev_seq = tcp_seqno + tlen;
1765
1766 if (first_wr || tlsp->prev_ack != ntohl(tcp->th_ack)) {
1767 KASSERT(nsegs != 0,
1768 ("trying to set RCV_NXT for subsequent TLS WR"));
1769 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_NXT,
1770 V_TCB_RCV_NXT(M_TCB_RCV_NXT),
1771 V_TCB_RCV_NXT(ntohl(tcp->th_ack)));
1772 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1773 fields++;
1774
1775 tlsp->prev_ack = ntohl(tcp->th_ack);
1776 }
1777
1778 if (first_wr || tlsp->prev_win != ntohs(tcp->th_win)) {
1779 KASSERT(nsegs != 0,
1780 ("trying to set RCV_WND for subsequent TLS WR"));
1781 write_set_tcb_field_ulp(tlsp, out, txq, W_TCB_RCV_WND,
1782 V_TCB_RCV_WND(M_TCB_RCV_WND),
1783 V_TCB_RCV_WND(ntohs(tcp->th_win)));
1784 out += roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1785 fields++;
1786
1787 tlsp->prev_win = ntohs(tcp->th_win);
1788 }
1789
1790 /* Recalculate 'nsegs' if cached value is not available. */
1791 if (nsegs == 0)
1792 nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_epg_hdrlen +
1793 offset, plen - (m_tls->m_epg_hdrlen + offset));
1794
1795 /* Calculate the size of the TLS work request. */
1796 twr_len = ktls_base_wr_size(tlsp);
1797
1798 imm_len = 0;
1799 if (offset == 0)
1800 imm_len += m_tls->m_epg_hdrlen;
1801 if (plen == tlen)
1802 imm_len += AES_BLOCK_LEN;
1803 twr_len += roundup2(imm_len, 16);
1804 twr_len += ktls_sgl_size(nsegs);
1805
1806 /*
1807 * If any field updates were required, determine if they can
1808 * be included in the TLS work request. If not, use the
1809 * FW_ULPTX_WR work request header at 'wr' as a dedicated work
1810 * request for the field updates and start a new work request
1811 * for the TLS work request afterward.
1812 */
1813 if (fields != 0) {
1814 wr_len = fields * roundup2(LEN__SET_TCB_FIELD_ULP, 16);
1815 if (twr_len + wr_len <= SGE_MAX_WR_LEN &&
1816 tlsp->sc->tlst.combo_wrs) {
1817 wr_len += twr_len;
1818 txpkt = (void *)out;
1819 } else {
1820 wr_len += sizeof(*wr);
1821 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
1822 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
1823 V_FW_WR_LEN16(wr_len / 16));
1824 wr->cookie = 0;
1825
1826 /*
1827 * If we were using scratch space, copy the
1828 * field updates work request to the ring.
1829 */
1830 if (using_scratch) {
1831 out = dst;
1832 copy_to_txd(eq, txq->ss, &out, wr_len);
1833 }
1834
1835 ndesc = howmany(wr_len, EQ_ESIZE);
1836 MPASS(ndesc <= available);
1837
1838 txq->raw_wrs++;
1839 txsd = &txq->sdesc[pidx];
1840 txsd->m = NULL;
1841 txsd->desc_used = ndesc;
1842 IDXINCR(pidx, ndesc, eq->sidx);
1843 dst = &eq->desc[pidx];
1844
1845 /*
1846 * Determine if we should use scratch space
1847 * for the TLS work request based on the
1848 * available space after advancing pidx for
1849 * the field updates work request.
1850 */
1851 wr_len = twr_len;
1852 using_scratch = (eq->sidx - pidx <
1853 howmany(wr_len, EQ_ESIZE));
1854 if (using_scratch)
1855 wr = (void *)txq->ss;
1856 else
1857 wr = dst;
1858 txpkt = (void *)(wr + 1);
1859 }
1860 } else {
1861 wr_len = twr_len;
1862 txpkt = (void *)out;
1863 }
1864
1865 wr_len = roundup2(wr_len, 16);
1866 MPASS(ndesc + howmany(wr_len, EQ_ESIZE) <= available);
1867
1868 /* FW_ULPTX_WR */
1869 wr->op_to_compl = htobe32(V_FW_WR_OP(FW_ULPTX_WR));
1870 wr->flowid_len16 = htobe32(F_FW_ULPTX_WR_DATA |
1871 V_FW_WR_LEN16(wr_len / 16));
1872 wr->cookie = 0;
1873
1874 /* ULP_TXPKT */
1875 txpkt->cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
1876 V_ULP_TXPKT_DATAMODIFY(0) |
1877 V_ULP_TXPKT_CHANNELID(tlsp->vi->pi->port_id) | V_ULP_TXPKT_DEST(0) |
1878 V_ULP_TXPKT_FID(txq->eq.cntxt_id) | V_ULP_TXPKT_RO(1));
1879 txpkt->len = htobe32(howmany(twr_len - sizeof(*wr), 16));
1880
1881 /* ULPTX_IDATA sub-command */
1882 idata = (void *)(txpkt + 1);
1883 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1884 V_ULP_TX_SC_MORE(1));
1885 idata->len = sizeof(struct cpl_tx_sec_pdu);
1886
1887 /*
1888 * The key context, CPL_TX_DATA, and immediate data are part
1889 * of this ULPTX_IDATA when using an inline key. When reading
1890 * the key from memory, the CPL_TX_DATA and immediate data are
1891 * part of a separate ULPTX_IDATA.
1892 */
1893 if (tlsp->inline_key)
1894 idata->len += tlsp->tx_key_info_size +
1895 sizeof(struct cpl_tx_data) + imm_len;
1896 idata->len = htobe32(idata->len);
1897
1898 /* CPL_TX_SEC_PDU */
1899 sec_pdu = (void *)(idata + 1);
1900
1901 /*
1902 * For short records, AAD is counted as header data in SCMD0,
1903 * the IV is next followed by a cipher region for the payload.
1904 */
1905 if (plen == tlen) {
1906 aad_start = 0;
1907 aad_stop = 0;
1908 iv_offset = 1;
1909 auth_start = 0;
1910 auth_stop = 0;
1911 auth_insert = 0;
1912 cipher_start = AES_BLOCK_LEN + 1;
1913 cipher_stop = 0;
1914
1915 sec_pdu->pldlen = htobe32(16 + plen -
1916 (m_tls->m_epg_hdrlen + offset));
1917
1918 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1919 sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs;
1920 sec_pdu->ivgen_hdrlen = htobe32(
1921 tlsp->scmd0_short.ivgen_hdrlen |
1922 V_SCMD_HDR_LEN(offset == 0 ? m_tls->m_epg_hdrlen : 0));
1923
1924 txq->kern_tls_short++;
1925 } else {
1926 /*
1927 * AAD is TLS header. IV is after AAD. The cipher region
1928 * starts after the IV. See comments in ccr_authenc() and
1929 * ccr_gmac() in t4_crypto.c regarding cipher and auth
1930 * start/stop values.
1931 */
1932 aad_start = 1;
1933 aad_stop = TLS_HEADER_LENGTH;
1934 iv_offset = TLS_HEADER_LENGTH + 1;
1935 cipher_start = m_tls->m_epg_hdrlen + 1;
1936 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
1937 cipher_stop = 0;
1938 auth_start = cipher_start;
1939 auth_stop = 0;
1940 auth_insert = 0;
1941 } else {
1942 cipher_stop = 0;
1943 auth_start = cipher_start;
1944 auth_stop = 0;
1945 auth_insert = 0;
1946 }
1947
1948 sec_pdu->pldlen = htobe32(plen);
1949
1950 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1951 sec_pdu->seqno_numivs = tlsp->scmd0.seqno_numivs;
1952 sec_pdu->ivgen_hdrlen = tlsp->scmd0.ivgen_hdrlen;
1953
1954 if (mtod(m_tls, vm_offset_t) == 0)
1955 txq->kern_tls_full++;
1956 else
1957 txq->kern_tls_partial++;
1958 }
1959 sec_pdu->op_ivinsrtofst = htobe32(
1960 V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
1961 V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
1962 V_CPL_TX_SEC_PDU_IVINSRTOFST(iv_offset));
1963 sec_pdu->aadstart_cipherstop_hi = htobe32(
1964 V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
1965 V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
1966 V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
1967 V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
1968 sec_pdu->cipherstop_lo_authinsert = htobe32(
1969 V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
1970 V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
1971 V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
1972 V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
1973
1974 sec_pdu->scmd1 = htobe64(m_tls->m_epg_seqno);
1975
1976 /* Key context */
1977 out = (void *)(sec_pdu + 1);
1978 if (tlsp->inline_key) {
1979 memcpy(out, &tlsp->keyctx, tlsp->tx_key_info_size);
1980 out += tlsp->tx_key_info_size;
1981 } else {
1982 /* ULPTX_SC_MEMRD to read key context. */
1983 memrd = (void *)out;
1984 memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
1985 V_ULP_TX_SC_MORE(1) |
1986 V_ULPTX_LEN16(tlsp->tx_key_info_size >> 4));
1987 memrd->addr = htobe32(tlsp->tx_key_addr >> 5);
1988
1989 /* ULPTX_IDATA for CPL_TX_DATA and TLS header. */
1990 idata = (void *)(memrd + 1);
1991 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
1992 V_ULP_TX_SC_MORE(1));
1993 idata->len = htobe32(sizeof(struct cpl_tx_data) + imm_len);
1994
1995 out = (void *)(idata + 1);
1996 }
1997
1998 /* CPL_TX_DATA */
1999 tx_data = (void *)out;
2000 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tlsp->tid));
2001 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
2002 mss = m->m_pkthdr.tso_segsz;
2003 tlsp->prev_mss = mss;
2004 } else if (tlsp->prev_mss != 0)
2005 mss = tlsp->prev_mss;
2006 else
2007 mss = tlsp->vi->ifp->if_mtu -
2008 (m->m_pkthdr.l3hlen + m->m_pkthdr.l4hlen);
2009 if (offset == 0) {
2010 tx_data->len = htobe32(V_TX_DATA_MSS(mss) | V_TX_LENGTH(tlen));
2011 tx_data->rsvd = htobe32(tcp_seqno);
2012 } else {
2013 tx_data->len = htobe32(V_TX_DATA_MSS(mss) |
2014 V_TX_LENGTH(tlen - (m_tls->m_epg_hdrlen + offset)));
2015 tx_data->rsvd = htobe32(tcp_seqno + m_tls->m_epg_hdrlen + offset);
2016 }
2017 tx_data->flags = htobe32(F_TX_BYPASS);
2018 if (last_wr && tcp->th_flags & TH_PUSH)
2019 tx_data->flags |= htobe32(F_TX_PUSH | F_TX_SHOVE);
2020
2021 /* Populate the TLS header */
2022 out = (void *)(tx_data + 1);
2023 if (offset == 0) {
2024 memcpy(out, m_tls->m_epg_hdr, m_tls->m_epg_hdrlen);
2025 out += m_tls->m_epg_hdrlen;
2026 }
2027
2028 /* AES IV for a short record. */
2029 if (plen == tlen) {
2030 iv = out;
2031 if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
2032 memcpy(iv, tlsp->keyctx.txhdr.txsalt, SALT_SIZE);
2033 memcpy(iv + 4, hdr + 1, 8);
2034 *(uint32_t *)(iv + 12) = htobe32(2 +
2035 offset / AES_BLOCK_LEN);
2036 } else
2037 memcpy(iv, hdr + 1, AES_BLOCK_LEN);
2038 out += AES_BLOCK_LEN;
2039 }
2040
2041 if (imm_len % 16 != 0) {
2042 /* Zero pad to an 8-byte boundary. */
2043 memset(out, 0, 8 - (imm_len % 8));
2044 out += 8 - (imm_len % 8);
2045
2046 /*
2047 * Insert a ULP_TX_SC_NOOP if needed so the SGL is
2048 * 16-byte aligned.
2049 */
2050 if (imm_len % 16 <= 8) {
2051 idata = (void *)out;
2052 idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
2053 idata->len = htobe32(0);
2054 out = (void *)(idata + 1);
2055 }
2056 }
2057
2058 /* SGL for record payload */
2059 sglist_reset(txq->gl);
2060 if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_epg_hdrlen + offset,
2061 plen - (m_tls->m_epg_hdrlen + offset)) != 0) {
2062 #ifdef INVARIANTS
2063 panic("%s: failed to append sglist", __func__);
2064 #endif
2065 }
2066 write_gl_to_buf(txq->gl, out);
2067
2068 if (using_scratch) {
2069 out = dst;
2070 copy_to_txd(eq, txq->ss, &out, wr_len);
2071 }
2072
2073 ndesc += howmany(wr_len, EQ_ESIZE);
2074 MPASS(ndesc <= available);
2075
2076 txq->kern_tls_records++;
2077 txq->kern_tls_octets += tlen - mtod(m_tls, vm_offset_t);
2078 if (mtod(m_tls, vm_offset_t) != 0) {
2079 if (offset == 0)
2080 txq->kern_tls_waste += mtod(m_tls, vm_offset_t);
2081 else
2082 txq->kern_tls_waste += mtod(m_tls, vm_offset_t) -
2083 (m_tls->m_epg_hdrlen + offset);
2084 }
2085
2086 txsd = &txq->sdesc[pidx];
2087 if (last_wr)
2088 txsd->m = m;
2089 else
2090 txsd->m = NULL;
2091 txsd->desc_used = howmany(wr_len, EQ_ESIZE);
2092
2093 return (ndesc);
2094 }
2095
2096 static int
ktls_write_tcp_fin(struct sge_txq * txq,void * dst,struct mbuf * m,u_int available,tcp_seq tcp_seqno,u_int pidx)2097 ktls_write_tcp_fin(struct sge_txq *txq, void *dst, struct mbuf *m,
2098 u_int available, tcp_seq tcp_seqno, u_int pidx)
2099 {
2100 struct tx_sdesc *txsd;
2101 struct fw_eth_tx_pkt_wr *wr;
2102 struct cpl_tx_pkt_core *cpl;
2103 uint32_t ctrl;
2104 uint64_t ctrl1;
2105 int len16, ndesc, pktlen;
2106 struct ether_header *eh;
2107 struct ip *ip, newip;
2108 struct ip6_hdr *ip6, newip6;
2109 struct tcphdr *tcp, newtcp;
2110 caddr_t out;
2111
2112 TXQ_LOCK_ASSERT_OWNED(txq);
2113 M_ASSERTPKTHDR(m);
2114
2115 wr = dst;
2116 pktlen = m->m_len;
2117 ctrl = sizeof(struct cpl_tx_pkt_core) + pktlen;
2118 len16 = howmany(sizeof(struct fw_eth_tx_pkt_wr) + ctrl, 16);
2119 ndesc = tx_len16_to_desc(len16);
2120 MPASS(ndesc <= available);
2121
2122 /* Firmware work request header */
2123 wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2124 V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
2125
2126 ctrl = V_FW_WR_LEN16(len16);
2127 wr->equiq_to_len16 = htobe32(ctrl);
2128 wr->r3 = 0;
2129
2130 cpl = (void *)(wr + 1);
2131
2132 /* CPL header */
2133 cpl->ctrl0 = txq->cpl_ctrl0;
2134 cpl->pack = 0;
2135 cpl->len = htobe16(pktlen);
2136
2137 out = (void *)(cpl + 1);
2138
2139 /* Copy over Ethernet header. */
2140 eh = mtod(m, struct ether_header *);
2141 copy_to_txd(&txq->eq, (caddr_t)eh, &out, m->m_pkthdr.l2hlen);
2142
2143 /* Fixup length in IP header and copy out. */
2144 if (ntohs(eh->ether_type) == ETHERTYPE_IP) {
2145 ip = (void *)((char *)eh + m->m_pkthdr.l2hlen);
2146 newip = *ip;
2147 newip.ip_len = htons(pktlen - m->m_pkthdr.l2hlen);
2148 copy_to_txd(&txq->eq, (caddr_t)&newip, &out, sizeof(newip));
2149 if (m->m_pkthdr.l3hlen > sizeof(*ip))
2150 copy_to_txd(&txq->eq, (caddr_t)(ip + 1), &out,
2151 m->m_pkthdr.l3hlen - sizeof(*ip));
2152 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP) |
2153 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
2154 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
2155 } else {
2156 ip6 = (void *)((char *)eh + m->m_pkthdr.l2hlen);
2157 newip6 = *ip6;
2158 newip6.ip6_plen = htons(pktlen - m->m_pkthdr.l2hlen);
2159 copy_to_txd(&txq->eq, (caddr_t)&newip6, &out, sizeof(newip6));
2160 MPASS(m->m_pkthdr.l3hlen == sizeof(*ip6));
2161 ctrl1 = V_TXPKT_CSUM_TYPE(TX_CSUM_TCPIP6) |
2162 V_T6_TXPKT_ETHHDR_LEN(m->m_pkthdr.l2hlen - ETHER_HDR_LEN) |
2163 V_TXPKT_IPHDR_LEN(m->m_pkthdr.l3hlen);
2164 }
2165 cpl->ctrl1 = htobe64(ctrl1);
2166 txq->txcsum++;
2167
2168 /* Set sequence number in TCP header. */
2169 tcp = (void *)((char *)eh + m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen);
2170 newtcp = *tcp;
2171 newtcp.th_seq = htonl(tcp_seqno);
2172 copy_to_txd(&txq->eq, (caddr_t)&newtcp, &out, sizeof(newtcp));
2173
2174 /* Copy rest of packet. */
2175 copy_to_txd(&txq->eq, (caddr_t)(tcp + 1), &out, m->m_len -
2176 (m->m_pkthdr.l2hlen + m->m_pkthdr.l3hlen + sizeof(*tcp)));
2177 txq->imm_wrs++;
2178
2179 txq->txpkt_wrs++;
2180
2181 txq->kern_tls_fin++;
2182
2183 txsd = &txq->sdesc[pidx];
2184 txsd->m = m;
2185 txsd->desc_used = ndesc;
2186
2187 return (ndesc);
2188 }
2189
2190 int
t6_ktls_write_wr(struct sge_txq * txq,void * dst,struct mbuf * m,u_int nsegs,u_int available)2191 t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs,
2192 u_int available)
2193 {
2194 struct sge_eq *eq = &txq->eq;
2195 struct tx_sdesc *txsd;
2196 struct tlspcb *tlsp;
2197 struct tcphdr *tcp;
2198 struct mbuf *m_tls;
2199 struct ether_header *eh;
2200 tcp_seq tcp_seqno;
2201 u_int ndesc, pidx, totdesc;
2202 uint16_t vlan_tag;
2203 bool has_fin, set_l2t_idx;
2204 void *tsopt;
2205
2206 M_ASSERTPKTHDR(m);
2207 MPASS(m->m_pkthdr.snd_tag != NULL);
2208 tlsp = mst_to_tls(m->m_pkthdr.snd_tag);
2209
2210 totdesc = 0;
2211 eh = mtod(m, struct ether_header *);
2212 tcp = (struct tcphdr *)((char *)eh + m->m_pkthdr.l2hlen +
2213 m->m_pkthdr.l3hlen);
2214 pidx = eq->pidx;
2215 has_fin = (tcp->th_flags & TH_FIN) != 0;
2216
2217 /*
2218 * If this TLS record has a FIN, then we will send any
2219 * requested options as part of the FIN packet.
2220 */
2221 if (!has_fin && ktls_has_tcp_options(tcp)) {
2222 ndesc = ktls_write_tcp_options(txq, dst, m, available, pidx);
2223 totdesc += ndesc;
2224 IDXINCR(pidx, ndesc, eq->sidx);
2225 dst = &eq->desc[pidx];
2226 #ifdef VERBOSE_TRACES
2227 CTR2(KTR_CXGBE, "%s: tid %d wrote TCP options packet", __func__,
2228 tlsp->tid);
2229 #endif
2230 }
2231
2232 /*
2233 * Allocate a new L2T entry if necessary. This may write out
2234 * a work request to the txq.
2235 */
2236 if (m->m_flags & M_VLANTAG)
2237 vlan_tag = m->m_pkthdr.ether_vtag;
2238 else
2239 vlan_tag = 0xfff;
2240 set_l2t_idx = false;
2241 if (tlsp->l2te == NULL || tlsp->l2te->vlan != vlan_tag ||
2242 memcmp(tlsp->l2te->dmac, eh->ether_dhost, ETHER_ADDR_LEN) != 0) {
2243 set_l2t_idx = true;
2244 if (tlsp->l2te)
2245 t4_l2t_release(tlsp->l2te);
2246 tlsp->l2te = t4_l2t_alloc_tls(tlsp->sc, txq, dst, &ndesc,
2247 vlan_tag, tlsp->vi->pi->lport, eh->ether_dhost);
2248 if (tlsp->l2te == NULL)
2249 CXGBE_UNIMPLEMENTED("failed to allocate TLS L2TE");
2250 if (ndesc != 0) {
2251 MPASS(ndesc <= available - totdesc);
2252
2253 txq->raw_wrs++;
2254 txsd = &txq->sdesc[pidx];
2255 txsd->m = NULL;
2256 txsd->desc_used = ndesc;
2257 totdesc += ndesc;
2258 IDXINCR(pidx, ndesc, eq->sidx);
2259 dst = &eq->desc[pidx];
2260 }
2261 }
2262
2263 /*
2264 * Iterate over each TLS record constructing a work request
2265 * for that record.
2266 */
2267 for (m_tls = m->m_next; m_tls != NULL; m_tls = m_tls->m_next) {
2268 MPASS(m_tls->m_flags & M_EXTPG);
2269
2270 /*
2271 * Determine the initial TCP sequence number for this
2272 * record.
2273 */
2274 tsopt = NULL;
2275 if (m_tls == m->m_next) {
2276 tcp_seqno = ntohl(tcp->th_seq) -
2277 mtod(m_tls, vm_offset_t);
2278 if (tlsp->using_timestamps)
2279 tsopt = ktls_find_tcp_timestamps(tcp);
2280 } else {
2281 MPASS(mtod(m_tls, vm_offset_t) == 0);
2282 tcp_seqno = tlsp->prev_seq;
2283 }
2284
2285 ndesc = ktls_write_tls_wr(tlsp, txq, dst, m, tcp, m_tls,
2286 nsegs, available - totdesc, tcp_seqno, tsopt, pidx,
2287 set_l2t_idx);
2288 totdesc += ndesc;
2289 IDXINCR(pidx, ndesc, eq->sidx);
2290 dst = &eq->desc[pidx];
2291
2292 /*
2293 * The value of nsegs from the header mbuf's metadata
2294 * is only valid for the first TLS record.
2295 */
2296 nsegs = 0;
2297
2298 /* Only need to set the L2T index once. */
2299 set_l2t_idx = false;
2300 }
2301
2302 if (has_fin) {
2303 /*
2304 * If the TCP header for this chain has FIN sent, then
2305 * explicitly send a packet that has FIN set. This
2306 * will also have PUSH set if requested. This assumes
2307 * we sent at least one TLS record work request and
2308 * uses the TCP sequence number after that reqeust as
2309 * the sequence number for the FIN packet.
2310 */
2311 ndesc = ktls_write_tcp_fin(txq, dst, m, available,
2312 tlsp->prev_seq, pidx);
2313 totdesc += ndesc;
2314 }
2315
2316 MPASS(totdesc <= available);
2317 return (totdesc);
2318 }
2319
2320 void
cxgbe_tls_tag_free(struct m_snd_tag * mst)2321 cxgbe_tls_tag_free(struct m_snd_tag *mst)
2322 {
2323 struct adapter *sc;
2324 struct tlspcb *tlsp;
2325
2326 tlsp = mst_to_tls(mst);
2327 sc = tlsp->sc;
2328
2329 CTR2(KTR_CXGBE, "%s: tid %d", __func__, tlsp->tid);
2330
2331 if (tlsp->l2te)
2332 t4_l2t_release(tlsp->l2te);
2333 if (tlsp->tid >= 0)
2334 release_tid(sc, tlsp->tid, tlsp->ctrlq);
2335 if (tlsp->ce)
2336 t4_release_clip_entry(sc, tlsp->ce);
2337 if (tlsp->tx_key_addr >= 0)
2338 free_keyid(tlsp, tlsp->tx_key_addr);
2339
2340 zfree(tlsp, M_CXGBE);
2341 }
2342
2343 void
t6_ktls_modload(void)2344 t6_ktls_modload(void)
2345 {
2346
2347 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, ktls_act_open_rpl,
2348 CPL_COOKIE_KERN_TLS);
2349 }
2350
2351 void
t6_ktls_modunload(void)2352 t6_ktls_modunload(void)
2353 {
2354
2355 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL, NULL,
2356 CPL_COOKIE_KERN_TLS);
2357 }
2358
2359 #else
2360
2361 int
cxgbe_tls_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** pt)2362 cxgbe_tls_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
2363 struct m_snd_tag **pt)
2364 {
2365 return (ENXIO);
2366 }
2367
2368 int
t6_ktls_parse_pkt(struct mbuf * m,int * nsegsp,int * len16p)2369 t6_ktls_parse_pkt(struct mbuf *m, int *nsegsp, int *len16p)
2370 {
2371 return (EINVAL);
2372 }
2373
2374 int
t6_ktls_write_wr(struct sge_txq * txq,void * dst,struct mbuf * m,u_int nsegs,u_int available)2375 t6_ktls_write_wr(struct sge_txq *txq, void *dst, struct mbuf *m, u_int nsegs,
2376 u_int available)
2377 {
2378 panic("can't happen");
2379 }
2380
2381 void
cxgbe_tls_tag_free(struct m_snd_tag * mst)2382 cxgbe_tls_tag_free(struct m_snd_tag *mst)
2383 {
2384 panic("can't happen");
2385 }
2386
2387 void
t6_ktls_modload(void)2388 t6_ktls_modload(void)
2389 {
2390 }
2391
2392 void
t6_ktls_modunload(void)2393 t6_ktls_modunload(void)
2394 {
2395 }
2396
2397 #endif
2398