1 /*-
2 * Copyright (c) 2019-2021 Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2022 NVIDIA corporation & affiliates.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29 #include "opt_kern_tls.h"
30 #include "opt_rss.h"
31 #include "opt_ratelimit.h"
32
33 #include <dev/mlx5/mlx5_en/en.h>
34
35 #include <dev/mlx5/tls.h>
36
37 #include <linux/delay.h>
38 #include <sys/ktls.h>
39 #include <opencrypto/cryptodev.h>
40
41 #ifdef KERN_TLS
42
43 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
44
45 /* software TLS context */
46 struct mlx5_ifc_sw_tls_cntx_bits {
47 struct mlx5_ifc_tls_static_params_bits param;
48 struct mlx5_ifc_tls_progress_params_bits progress;
49 struct {
50 uint8_t key_data[8][0x20];
51 uint8_t key_len[0x20];
52 } key;
53 };
54
55 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params));
56 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc));
57
58 static const char *mlx5e_tls_stats_desc[] = {
59 MLX5E_TLS_STATS(MLX5E_STATS_DESC)
60 };
61
62 static void mlx5e_tls_work(struct work_struct *);
63
64 static int
mlx5e_tls_tag_import(void * arg,void ** store,int cnt,int domain,int flags)65 mlx5e_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags)
66 {
67 struct mlx5e_tls_tag *ptag;
68 int i;
69
70 for (i = 0; i != cnt; i++) {
71 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS,
72 mlx5_dev_domainset(arg), flags | M_ZERO);
73 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF);
74 INIT_WORK(&ptag->work, mlx5e_tls_work);
75 store[i] = ptag;
76 }
77 return (i);
78 }
79
80 static void
mlx5e_tls_tag_release(void * arg,void ** store,int cnt)81 mlx5e_tls_tag_release(void *arg, void **store, int cnt)
82 {
83 struct mlx5e_tls_tag *ptag;
84 struct mlx5e_priv *priv;
85 struct mlx5e_tls *ptls;
86 int i;
87
88 for (i = 0; i != cnt; i++) {
89 ptag = store[i];
90 ptls = ptag->tls;
91 priv = container_of(ptls, struct mlx5e_priv, tls);
92
93 flush_work(&ptag->work);
94
95 if (ptag->tisn != 0) {
96 mlx5_tls_close_tis(priv->mdev, ptag->tisn);
97 atomic_add_32(&ptls->num_resources, -1U);
98 }
99
100 mtx_destroy(&ptag->mtx);
101
102 free(ptag, M_MLX5E_TLS);
103 }
104 }
105
106 static void
mlx5e_tls_tag_zfree(struct mlx5e_tls_tag * ptag)107 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag)
108 {
109 /* make sure any unhandled taskqueue events are ignored */
110 ptag->state = MLX5E_TLS_ST_FREED;
111
112 /* reset some variables */
113 ptag->dek_index = 0;
114 ptag->dek_index_ok = 0;
115
116 /* avoid leaking keys */
117 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
118
119 /* update number of TIS contexts */
120 if (ptag->tisn == 0)
121 atomic_add_32(&ptag->tls->num_resources, -1U);
122
123 /* return tag to UMA */
124 uma_zfree(ptag->tls->zone, ptag);
125 }
126
127 int
mlx5e_tls_init(struct mlx5e_priv * priv)128 mlx5e_tls_init(struct mlx5e_priv *priv)
129 {
130 struct mlx5e_tls *ptls = &priv->tls;
131 struct sysctl_oid *node;
132 uint32_t x;
133
134 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0 ||
135 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0)
136 return (0);
137
138 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq");
139 if (ptls->wq == NULL)
140 return (ENOMEM);
141
142 sysctl_ctx_init(&ptls->ctx);
143
144 snprintf(ptls->zname, sizeof(ptls->zname),
145 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev));
146
147 ptls->zone = uma_zcache_create(ptls->zname,
148 sizeof(struct mlx5e_tls_tag), NULL, NULL, NULL, NULL,
149 mlx5e_tls_tag_import, mlx5e_tls_tag_release, priv->mdev,
150 UMA_ZONE_UNMANAGED);
151
152 /* shared between RX and TX TLS */
153 ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
154
155 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
156 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
157
158 ptls->init = 1;
159
160 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
161 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
162 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload");
163 if (node == NULL)
164 return (0);
165
166 mlx5e_create_counter_stats(&ptls->ctx,
167 SYSCTL_CHILDREN(node), "stats",
168 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM,
169 ptls->stats.arg);
170
171 return (0);
172 }
173
174 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)175 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
176 {
177 struct mlx5e_tls *ptls = &priv->tls;
178 uint32_t x;
179
180 if (ptls->init == 0)
181 return;
182
183 ptls->init = 0;
184 flush_workqueue(ptls->wq);
185 sysctl_ctx_free(&ptls->ctx);
186 uma_zdestroy(ptls->zone);
187 destroy_workqueue(ptls->wq);
188
189 /* check if all resources are freed */
190 MPASS(priv->tls.num_resources == 0);
191
192 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
193 counter_u64_free(ptls->stats.arg[x]);
194 }
195
196 static void
mlx5e_tls_work(struct work_struct * work)197 mlx5e_tls_work(struct work_struct *work)
198 {
199 struct mlx5e_tls_tag *ptag;
200 struct mlx5e_priv *priv;
201 int err;
202
203 ptag = container_of(work, struct mlx5e_tls_tag, work);
204 priv = container_of(ptag->tls, struct mlx5e_priv, tls);
205
206 switch (ptag->state) {
207 case MLX5E_TLS_ST_INIT:
208 /* try to open TIS, if not present */
209 if (ptag->tisn == 0) {
210 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
211 priv->pdn, &ptag->tisn);
212 if (err) {
213 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
214 break;
215 }
216 }
217 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn);
218
219 /* try to allocate a DEK context ID */
220 err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
221 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
222 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
223 &ptag->dek_index);
224 if (err) {
225 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
226 break;
227 }
228
229 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
230
231 ptag->dek_index_ok = 1;
232
233 MLX5E_TLS_TAG_LOCK(ptag);
234 if (ptag->state == MLX5E_TLS_ST_INIT)
235 ptag->state = MLX5E_TLS_ST_SETUP;
236 MLX5E_TLS_TAG_UNLOCK(ptag);
237 break;
238
239 case MLX5E_TLS_ST_RELEASE:
240 /* try to destroy DEK context by ID */
241 if (ptag->dek_index_ok)
242 err = mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
243
244 /* free tag */
245 mlx5e_tls_tag_zfree(ptag);
246 break;
247
248 default:
249 break;
250 }
251 }
252
253 static int
mlx5e_tls_set_params(void * ctx,const struct tls_session_params * en)254 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en)
255 {
256
257 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2);
258 if (en->tls_vminor == TLS_MINOR_VER_TWO)
259 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */
260 else
261 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */
262 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1);
263 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */
264
265 /* copy the initial vector in place */
266 switch (en->iv_len) {
267 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv):
268 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) +
269 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv):
270 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
271 en->iv, en->iv_len);
272 break;
273 default:
274 return (EINVAL);
275 }
276
277 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) {
278 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data),
279 en->cipher_key, en->cipher_key_len);
280 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len);
281 } else {
282 return (EINVAL);
283 }
284 return (0);
285 }
286
287 /* Verify zero default */
288 CTASSERT(MLX5E_TLS_ST_INIT == 0);
289
290 int
mlx5e_tls_snd_tag_alloc(struct ifnet * ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)291 mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
292 union if_snd_tag_alloc_params *params,
293 struct m_snd_tag **ppmt)
294 {
295 union if_snd_tag_alloc_params rl_params;
296 struct mlx5e_priv *priv;
297 struct mlx5e_tls_tag *ptag;
298 const struct tls_session_params *en;
299 int error;
300
301 priv = ifp->if_softc;
302
303 if (priv->gone != 0 || priv->tls.init == 0)
304 return (EOPNOTSUPP);
305
306 /* allocate new tag from zone, if any */
307 ptag = uma_zalloc(priv->tls.zone, M_NOWAIT);
308 if (ptag == NULL)
309 return (ENOMEM);
310
311 /* sanity check default values */
312 MPASS(ptag->dek_index == 0);
313 MPASS(ptag->dek_index_ok == 0);
314
315 /* setup TLS tag */
316 ptag->tls = &priv->tls;
317
318 /* check if there is no TIS context */
319 if (ptag->tisn == 0) {
320 uint32_t value;
321
322 value = atomic_fetchadd_32(&priv->tls.num_resources, 1U);
323
324 /* check resource limits */
325 if (value >= priv->tls.max_resources) {
326 error = ENOMEM;
327 goto failure;
328 }
329 }
330
331 en = ¶ms->tls.tls->params;
332
333 /* only TLS v1.2 and v1.3 is currently supported */
334 if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
335 (en->tls_vminor != TLS_MINOR_VER_TWO
336 #ifdef TLS_MINOR_VER_THREE
337 && en->tls_vminor != TLS_MINOR_VER_THREE
338 #endif
339 )) {
340 error = EPROTONOSUPPORT;
341 goto failure;
342 }
343
344 switch (en->cipher_algorithm) {
345 case CRYPTO_AES_NIST_GCM_16:
346 switch (en->cipher_key_len) {
347 case 128 / 8:
348 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
349 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
350 error = EPROTONOSUPPORT;
351 goto failure;
352 }
353 } else {
354 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
355 error = EPROTONOSUPPORT;
356 goto failure;
357 }
358 }
359 error = mlx5e_tls_set_params(ptag->crypto_params, en);
360 if (error)
361 goto failure;
362 break;
363
364 case 256 / 8:
365 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
366 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
367 error = EPROTONOSUPPORT;
368 goto failure;
369 }
370 } else {
371 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
372 error = EPROTONOSUPPORT;
373 goto failure;
374 }
375 }
376 error = mlx5e_tls_set_params(ptag->crypto_params, en);
377 if (error)
378 goto failure;
379 break;
380
381 default:
382 error = EINVAL;
383 goto failure;
384 }
385 break;
386 default:
387 error = EPROTONOSUPPORT;
388 goto failure;
389 }
390
391 memset(&rl_params, 0, sizeof(rl_params));
392 rl_params.hdr = params->hdr;
393 switch (params->hdr.type) {
394 #ifdef RATELIMIT
395 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
396 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
397 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
398 break;
399 #endif
400 case IF_SND_TAG_TYPE_TLS:
401 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
402 break;
403 default:
404 error = EOPNOTSUPP;
405 goto failure;
406 }
407
408 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag);
409 if (error)
410 goto failure;
411
412 /* store pointer to mbuf tag */
413 MPASS(ptag->tag.refcount == 0);
414 m_snd_tag_init(&ptag->tag, ifp, params->hdr.type);
415 *ppmt = &ptag->tag;
416
417 /* reset state */
418 ptag->state = MLX5E_TLS_ST_INIT;
419
420 queue_work(priv->tls.wq, &ptag->work);
421 flush_work(&ptag->work);
422
423 return (0);
424
425 failure:
426 mlx5e_tls_tag_zfree(ptag);
427 return (error);
428 }
429
430 int
mlx5e_tls_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)431 mlx5e_tls_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
432 {
433 #ifdef RATELIMIT
434 union if_snd_tag_modify_params rl_params;
435 struct mlx5e_tls_tag *ptag =
436 container_of(pmt, struct mlx5e_tls_tag, tag);
437 int error;
438 #endif
439
440 switch (pmt->type) {
441 #ifdef RATELIMIT
442 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
443 memset(&rl_params, 0, sizeof(rl_params));
444 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
445 error = ptag->rl_tag->ifp->if_snd_tag_modify(ptag->rl_tag,
446 &rl_params);
447 return (error);
448 #endif
449 default:
450 return (EOPNOTSUPP);
451 }
452 }
453
454 int
mlx5e_tls_snd_tag_query(struct m_snd_tag * pmt,union if_snd_tag_query_params * params)455 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
456 {
457 struct mlx5e_tls_tag *ptag =
458 container_of(pmt, struct mlx5e_tls_tag, tag);
459 int error;
460
461 switch (pmt->type) {
462 #ifdef RATELIMIT
463 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
464 #endif
465 case IF_SND_TAG_TYPE_TLS:
466 error = ptag->rl_tag->ifp->if_snd_tag_query(ptag->rl_tag,
467 params);
468 break;
469 default:
470 error = EOPNOTSUPP;
471 break;
472 }
473 return (error);
474 }
475
476 void
mlx5e_tls_snd_tag_free(struct m_snd_tag * pmt)477 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
478 {
479 struct mlx5e_tls_tag *ptag =
480 container_of(pmt, struct mlx5e_tls_tag, tag);
481 struct mlx5e_priv *priv;
482
483 m_snd_tag_rele(ptag->rl_tag);
484
485 MLX5E_TLS_TAG_LOCK(ptag);
486 ptag->state = MLX5E_TLS_ST_RELEASE;
487 MLX5E_TLS_TAG_UNLOCK(ptag);
488
489 priv = ptag->tag.ifp->if_softc;
490 queue_work(priv->tls.wq, &ptag->work);
491 }
492
493 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0);
494
495 static void
mlx5e_tls_send_static_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)496 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
497 {
498 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
499 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS);
500 struct mlx5e_tx_umr_wqe *wqe;
501 u16 pi;
502
503 pi = sq->pc & sq->wq.sz_m1;
504 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
505
506 memset(wqe, 0, sizeof(*wqe));
507
508 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
509 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24));
510 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
511 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8);
512
513 if (mlx5e_do_send_cqe(sq))
514 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
515 else
516 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
517
518 /* fill out UMR control segment */
519 wqe->umr.flags = 0x80; /* inline data */
520 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16);
521
522 /* copy in the static crypto parameters */
523 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param),
524 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param));
525
526 /* copy data for doorbell */
527 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
528
529 sq->mbuf[pi].mbuf = NULL;
530 sq->mbuf[pi].num_bytes = 0;
531 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
532 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
533
534 sq->pc += sq->mbuf[pi].num_wqebbs;
535 }
536
537 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) ==
538 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv));
539
540 static void
mlx5e_tls_send_progress_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)541 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
542 {
543 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
544 MLX5_SEND_WQE_DS);
545 struct mlx5e_tx_psv_wqe *wqe;
546 u16 pi;
547
548 pi = sq->pc & sq->wq.sz_m1;
549 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
550
551 memset(wqe, 0, sizeof(*wqe));
552
553 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
554 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24));
555 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
556
557 if (mlx5e_do_send_cqe(sq))
558 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
559
560 /* copy in the PSV control segment */
561 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress),
562 sizeof(wqe->psv));
563
564 /* copy data for doorbell */
565 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
566
567 sq->mbuf[pi].mbuf = NULL;
568 sq->mbuf[pi].num_bytes = 0;
569 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
570 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
571
572 sq->pc += sq->mbuf[pi].num_wqebbs;
573 }
574
575 static void
mlx5e_tls_send_nop(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)576 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
577 {
578 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS;
579 struct mlx5e_tx_wqe *wqe;
580 u16 pi;
581
582 pi = sq->pc & sq->wq.sz_m1;
583 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
584
585 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
586
587 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
588 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
589 if (mlx5e_do_send_cqe(sq))
590 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
591 else
592 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
593
594 /* Copy data for doorbell */
595 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
596
597 sq->mbuf[pi].mbuf = NULL;
598 sq->mbuf[pi].num_bytes = 0;
599 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
600 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
601
602 sq->pc += sq->mbuf[pi].num_wqebbs;
603 }
604
605 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1)
606
607 static struct mbuf *
sbtls_recover_record(struct mbuf * mb,int wait,uint32_t tcp_old,uint32_t * ptcp_seq,bool * pis_start)608 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start)
609 {
610 struct mbuf *mr, *top;
611 uint32_t offset;
612 uint32_t delta;
613
614 /* check format of incoming mbuf */
615 if (mb->m_next == NULL ||
616 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) {
617 top = NULL;
618 goto done;
619 }
620
621 /* get unmapped data offset */
622 offset = mtod(mb->m_next, uintptr_t);
623
624 /* check if we don't need to re-transmit anything */
625 if (offset == 0) {
626 top = SBTLS_MBUF_NO_DATA;
627 *pis_start = true;
628 goto done;
629 }
630
631 /* try to get a new packet header */
632 top = m_gethdr(wait, MT_DATA);
633 if (top == NULL)
634 goto done;
635
636 mr = m_get(wait, MT_DATA);
637 if (mr == NULL) {
638 m_free(top);
639 top = NULL;
640 goto done;
641 }
642
643 top->m_next = mr;
644
645 mb_dupcl(mr, mb->m_next);
646
647 /* the beginning of the TLS record */
648 mr->m_data = NULL;
649
650 /* setup packet header length */
651 top->m_pkthdr.len = mr->m_len = offset;
652 top->m_len = 0;
653
654 /* check for partial re-transmit */
655 delta = *ptcp_seq - tcp_old;
656
657 if (delta < offset) {
658 m_adj(top, offset - delta);
659 offset = delta;
660
661 /* continue where we left off */
662 *pis_start = false;
663 } else {
664 *pis_start = true;
665 }
666
667 /*
668 * Rewind the TCP sequence number by the amount of data
669 * retransmitted:
670 */
671 *ptcp_seq -= offset;
672 done:
673 return (top);
674 }
675
676 static int
mlx5e_sq_tls_populate(struct mbuf * mb,uint64_t * pseq)677 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
678 {
679
680 for (; mb != NULL; mb = mb->m_next) {
681 if (!(mb->m_flags & M_EXTPG))
682 continue;
683 *pseq = mb->m_epg_seqno;
684 return (1);
685 }
686 return (0);
687 }
688
689 int
mlx5e_sq_tls_xmit(struct mlx5e_sq * sq,struct mlx5e_xmit_args * parg,struct mbuf ** ppmb)690 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb)
691 {
692 struct mlx5e_tls_tag *ptls_tag;
693 struct m_snd_tag *ptag;
694 const struct tcphdr *th;
695 struct mbuf *mb = *ppmb;
696 u64 rcd_sn;
697 u32 header_size;
698 u32 mb_seq;
699
700 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
701 return (MLX5E_TLS_CONTINUE);
702
703 ptag = mb->m_pkthdr.snd_tag;
704
705 if (
706 #ifdef RATELIMIT
707 ptag->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
708 #endif
709 ptag->type != IF_SND_TAG_TYPE_TLS)
710 return (MLX5E_TLS_CONTINUE);
711
712 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);
713
714 header_size = mlx5e_get_full_header_size(mb, &th);
715 if (unlikely(header_size == 0 || th == NULL))
716 return (MLX5E_TLS_FAILURE);
717
718 /*
719 * Send non-TLS TCP packets AS-IS:
720 */
721 if (header_size == mb->m_pkthdr.len ||
722 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) {
723 parg->tisn = 0;
724 parg->ihs = header_size;
725 return (MLX5E_TLS_CONTINUE);
726 }
727
728 mb_seq = ntohl(th->th_seq);
729
730 MLX5E_TLS_TAG_LOCK(ptls_tag);
731 switch (ptls_tag->state) {
732 case MLX5E_TLS_ST_INIT:
733 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
734 return (MLX5E_TLS_FAILURE);
735 case MLX5E_TLS_ST_SETUP:
736 ptls_tag->state = MLX5E_TLS_ST_TXRDY;
737 ptls_tag->expected_seq = ~mb_seq; /* force setup */
738 default:
739 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
740 break;
741 }
742
743 if (unlikely(ptls_tag->expected_seq != mb_seq)) {
744 bool is_start;
745 struct mbuf *r_mb;
746 uint32_t tcp_seq = mb_seq;
747
748 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start);
749 if (r_mb == NULL) {
750 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1);
751 return (MLX5E_TLS_FAILURE);
752 }
753
754 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1);
755
756 /* check if this is the first fragment of a TLS record */
757 if (is_start) {
758 /* setup TLS static parameters */
759 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params,
760 param.initial_record_number, rcd_sn);
761
762 /*
763 * NOTE: The sendqueue should have enough room to
764 * carry both the static and the progress parameters
765 * when we get here!
766 */
767 mlx5e_tls_send_static_parameters(sq, ptls_tag);
768 mlx5e_tls_send_progress_parameters(sq, ptls_tag);
769
770 if (r_mb == SBTLS_MBUF_NO_DATA) {
771 mlx5e_tls_send_nop(sq, ptls_tag);
772 ptls_tag->expected_seq = mb_seq;
773 return (MLX5E_TLS_LOOP);
774 }
775 }
776
777 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len);
778
779 /* setup transmit arguments */
780 parg->tisn = ptls_tag->tisn;
781 parg->mst = &ptls_tag->tag;
782
783 /* try to send DUMP data */
784 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) {
785 m_freem(r_mb);
786 ptls_tag->expected_seq = tcp_seq;
787 return (MLX5E_TLS_FAILURE);
788 } else {
789 ptls_tag->expected_seq = mb_seq;
790 return (MLX5E_TLS_LOOP);
791 }
792 } else {
793 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1);
794 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len);
795 }
796 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size;
797
798 parg->tisn = ptls_tag->tisn;
799 parg->ihs = header_size;
800 parg->mst = &ptls_tag->tag;
801 return (MLX5E_TLS_CONTINUE);
802 }
803
804 #else
805
806 int
mlx5e_tls_init(struct mlx5e_priv * priv)807 mlx5e_tls_init(struct mlx5e_priv *priv)
808 {
809
810 return (0);
811 }
812
813 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)814 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
815 {
816 /* NOP */
817 }
818
819 #endif /* KERN_TLS */
820