1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_mempool.h>
7 #include <rte_eal_paging.h>
8 #include <rte_errno.h>
9 #include <rte_log.h>
10 #include <rte_bus_pci.h>
11 #include <rte_memory.h>
12
13 #include <mlx5_glue.h>
14 #include <mlx5_common.h>
15 #include <mlx5_devx_cmds.h>
16 #include <mlx5_common_os.h>
17
18 #include "mlx5_crypto_utils.h"
19 #include "mlx5_crypto.h"
20
21 #define MLX5_CRYPTO_DRIVER_NAME crypto_mlx5
22 #define MLX5_CRYPTO_LOG_NAME pmd.crypto.mlx5
23 #define MLX5_CRYPTO_MAX_QPS 128
24 #define MLX5_CRYPTO_MAX_SEGS 56
25
26 #define MLX5_CRYPTO_FEATURE_FLAGS \
27 (RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | \
28 RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | \
29 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | \
30 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | \
31 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | \
32 RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY | \
33 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)
34
35 TAILQ_HEAD(mlx5_crypto_privs, mlx5_crypto_priv) mlx5_crypto_priv_list =
36 TAILQ_HEAD_INITIALIZER(mlx5_crypto_priv_list);
37 static pthread_mutex_t priv_list_lock;
38
39 int mlx5_crypto_logtype;
40
41 uint8_t mlx5_crypto_driver_id;
42
43 const struct rte_cryptodev_capabilities mlx5_crypto_caps[] = {
44 { /* AES XTS */
45 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
46 {.sym = {
47 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
48 {.cipher = {
49 .algo = RTE_CRYPTO_CIPHER_AES_XTS,
50 .block_size = 16,
51 .key_size = {
52 .min = 32,
53 .max = 64,
54 .increment = 32
55 },
56 .iv_size = {
57 .min = 16,
58 .max = 16,
59 .increment = 0
60 },
61 .dataunit_set =
62 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES |
63 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES |
64 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES,
65 }, }
66 }, }
67 },
68 };
69
70 static const char mlx5_crypto_drv_name[] = RTE_STR(MLX5_CRYPTO_DRIVER_NAME);
71
72 static const struct rte_driver mlx5_drv = {
73 .name = mlx5_crypto_drv_name,
74 .alias = mlx5_crypto_drv_name
75 };
76
77 static struct cryptodev_driver mlx5_cryptodev_driver;
78
79 struct mlx5_crypto_session {
80 uint32_t bs_bpt_eo_es;
81 /**< bsf_size, bsf_p_type, encryption_order and encryption standard,
82 * saved in big endian format.
83 */
84 uint32_t bsp_res;
85 /**< crypto_block_size_pointer and reserved 24 bits saved in big
86 * endian format.
87 */
88 uint32_t iv_offset:16;
89 /**< Starting point for Initialisation Vector. */
90 struct mlx5_crypto_dek *dek; /**< Pointer to dek struct. */
91 uint32_t dek_id; /**< DEK ID */
92 } __rte_packed;
93
94 static void
mlx5_crypto_dev_infos_get(struct rte_cryptodev * dev,struct rte_cryptodev_info * dev_info)95 mlx5_crypto_dev_infos_get(struct rte_cryptodev *dev,
96 struct rte_cryptodev_info *dev_info)
97 {
98 RTE_SET_USED(dev);
99 if (dev_info != NULL) {
100 dev_info->driver_id = mlx5_crypto_driver_id;
101 dev_info->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
102 dev_info->capabilities = mlx5_crypto_caps;
103 dev_info->max_nb_queue_pairs = MLX5_CRYPTO_MAX_QPS;
104 dev_info->min_mbuf_headroom_req = 0;
105 dev_info->min_mbuf_tailroom_req = 0;
106 dev_info->sym.max_nb_sessions = 0;
107 /*
108 * If 0, the device does not have any limitation in number of
109 * sessions that can be used.
110 */
111 }
112 }
113
114 static int
mlx5_crypto_dev_configure(struct rte_cryptodev * dev,struct rte_cryptodev_config * config)115 mlx5_crypto_dev_configure(struct rte_cryptodev *dev,
116 struct rte_cryptodev_config *config)
117 {
118 struct mlx5_crypto_priv *priv = dev->data->dev_private;
119
120 if (config == NULL) {
121 DRV_LOG(ERR, "Invalid crypto dev configure parameters.");
122 return -EINVAL;
123 }
124 if ((config->ff_disable & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) != 0) {
125 DRV_LOG(ERR,
126 "Disabled symmetric crypto feature is not supported.");
127 return -ENOTSUP;
128 }
129 if (mlx5_crypto_dek_setup(priv) != 0) {
130 DRV_LOG(ERR, "Dek hash list creation has failed.");
131 return -ENOMEM;
132 }
133 priv->dev_config = *config;
134 DRV_LOG(DEBUG, "Device %u was configured.", dev->driver_id);
135 return 0;
136 }
137
138 static void
mlx5_crypto_dev_stop(struct rte_cryptodev * dev)139 mlx5_crypto_dev_stop(struct rte_cryptodev *dev)
140 {
141 RTE_SET_USED(dev);
142 }
143
144 static int
mlx5_crypto_dev_start(struct rte_cryptodev * dev)145 mlx5_crypto_dev_start(struct rte_cryptodev *dev)
146 {
147 struct mlx5_crypto_priv *priv = dev->data->dev_private;
148
149 return mlx5_dev_mempool_subscribe(priv->cdev);
150 }
151
152 static int
mlx5_crypto_dev_close(struct rte_cryptodev * dev)153 mlx5_crypto_dev_close(struct rte_cryptodev *dev)
154 {
155 struct mlx5_crypto_priv *priv = dev->data->dev_private;
156
157 mlx5_crypto_dek_unset(priv);
158 DRV_LOG(DEBUG, "Device %u was closed.", dev->driver_id);
159 return 0;
160 }
161
162 static unsigned int
mlx5_crypto_sym_session_get_size(struct rte_cryptodev * dev __rte_unused)163 mlx5_crypto_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
164 {
165 return sizeof(struct mlx5_crypto_session);
166 }
167
168 static int
mlx5_crypto_sym_session_configure(struct rte_cryptodev * dev,struct rte_crypto_sym_xform * xform,struct rte_cryptodev_sym_session * session,struct rte_mempool * mp)169 mlx5_crypto_sym_session_configure(struct rte_cryptodev *dev,
170 struct rte_crypto_sym_xform *xform,
171 struct rte_cryptodev_sym_session *session,
172 struct rte_mempool *mp)
173 {
174 struct mlx5_crypto_priv *priv = dev->data->dev_private;
175 struct mlx5_crypto_session *sess_private_data;
176 struct rte_crypto_cipher_xform *cipher;
177 uint8_t encryption_order;
178 int ret;
179
180 if (unlikely(xform->next != NULL)) {
181 DRV_LOG(ERR, "Xform next is not supported.");
182 return -ENOTSUP;
183 }
184 if (unlikely((xform->type != RTE_CRYPTO_SYM_XFORM_CIPHER) ||
185 (xform->cipher.algo != RTE_CRYPTO_CIPHER_AES_XTS))) {
186 DRV_LOG(ERR, "Only AES-XTS algorithm is supported.");
187 return -ENOTSUP;
188 }
189 ret = rte_mempool_get(mp, (void *)&sess_private_data);
190 if (ret != 0) {
191 DRV_LOG(ERR,
192 "Failed to get session %p private data from mempool.",
193 sess_private_data);
194 return -ENOMEM;
195 }
196 cipher = &xform->cipher;
197 sess_private_data->dek = mlx5_crypto_dek_prepare(priv, cipher);
198 if (sess_private_data->dek == NULL) {
199 rte_mempool_put(mp, sess_private_data);
200 DRV_LOG(ERR, "Failed to prepare dek.");
201 return -ENOMEM;
202 }
203 if (cipher->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
204 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY;
205 else
206 encryption_order = MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE;
207 sess_private_data->bs_bpt_eo_es = rte_cpu_to_be_32
208 (MLX5_BSF_SIZE_64B << MLX5_BSF_SIZE_OFFSET |
209 MLX5_BSF_P_TYPE_CRYPTO << MLX5_BSF_P_TYPE_OFFSET |
210 encryption_order << MLX5_ENCRYPTION_ORDER_OFFSET |
211 MLX5_ENCRYPTION_STANDARD_AES_XTS);
212 switch (xform->cipher.dataunit_len) {
213 case 0:
214 sess_private_data->bsp_res = 0;
215 break;
216 case 512:
217 sess_private_data->bsp_res = rte_cpu_to_be_32
218 ((uint32_t)MLX5_BLOCK_SIZE_512B <<
219 MLX5_BLOCK_SIZE_OFFSET);
220 break;
221 case 4096:
222 sess_private_data->bsp_res = rte_cpu_to_be_32
223 ((uint32_t)MLX5_BLOCK_SIZE_4096B <<
224 MLX5_BLOCK_SIZE_OFFSET);
225 break;
226 case 1048576:
227 sess_private_data->bsp_res = rte_cpu_to_be_32
228 ((uint32_t)MLX5_BLOCK_SIZE_1MB <<
229 MLX5_BLOCK_SIZE_OFFSET);
230 break;
231 default:
232 DRV_LOG(ERR, "Cipher data unit length is not supported.");
233 return -ENOTSUP;
234 }
235 sess_private_data->iv_offset = cipher->iv.offset;
236 sess_private_data->dek_id =
237 rte_cpu_to_be_32(sess_private_data->dek->obj->id &
238 0xffffff);
239 set_sym_session_private_data(session, dev->driver_id,
240 sess_private_data);
241 DRV_LOG(DEBUG, "Session %p was configured.", sess_private_data);
242 return 0;
243 }
244
245 static void
mlx5_crypto_sym_session_clear(struct rte_cryptodev * dev,struct rte_cryptodev_sym_session * sess)246 mlx5_crypto_sym_session_clear(struct rte_cryptodev *dev,
247 struct rte_cryptodev_sym_session *sess)
248 {
249 struct mlx5_crypto_priv *priv = dev->data->dev_private;
250 struct mlx5_crypto_session *spriv = get_sym_session_private_data(sess,
251 dev->driver_id);
252
253 if (unlikely(spriv == NULL)) {
254 DRV_LOG(ERR, "Failed to get session %p private data.", spriv);
255 return;
256 }
257 mlx5_crypto_dek_destroy(priv, spriv->dek);
258 set_sym_session_private_data(sess, dev->driver_id, NULL);
259 rte_mempool_put(rte_mempool_from_obj(spriv), spriv);
260 DRV_LOG(DEBUG, "Session %p was cleared.", spriv);
261 }
262
263 static void
mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp * qp,uint16_t n)264 mlx5_crypto_indirect_mkeys_release(struct mlx5_crypto_qp *qp, uint16_t n)
265 {
266 uint16_t i;
267
268 for (i = 0; i < n; i++)
269 if (qp->mkey[i])
270 claim_zero(mlx5_devx_cmd_destroy(qp->mkey[i]));
271 }
272
273 static void
mlx5_crypto_qp_release(struct mlx5_crypto_qp * qp)274 mlx5_crypto_qp_release(struct mlx5_crypto_qp *qp)
275 {
276 if (qp == NULL)
277 return;
278 mlx5_devx_qp_destroy(&qp->qp_obj);
279 mlx5_mr_btree_free(&qp->mr_ctrl.cache_bh);
280 mlx5_devx_cq_destroy(&qp->cq_obj);
281 rte_free(qp);
282 }
283
284 static int
mlx5_crypto_queue_pair_release(struct rte_cryptodev * dev,uint16_t qp_id)285 mlx5_crypto_queue_pair_release(struct rte_cryptodev *dev, uint16_t qp_id)
286 {
287 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
288
289 mlx5_crypto_indirect_mkeys_release(qp, qp->entries_n);
290 mlx5_crypto_qp_release(qp);
291 dev->data->queue_pairs[qp_id] = NULL;
292 return 0;
293 }
294
295 static __rte_noinline uint32_t
mlx5_crypto_get_block_size(struct rte_crypto_op * op)296 mlx5_crypto_get_block_size(struct rte_crypto_op *op)
297 {
298 uint32_t bl = op->sym->cipher.data.length;
299
300 switch (bl) {
301 case (1 << 20):
302 return RTE_BE32(MLX5_BLOCK_SIZE_1MB << MLX5_BLOCK_SIZE_OFFSET);
303 case (1 << 12):
304 return RTE_BE32(MLX5_BLOCK_SIZE_4096B <<
305 MLX5_BLOCK_SIZE_OFFSET);
306 case (1 << 9):
307 return RTE_BE32(MLX5_BLOCK_SIZE_512B << MLX5_BLOCK_SIZE_OFFSET);
308 default:
309 DRV_LOG(ERR, "Unknown block size: %u.", bl);
310 return UINT32_MAX;
311 }
312 }
313
314 static __rte_always_inline uint32_t
mlx5_crypto_klm_set(struct mlx5_crypto_qp * qp,struct rte_mbuf * mbuf,struct mlx5_wqe_dseg * klm,uint32_t offset,uint32_t * remain)315 mlx5_crypto_klm_set(struct mlx5_crypto_qp *qp, struct rte_mbuf *mbuf,
316 struct mlx5_wqe_dseg *klm, uint32_t offset,
317 uint32_t *remain)
318 {
319 uint32_t data_len = (rte_pktmbuf_data_len(mbuf) - offset);
320 uintptr_t addr = rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset);
321
322 if (data_len > *remain)
323 data_len = *remain;
324 *remain -= data_len;
325 klm->bcount = rte_cpu_to_be_32(data_len);
326 klm->pbuf = rte_cpu_to_be_64(addr);
327 klm->lkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
328 return klm->lkey;
329
330 }
331
332 static __rte_always_inline uint32_t
mlx5_crypto_klms_set(struct mlx5_crypto_qp * qp,struct rte_crypto_op * op,struct rte_mbuf * mbuf,struct mlx5_wqe_dseg * klm)333 mlx5_crypto_klms_set(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op,
334 struct rte_mbuf *mbuf, struct mlx5_wqe_dseg *klm)
335 {
336 uint32_t remain_len = op->sym->cipher.data.length;
337 uint32_t nb_segs = mbuf->nb_segs;
338 uint32_t klm_n = 1u;
339
340 /* First mbuf needs to take the cipher offset. */
341 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, klm,
342 op->sym->cipher.data.offset, &remain_len) == UINT32_MAX)) {
343 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
344 return 0;
345 }
346 while (remain_len) {
347 nb_segs--;
348 mbuf = mbuf->next;
349 if (unlikely(mbuf == NULL || nb_segs == 0)) {
350 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
351 return 0;
352 }
353 if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
354 &remain_len) == UINT32_MAX)) {
355 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
356 return 0;
357 }
358 klm_n++;
359 }
360 return klm_n;
361 }
362
363 static __rte_always_inline int
mlx5_crypto_wqe_set(struct mlx5_crypto_priv * priv,struct mlx5_crypto_qp * qp,struct rte_crypto_op * op,struct mlx5_umr_wqe * umr)364 mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
365 struct mlx5_crypto_qp *qp,
366 struct rte_crypto_op *op,
367 struct mlx5_umr_wqe *umr)
368 {
369 struct mlx5_crypto_session *sess = get_sym_session_private_data
370 (op->sym->session, mlx5_crypto_driver_id);
371 struct mlx5_wqe_cseg *cseg = &umr->ctr;
372 struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
373 struct mlx5_wqe_dseg *klms = &umr->kseg[0];
374 struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
375 RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
376 uint32_t ds;
377 bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
378 /* Set UMR WQE. */
379 uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
380 ipl ? op->sym->m_src : op->sym->m_dst, klms);
381
382 if (unlikely(klm_n == 0))
383 return 0;
384 bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
385 if (unlikely(!sess->bsp_res)) {
386 bsf->bsp_res = mlx5_crypto_get_block_size(op);
387 if (unlikely(bsf->bsp_res == UINT32_MAX)) {
388 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
389 return 0;
390 }
391 } else {
392 bsf->bsp_res = sess->bsp_res;
393 }
394 bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
395 memcpy(bsf->xts_initial_tweak,
396 rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
397 bsf->res_dp = sess->dek_id;
398 mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
399 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
400 qp->db_pi += priv->umr_wqe_stride;
401 /* Set RDMA_WRITE WQE. */
402 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
403 klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
404 if (!ipl) {
405 klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
406 if (unlikely(klm_n == 0))
407 return 0;
408 } else {
409 memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
410 }
411 ds = 2 + klm_n;
412 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
413 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
414 MLX5_OPCODE_RDMA_WRITE);
415 ds = RTE_ALIGN(ds, 4);
416 qp->db_pi += ds >> 2;
417 /* Set NOP WQE if needed. */
418 if (priv->max_rdmar_ds > ds) {
419 cseg += ds;
420 ds = priv->max_rdmar_ds - ds;
421 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
422 cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
423 MLX5_OPCODE_NOP);
424 qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
425 }
426 qp->wqe = (uint8_t *)cseg;
427 return 1;
428 }
429
430 static uint16_t
mlx5_crypto_enqueue_burst(void * queue_pair,struct rte_crypto_op ** ops,uint16_t nb_ops)431 mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
432 uint16_t nb_ops)
433 {
434 struct mlx5_crypto_qp *qp = queue_pair;
435 struct mlx5_crypto_priv *priv = qp->priv;
436 struct mlx5_umr_wqe *umr;
437 struct rte_crypto_op *op;
438 uint16_t mask = qp->entries_n - 1;
439 uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
440 uint32_t idx;
441
442 if (remain < nb_ops)
443 nb_ops = remain;
444 else
445 remain = nb_ops;
446 if (unlikely(remain == 0))
447 return 0;
448 do {
449 idx = qp->pi & mask;
450 op = *ops++;
451 umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
452 priv->wqe_set_size * idx);
453 if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
454 qp->stats.enqueue_err_count++;
455 if (remain != nb_ops) {
456 qp->stats.enqueued_count -= remain;
457 break;
458 }
459 return 0;
460 }
461 qp->ops[idx] = op;
462 qp->pi++;
463 } while (--remain);
464 qp->stats.enqueued_count += nb_ops;
465 mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe,
466 qp->db_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR],
467 !priv->uar.dbnc);
468 return nb_ops;
469 }
470
471 static __rte_noinline void
mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp * qp,struct rte_crypto_op * op)472 mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
473 {
474 const uint32_t idx = qp->ci & (qp->entries_n - 1);
475 volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
476 &qp->cq_obj.cqes[idx];
477
478 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
479 qp->stats.dequeue_err_count++;
480 DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
481 }
482
483 static uint16_t
mlx5_crypto_dequeue_burst(void * queue_pair,struct rte_crypto_op ** ops,uint16_t nb_ops)484 mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
485 uint16_t nb_ops)
486 {
487 struct mlx5_crypto_qp *qp = queue_pair;
488 volatile struct mlx5_cqe *restrict cqe;
489 struct rte_crypto_op *restrict op;
490 const unsigned int cq_size = qp->entries_n;
491 const unsigned int mask = cq_size - 1;
492 uint32_t idx;
493 uint32_t next_idx = qp->ci & mask;
494 const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
495 uint16_t i = 0;
496 int ret;
497
498 if (unlikely(max == 0))
499 return 0;
500 do {
501 idx = next_idx;
502 next_idx = (qp->ci + 1) & mask;
503 op = qp->ops[idx];
504 cqe = &qp->cq_obj.cqes[idx];
505 ret = check_cqe(cqe, cq_size, qp->ci);
506 rte_io_rmb();
507 if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
508 if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
509 mlx5_crypto_cqe_err_handle(qp, op);
510 break;
511 }
512 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
513 ops[i++] = op;
514 qp->ci++;
515 } while (i < max);
516 if (likely(i != 0)) {
517 rte_io_wmb();
518 qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
519 qp->stats.dequeued_count += i;
520 }
521 return i;
522 }
523
524 static void
mlx5_crypto_qp_init(struct mlx5_crypto_priv * priv,struct mlx5_crypto_qp * qp)525 mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
526 {
527 uint32_t i;
528
529 for (i = 0 ; i < qp->entries_n; i++) {
530 struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
531 i * priv->wqe_set_size);
532 struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
533 (cseg + 1);
534 struct mlx5_wqe_umr_bsf_seg *bsf =
535 (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
536 priv->umr_wqe_size)) - 1;
537 struct mlx5_wqe_rseg *rseg;
538
539 /* Init UMR WQE. */
540 cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
541 (priv->umr_wqe_size / MLX5_WSEG_SIZE));
542 cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
543 MLX5_COMP_MODE_OFFSET);
544 cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
545 ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
546 ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
547 ucseg->ko_to_bs = rte_cpu_to_be_32
548 ((MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size) <<
549 MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
550 bsf->keytag = priv->keytag;
551 /* Init RDMA WRITE WQE. */
552 cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
553 cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
554 MLX5_COMP_MODE_OFFSET) |
555 MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
556 rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
557 rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
558 }
559 }
560
561 static int
mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv * priv,struct mlx5_crypto_qp * qp)562 mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
563 struct mlx5_crypto_qp *qp)
564 {
565 struct mlx5_umr_wqe *umr;
566 uint32_t i;
567 struct mlx5_devx_mkey_attr attr = {
568 .pd = priv->cdev->pdn,
569 .umr_en = 1,
570 .crypto_en = 1,
571 .set_remote_rw = 1,
572 .klm_num = MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
573 };
574
575 for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
576 i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
577 attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
578 qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
579 if (!qp->mkey[i])
580 goto error;
581 }
582 return 0;
583 error:
584 DRV_LOG(ERR, "Failed to allocate indirect mkey.");
585 mlx5_crypto_indirect_mkeys_release(qp, i);
586 return -1;
587 }
588
589 static int
mlx5_crypto_queue_pair_setup(struct rte_cryptodev * dev,uint16_t qp_id,const struct rte_cryptodev_qp_conf * qp_conf,int socket_id)590 mlx5_crypto_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
591 const struct rte_cryptodev_qp_conf *qp_conf,
592 int socket_id)
593 {
594 struct mlx5_crypto_priv *priv = dev->data->dev_private;
595 struct mlx5_devx_qp_attr attr = {0};
596 struct mlx5_crypto_qp *qp;
597 uint16_t log_nb_desc = rte_log2_u32(qp_conf->nb_descriptors);
598 uint32_t ret;
599 uint32_t alloc_size = sizeof(*qp);
600 uint32_t log_wqbb_n;
601 struct mlx5_devx_cq_attr cq_attr = {
602 .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj),
603 };
604
605 if (dev->data->queue_pairs[qp_id] != NULL)
606 mlx5_crypto_queue_pair_release(dev, qp_id);
607 alloc_size = RTE_ALIGN(alloc_size, RTE_CACHE_LINE_SIZE);
608 alloc_size += (sizeof(struct rte_crypto_op *) +
609 sizeof(struct mlx5_devx_obj *)) *
610 RTE_BIT32(log_nb_desc);
611 qp = rte_zmalloc_socket(__func__, alloc_size, RTE_CACHE_LINE_SIZE,
612 socket_id);
613 if (qp == NULL) {
614 DRV_LOG(ERR, "Failed to allocate QP memory.");
615 rte_errno = ENOMEM;
616 return -rte_errno;
617 }
618 if (mlx5_devx_cq_create(priv->cdev->ctx, &qp->cq_obj, log_nb_desc,
619 &cq_attr, socket_id) != 0) {
620 DRV_LOG(ERR, "Failed to create CQ.");
621 goto error;
622 }
623 log_wqbb_n = rte_log2_u32(RTE_BIT32(log_nb_desc) *
624 (priv->wqe_set_size / MLX5_SEND_WQE_BB));
625 attr.pd = priv->cdev->pdn;
626 attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj);
627 attr.cqn = qp->cq_obj.cq->id;
628 attr.num_of_receive_wqes = 0;
629 attr.num_of_send_wqbbs = RTE_BIT32(log_wqbb_n);
630 attr.ts_format =
631 mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
632 ret = mlx5_devx_qp_create(priv->cdev->ctx, &qp->qp_obj,
633 attr.num_of_send_wqbbs * MLX5_WQE_SIZE,
634 &attr, socket_id);
635 if (ret) {
636 DRV_LOG(ERR, "Failed to create QP.");
637 goto error;
638 }
639 if (mlx5_mr_ctrl_init(&qp->mr_ctrl, &priv->cdev->mr_scache.dev_gen,
640 priv->dev_config.socket_id) != 0) {
641 DRV_LOG(ERR, "Cannot allocate MR Btree for qp %u.",
642 (uint32_t)qp_id);
643 rte_errno = ENOMEM;
644 goto error;
645 }
646 /*
647 * In Order to configure self loopback, when calling devx qp2rts the
648 * remote QP id that is used is the id of the same QP.
649 */
650 if (mlx5_devx_qp2rts(&qp->qp_obj, qp->qp_obj.qp->id))
651 goto error;
652 qp->mkey = (struct mlx5_devx_obj **)RTE_ALIGN((uintptr_t)(qp + 1),
653 RTE_CACHE_LINE_SIZE);
654 qp->ops = (struct rte_crypto_op **)(qp->mkey + RTE_BIT32(log_nb_desc));
655 qp->entries_n = 1 << log_nb_desc;
656 if (mlx5_crypto_indirect_mkeys_prepare(priv, qp)) {
657 DRV_LOG(ERR, "Cannot allocate indirect memory regions.");
658 rte_errno = ENOMEM;
659 goto error;
660 }
661 mlx5_crypto_qp_init(priv, qp);
662 qp->priv = priv;
663 dev->data->queue_pairs[qp_id] = qp;
664 return 0;
665 error:
666 mlx5_crypto_qp_release(qp);
667 return -1;
668 }
669
670 static void
mlx5_crypto_stats_get(struct rte_cryptodev * dev,struct rte_cryptodev_stats * stats)671 mlx5_crypto_stats_get(struct rte_cryptodev *dev,
672 struct rte_cryptodev_stats *stats)
673 {
674 int qp_id;
675
676 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
677 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
678
679 stats->enqueued_count += qp->stats.enqueued_count;
680 stats->dequeued_count += qp->stats.dequeued_count;
681 stats->enqueue_err_count += qp->stats.enqueue_err_count;
682 stats->dequeue_err_count += qp->stats.dequeue_err_count;
683 }
684 }
685
686 static void
mlx5_crypto_stats_reset(struct rte_cryptodev * dev)687 mlx5_crypto_stats_reset(struct rte_cryptodev *dev)
688 {
689 int qp_id;
690
691 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
692 struct mlx5_crypto_qp *qp = dev->data->queue_pairs[qp_id];
693
694 memset(&qp->stats, 0, sizeof(qp->stats));
695 }
696 }
697
698 static struct rte_cryptodev_ops mlx5_crypto_ops = {
699 .dev_configure = mlx5_crypto_dev_configure,
700 .dev_start = mlx5_crypto_dev_start,
701 .dev_stop = mlx5_crypto_dev_stop,
702 .dev_close = mlx5_crypto_dev_close,
703 .dev_infos_get = mlx5_crypto_dev_infos_get,
704 .stats_get = mlx5_crypto_stats_get,
705 .stats_reset = mlx5_crypto_stats_reset,
706 .queue_pair_setup = mlx5_crypto_queue_pair_setup,
707 .queue_pair_release = mlx5_crypto_queue_pair_release,
708 .sym_session_get_size = mlx5_crypto_sym_session_get_size,
709 .sym_session_configure = mlx5_crypto_sym_session_configure,
710 .sym_session_clear = mlx5_crypto_sym_session_clear,
711 .sym_get_raw_dp_ctx_size = NULL,
712 .sym_configure_raw_dp_ctx = NULL,
713 };
714
715 static int
mlx5_crypto_args_check_handler(const char * key,const char * val,void * opaque)716 mlx5_crypto_args_check_handler(const char *key, const char *val, void *opaque)
717 {
718 struct mlx5_crypto_devarg_params *devarg_prms = opaque;
719 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
720 unsigned long tmp;
721 FILE *file;
722 int ret;
723 int i;
724
725 if (strcmp(key, "wcs_file") == 0) {
726 file = fopen(val, "rb");
727 if (file == NULL) {
728 rte_errno = ENOTSUP;
729 return -rte_errno;
730 }
731 for (i = 0 ; i < MLX5_CRYPTO_CREDENTIAL_SIZE ; i++) {
732 ret = fscanf(file, "%02hhX", &attr->credential[i]);
733 if (ret <= 0) {
734 fclose(file);
735 DRV_LOG(ERR,
736 "Failed to read credential from file.");
737 rte_errno = EINVAL;
738 return -rte_errno;
739 }
740 }
741 fclose(file);
742 devarg_prms->login_devarg = true;
743 return 0;
744 }
745 errno = 0;
746 tmp = strtoul(val, NULL, 0);
747 if (errno) {
748 DRV_LOG(WARNING, "%s: \"%s\" is an invalid integer.", key, val);
749 return -errno;
750 }
751 if (strcmp(key, "max_segs_num") == 0) {
752 if (!tmp) {
753 DRV_LOG(ERR, "max_segs_num must be greater than 0.");
754 rte_errno = EINVAL;
755 return -rte_errno;
756 }
757 devarg_prms->max_segs_num = (uint32_t)tmp;
758 } else if (strcmp(key, "import_kek_id") == 0) {
759 attr->session_import_kek_ptr = (uint32_t)tmp;
760 } else if (strcmp(key, "credential_id") == 0) {
761 attr->credential_pointer = (uint32_t)tmp;
762 } else if (strcmp(key, "keytag") == 0) {
763 devarg_prms->keytag = tmp;
764 }
765 return 0;
766 }
767
768 static int
mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl * mkvlist,struct mlx5_crypto_devarg_params * devarg_prms)769 mlx5_crypto_parse_devargs(struct mlx5_kvargs_ctrl *mkvlist,
770 struct mlx5_crypto_devarg_params *devarg_prms)
771 {
772 struct mlx5_devx_crypto_login_attr *attr = &devarg_prms->login_attr;
773 const char **params = (const char *[]){
774 "credential_id",
775 "import_kek_id",
776 "keytag",
777 "max_segs_num",
778 "wcs_file",
779 NULL,
780 };
781
782 /* Default values. */
783 attr->credential_pointer = 0;
784 attr->session_import_kek_ptr = 0;
785 devarg_prms->keytag = 0;
786 devarg_prms->max_segs_num = 8;
787 if (mkvlist == NULL) {
788 DRV_LOG(ERR,
789 "No login devargs in order to enable crypto operations in the device.");
790 rte_errno = EINVAL;
791 return -1;
792 }
793 if (mlx5_kvargs_process(mkvlist, params, mlx5_crypto_args_check_handler,
794 devarg_prms) != 0) {
795 DRV_LOG(ERR, "Devargs handler function Failed.");
796 rte_errno = EINVAL;
797 return -1;
798 }
799 if (devarg_prms->login_devarg == false) {
800 DRV_LOG(ERR,
801 "No login credential devarg in order to enable crypto operations in the device.");
802 rte_errno = EINVAL;
803 return -1;
804 }
805 return 0;
806 }
807
808 /*
809 * Calculate UMR WQE size and RDMA Write WQE size with the
810 * following limitations:
811 * - Each WQE size is multiple of 64.
812 * - The summarize of both UMR WQE and RDMA_W WQE is a power of 2.
813 * - The number of entries in the UMR WQE's KLM list is multiple of 4.
814 */
815 static void
mlx5_crypto_get_wqe_sizes(uint32_t segs_num,uint32_t * umr_size,uint32_t * rdmaw_size)816 mlx5_crypto_get_wqe_sizes(uint32_t segs_num, uint32_t *umr_size,
817 uint32_t *rdmaw_size)
818 {
819 uint32_t diff, wqe_set_size;
820
821 *umr_size = MLX5_CRYPTO_UMR_WQE_STATIC_SIZE +
822 RTE_ALIGN(segs_num, 4) *
823 sizeof(struct mlx5_wqe_dseg);
824 /* Make sure UMR WQE size is multiple of WQBB. */
825 *umr_size = RTE_ALIGN(*umr_size, MLX5_SEND_WQE_BB);
826 *rdmaw_size = sizeof(struct mlx5_rdma_write_wqe) +
827 sizeof(struct mlx5_wqe_dseg) *
828 (segs_num <= 2 ? 2 : 2 +
829 RTE_ALIGN(segs_num - 2, 4));
830 /* Make sure RDMA_WRITE WQE size is multiple of WQBB. */
831 *rdmaw_size = RTE_ALIGN(*rdmaw_size, MLX5_SEND_WQE_BB);
832 wqe_set_size = *rdmaw_size + *umr_size;
833 diff = rte_align32pow2(wqe_set_size) - wqe_set_size;
834 /* Make sure wqe_set size is power of 2. */
835 if (diff)
836 *umr_size += diff;
837 }
838
839 static uint8_t
mlx5_crypto_max_segs_num(uint16_t max_wqe_size)840 mlx5_crypto_max_segs_num(uint16_t max_wqe_size)
841 {
842 int klms_sizes = max_wqe_size - MLX5_CRYPTO_UMR_WQE_STATIC_SIZE;
843 uint32_t max_segs_cap = RTE_ALIGN_FLOOR(klms_sizes, MLX5_SEND_WQE_BB) /
844 sizeof(struct mlx5_wqe_dseg);
845
846 MLX5_ASSERT(klms_sizes >= MLX5_SEND_WQE_BB);
847 while (max_segs_cap) {
848 uint32_t umr_wqe_size, rdmw_wqe_size;
849
850 mlx5_crypto_get_wqe_sizes(max_segs_cap, &umr_wqe_size,
851 &rdmw_wqe_size);
852 if (umr_wqe_size <= max_wqe_size &&
853 rdmw_wqe_size <= max_wqe_size)
854 break;
855 max_segs_cap -= 4;
856 }
857 return max_segs_cap;
858 }
859
860 static int
mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv * priv,uint16_t max_wqe_size,uint32_t max_segs_num)861 mlx5_crypto_configure_wqe_size(struct mlx5_crypto_priv *priv,
862 uint16_t max_wqe_size, uint32_t max_segs_num)
863 {
864 uint32_t rdmw_wqe_size, umr_wqe_size;
865
866 mlx5_crypto_get_wqe_sizes(max_segs_num, &umr_wqe_size,
867 &rdmw_wqe_size);
868 priv->wqe_set_size = rdmw_wqe_size + umr_wqe_size;
869 if (umr_wqe_size > max_wqe_size ||
870 rdmw_wqe_size > max_wqe_size) {
871 DRV_LOG(ERR, "Invalid max_segs_num: %u. should be %u or lower.",
872 max_segs_num,
873 mlx5_crypto_max_segs_num(max_wqe_size));
874 rte_errno = EINVAL;
875 return -EINVAL;
876 }
877 priv->umr_wqe_size = (uint16_t)umr_wqe_size;
878 priv->umr_wqe_stride = priv->umr_wqe_size / MLX5_SEND_WQE_BB;
879 priv->max_rdmar_ds = rdmw_wqe_size / sizeof(struct mlx5_wqe_dseg);
880 return 0;
881 }
882
883 static int
mlx5_crypto_dev_probe(struct mlx5_common_device * cdev,struct mlx5_kvargs_ctrl * mkvlist)884 mlx5_crypto_dev_probe(struct mlx5_common_device *cdev,
885 struct mlx5_kvargs_ctrl *mkvlist)
886 {
887 struct rte_cryptodev *crypto_dev;
888 struct mlx5_devx_obj *login;
889 struct mlx5_crypto_priv *priv;
890 struct mlx5_crypto_devarg_params devarg_prms = { 0 };
891 struct rte_cryptodev_pmd_init_params init_params = {
892 .name = "",
893 .private_data_size = sizeof(struct mlx5_crypto_priv),
894 .socket_id = cdev->dev->numa_node,
895 .max_nb_queue_pairs =
896 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
897 };
898 const char *ibdev_name = mlx5_os_get_ctx_device_name(cdev->ctx);
899 int ret;
900
901 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
902 DRV_LOG(ERR, "Non-primary process type is not supported.");
903 rte_errno = ENOTSUP;
904 return -rte_errno;
905 }
906 if (!cdev->config.hca_attr.crypto || !cdev->config.hca_attr.aes_xts) {
907 DRV_LOG(ERR, "Not enough capabilities to support crypto "
908 "operations, maybe old FW/OFED version?");
909 rte_errno = ENOTSUP;
910 return -ENOTSUP;
911 }
912 ret = mlx5_crypto_parse_devargs(mkvlist, &devarg_prms);
913 if (ret) {
914 DRV_LOG(ERR, "Failed to parse devargs.");
915 return -rte_errno;
916 }
917 crypto_dev = rte_cryptodev_pmd_create(ibdev_name, cdev->dev,
918 &init_params);
919 if (crypto_dev == NULL) {
920 DRV_LOG(ERR, "Failed to create device \"%s\".", ibdev_name);
921 return -ENODEV;
922 }
923 DRV_LOG(INFO,
924 "Crypto device %s was created successfully.", ibdev_name);
925 crypto_dev->dev_ops = &mlx5_crypto_ops;
926 crypto_dev->dequeue_burst = mlx5_crypto_dequeue_burst;
927 crypto_dev->enqueue_burst = mlx5_crypto_enqueue_burst;
928 crypto_dev->feature_flags = MLX5_CRYPTO_FEATURE_FLAGS;
929 crypto_dev->driver_id = mlx5_crypto_driver_id;
930 priv = crypto_dev->data->dev_private;
931 priv->cdev = cdev;
932 priv->crypto_dev = crypto_dev;
933 if (mlx5_devx_uar_prepare(cdev, &priv->uar) != 0) {
934 rte_cryptodev_pmd_destroy(priv->crypto_dev);
935 return -1;
936 }
937 login = mlx5_devx_cmd_create_crypto_login_obj(cdev->ctx,
938 &devarg_prms.login_attr);
939 if (login == NULL) {
940 DRV_LOG(ERR, "Failed to configure login.");
941 mlx5_devx_uar_release(&priv->uar);
942 rte_cryptodev_pmd_destroy(priv->crypto_dev);
943 return -rte_errno;
944 }
945 priv->login_obj = login;
946 priv->keytag = rte_cpu_to_be_64(devarg_prms.keytag);
947 ret = mlx5_crypto_configure_wqe_size(priv,
948 cdev->config.hca_attr.max_wqe_sz_sq, devarg_prms.max_segs_num);
949 if (ret) {
950 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
951 mlx5_devx_uar_release(&priv->uar);
952 rte_cryptodev_pmd_destroy(priv->crypto_dev);
953 return -1;
954 }
955 DRV_LOG(INFO, "Max number of segments: %u.",
956 (unsigned int)RTE_MIN(
957 MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
958 (uint16_t)(priv->max_rdmar_ds - 2)));
959 pthread_mutex_lock(&priv_list_lock);
960 TAILQ_INSERT_TAIL(&mlx5_crypto_priv_list, priv, next);
961 pthread_mutex_unlock(&priv_list_lock);
962
963 rte_cryptodev_pmd_probing_finish(crypto_dev);
964
965 return 0;
966 }
967
968 static int
mlx5_crypto_dev_remove(struct mlx5_common_device * cdev)969 mlx5_crypto_dev_remove(struct mlx5_common_device *cdev)
970 {
971 struct mlx5_crypto_priv *priv = NULL;
972
973 pthread_mutex_lock(&priv_list_lock);
974 TAILQ_FOREACH(priv, &mlx5_crypto_priv_list, next)
975 if (priv->crypto_dev->device == cdev->dev)
976 break;
977 if (priv)
978 TAILQ_REMOVE(&mlx5_crypto_priv_list, priv, next);
979 pthread_mutex_unlock(&priv_list_lock);
980 if (priv) {
981 claim_zero(mlx5_devx_cmd_destroy(priv->login_obj));
982 mlx5_devx_uar_release(&priv->uar);
983 rte_cryptodev_pmd_destroy(priv->crypto_dev);
984 }
985 return 0;
986 }
987
988 static const struct rte_pci_id mlx5_crypto_pci_id_map[] = {
989 {
990 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
991 PCI_DEVICE_ID_MELLANOX_CONNECTX6)
992 },
993 {
994 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
995 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX)
996 },
997 {
998 RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
999 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF)
1000 },
1001 {
1002 .vendor_id = 0
1003 }
1004 };
1005
1006 static struct mlx5_class_driver mlx5_crypto_driver = {
1007 .drv_class = MLX5_CLASS_CRYPTO,
1008 .name = RTE_STR(MLX5_CRYPTO_DRIVER_NAME),
1009 .id_table = mlx5_crypto_pci_id_map,
1010 .probe = mlx5_crypto_dev_probe,
1011 .remove = mlx5_crypto_dev_remove,
1012 };
1013
RTE_INIT(rte_mlx5_crypto_init)1014 RTE_INIT(rte_mlx5_crypto_init)
1015 {
1016 pthread_mutex_init(&priv_list_lock, NULL);
1017 mlx5_common_init();
1018 if (mlx5_glue != NULL)
1019 mlx5_class_driver_register(&mlx5_crypto_driver);
1020 }
1021
1022 RTE_PMD_REGISTER_CRYPTO_DRIVER(mlx5_cryptodev_driver, mlx5_drv,
1023 mlx5_crypto_driver_id);
1024
1025 RTE_LOG_REGISTER_DEFAULT(mlx5_crypto_logtype, NOTICE)
1026 RTE_PMD_EXPORT_NAME(MLX5_CRYPTO_DRIVER_NAME, __COUNTER__);
1027 RTE_PMD_REGISTER_PCI_TABLE(MLX5_CRYPTO_DRIVER_NAME, mlx5_crypto_pci_id_map);
1028 RTE_PMD_REGISTER_KMOD_DEP(MLX5_CRYPTO_DRIVER_NAME, "* ib_uverbs & mlx5_core & mlx5_ib");
1029