1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Broadcom
3 * All rights reserved.
4 */
5
6 #include <rte_common.h>
7 #include <rte_dev.h>
8 #include <rte_errno.h>
9 #include <rte_malloc.h>
10 #include <rte_cryptodev_pmd.h>
11
12 #include "bcmfs_device.h"
13 #include "bcmfs_logs.h"
14 #include "bcmfs_qp.h"
15 #include "bcmfs_sym_pmd.h"
16 #include "bcmfs_sym_req.h"
17 #include "bcmfs_sym_session.h"
18 #include "bcmfs_sym_capabilities.h"
19
20 uint8_t cryptodev_bcmfs_driver_id;
21
22 static int bcmfs_sym_qp_release(struct rte_cryptodev *dev,
23 uint16_t queue_pair_id);
24
25 static int
bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev * dev,__rte_unused struct rte_cryptodev_config * config)26 bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
27 __rte_unused struct rte_cryptodev_config *config)
28 {
29 return 0;
30 }
31
32 static int
bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev * dev)33 bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
34 {
35 return 0;
36 }
37
38 static void
bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev * dev)39 bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
40 {
41 }
42
43 static int
bcmfs_sym_dev_close(struct rte_cryptodev * dev)44 bcmfs_sym_dev_close(struct rte_cryptodev *dev)
45 {
46 int i, ret;
47
48 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
49 ret = bcmfs_sym_qp_release(dev, i);
50 if (ret < 0)
51 return ret;
52 }
53
54 return 0;
55 }
56
57 static void
bcmfs_sym_dev_info_get(struct rte_cryptodev * dev,struct rte_cryptodev_info * dev_info)58 bcmfs_sym_dev_info_get(struct rte_cryptodev *dev,
59 struct rte_cryptodev_info *dev_info)
60 {
61 struct bcmfs_sym_dev_private *internals = dev->data->dev_private;
62 struct bcmfs_device *fsdev = internals->fsdev;
63
64 if (dev_info != NULL) {
65 dev_info->driver_id = cryptodev_bcmfs_driver_id;
66 dev_info->feature_flags = dev->feature_flags;
67 dev_info->max_nb_queue_pairs = fsdev->max_hw_qps;
68 /* No limit of number of sessions */
69 dev_info->sym.max_nb_sessions = 0;
70 dev_info->capabilities = bcmfs_sym_get_capabilities();
71 }
72 }
73
74 static void
bcmfs_sym_stats_get(struct rte_cryptodev * dev,struct rte_cryptodev_stats * stats)75 bcmfs_sym_stats_get(struct rte_cryptodev *dev,
76 struct rte_cryptodev_stats *stats)
77 {
78 struct bcmfs_qp_stats bcmfs_stats = {0};
79 struct bcmfs_sym_dev_private *bcmfs_priv;
80 struct bcmfs_device *fsdev;
81
82 if (stats == NULL || dev == NULL) {
83 BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
84 return;
85 }
86 bcmfs_priv = dev->data->dev_private;
87 fsdev = bcmfs_priv->fsdev;
88
89 bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats);
90
91 stats->enqueued_count = bcmfs_stats.enqueued_count;
92 stats->dequeued_count = bcmfs_stats.dequeued_count;
93 stats->enqueue_err_count = bcmfs_stats.enqueue_err_count;
94 stats->dequeue_err_count = bcmfs_stats.dequeue_err_count;
95 }
96
97 static void
bcmfs_sym_stats_reset(struct rte_cryptodev * dev)98 bcmfs_sym_stats_reset(struct rte_cryptodev *dev)
99 {
100 struct bcmfs_sym_dev_private *bcmfs_priv;
101 struct bcmfs_device *fsdev;
102
103 if (dev == NULL) {
104 BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev);
105 return;
106 }
107 bcmfs_priv = dev->data->dev_private;
108 fsdev = bcmfs_priv->fsdev;
109
110 bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps);
111 }
112
113 static int
bcmfs_sym_qp_release(struct rte_cryptodev * dev,uint16_t queue_pair_id)114 bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
115 {
116 struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private;
117 struct bcmfs_qp *qp = (struct bcmfs_qp *)
118 (dev->data->queue_pairs[queue_pair_id]);
119
120 BCMFS_LOG(DEBUG, "Release sym qp %u on device %d",
121 queue_pair_id, dev->data->dev_id);
122
123 rte_mempool_free(qp->sr_mp);
124
125 bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL;
126
127 return bcmfs_qp_release((struct bcmfs_qp **)
128 &dev->data->queue_pairs[queue_pair_id]);
129 }
130
131 static void
spu_req_init(struct bcmfs_sym_request * sr,rte_iova_t iova __rte_unused)132 spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused)
133 {
134 memset(sr, 0, sizeof(*sr));
135 sr->fptr = iova + offsetof(struct bcmfs_sym_request, fmd);
136 sr->optr = iova + offsetof(struct bcmfs_sym_request, omd);
137 sr->dptr = iova + offsetof(struct bcmfs_sym_request, digest);
138 sr->rptr = iova + offsetof(struct bcmfs_sym_request, resp);
139 }
140
141 static void
req_pool_obj_init(__rte_unused struct rte_mempool * mp,__rte_unused void * opaque,void * obj,__rte_unused unsigned int obj_idx)142 req_pool_obj_init(__rte_unused struct rte_mempool *mp,
143 __rte_unused void *opaque, void *obj,
144 __rte_unused unsigned int obj_idx)
145 {
146 spu_req_init(obj, rte_mempool_virt2iova(obj));
147 }
148
149 static struct rte_mempool *
bcmfs_sym_req_pool_create(struct rte_cryptodev * cdev __rte_unused,uint32_t nobjs,uint16_t qp_id,int socket_id)150 bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused,
151 uint32_t nobjs, uint16_t qp_id,
152 int socket_id)
153 {
154 char softreq_pool_name[RTE_RING_NAMESIZE];
155 struct rte_mempool *mp;
156
157 snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d",
158 "bcm_sym", qp_id);
159
160 mp = rte_mempool_create(softreq_pool_name,
161 RTE_ALIGN_MUL_CEIL(nobjs, 64),
162 sizeof(struct bcmfs_sym_request),
163 64, 0, NULL, NULL, req_pool_obj_init, NULL,
164 socket_id, 0);
165 if (mp == NULL)
166 BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d",
167 qp_id, rte_errno);
168
169 return mp;
170 }
171
172 static int
bcmfs_sym_qp_setup(struct rte_cryptodev * cdev,uint16_t qp_id,const struct rte_cryptodev_qp_conf * qp_conf,int socket_id)173 bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id,
174 const struct rte_cryptodev_qp_conf *qp_conf,
175 int socket_id)
176 {
177 int ret = 0;
178 struct bcmfs_qp *qp = NULL;
179 struct bcmfs_qp_config bcmfs_qp_conf;
180
181 struct bcmfs_qp **qp_addr =
182 (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id];
183 struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private;
184 struct bcmfs_device *fsdev = bcmfs_private->fsdev;
185
186
187 /* If qp is already in use free ring memory and qp metadata. */
188 if (*qp_addr != NULL) {
189 ret = bcmfs_sym_qp_release(cdev, qp_id);
190 if (ret < 0)
191 return ret;
192 }
193
194 if (qp_id >= fsdev->max_hw_qps) {
195 BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id);
196 return -EINVAL;
197 }
198
199 bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
200 bcmfs_qp_conf.socket_id = socket_id;
201 bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ;
202 bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id);
203 bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops;
204
205 ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf);
206 if (ret != 0)
207 return ret;
208
209 qp = (struct bcmfs_qp *)*qp_addr;
210
211 qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors,
212 qp_id, socket_id);
213 if (qp->sr_mp == NULL)
214 return -ENOMEM;
215
216 /* store a link to the qp in the bcmfs_device */
217 bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr;
218
219 cdev->data->queue_pairs[qp_id] = qp;
220 BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id);
221
222 return 0;
223 }
224
225 static struct rte_cryptodev_ops crypto_bcmfs_ops = {
226 /* Device related operations */
227 .dev_configure = bcmfs_sym_dev_config,
228 .dev_start = bcmfs_sym_dev_start,
229 .dev_stop = bcmfs_sym_dev_stop,
230 .dev_close = bcmfs_sym_dev_close,
231 .dev_infos_get = bcmfs_sym_dev_info_get,
232 /* Stats Collection */
233 .stats_get = bcmfs_sym_stats_get,
234 .stats_reset = bcmfs_sym_stats_reset,
235 /* Queue-Pair management */
236 .queue_pair_setup = bcmfs_sym_qp_setup,
237 .queue_pair_release = bcmfs_sym_qp_release,
238 /* Crypto session related operations */
239 .sym_session_get_size = bcmfs_sym_session_get_private_size,
240 .sym_session_configure = bcmfs_sym_session_configure,
241 .sym_session_clear = bcmfs_sym_session_clear
242 };
243
244 /** Enqueue burst */
245 static uint16_t
bcmfs_sym_pmd_enqueue_op_burst(void * queue_pair,struct rte_crypto_op ** ops,uint16_t nb_ops)246 bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair,
247 struct rte_crypto_op **ops,
248 uint16_t nb_ops)
249 {
250 int i, j;
251 int retval;
252 uint16_t enq = 0;
253 struct bcmfs_sym_request *sreq;
254 struct bcmfs_sym_session *sess;
255 struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair;
256
257 if (nb_ops == 0)
258 return 0;
259
260 if (nb_ops > BCMFS_MAX_REQS_BUFF)
261 nb_ops = BCMFS_MAX_REQS_BUFF;
262
263 /* We do not process more than available space */
264 if (nb_ops > (qp->nb_descriptors - qp->nb_pending_requests))
265 nb_ops = qp->nb_descriptors - qp->nb_pending_requests;
266
267 for (i = 0; i < nb_ops; i++) {
268 sess = bcmfs_sym_get_session(ops[i]);
269 if (unlikely(sess == NULL))
270 goto enqueue_err;
271
272 if (rte_mempool_get(qp->sr_mp, (void **)&sreq))
273 goto enqueue_err;
274
275 /* save rte_crypto_op */
276 sreq->op = ops[i];
277
278 /* save context */
279 qp->infl_msgs[i] = &sreq->msgs;
280 qp->infl_msgs[i]->ctx = (void *)sreq;
281
282 /* pre process the request crypto h/w acceleration */
283 retval = bcmfs_process_sym_crypto_op(ops[i], sess, sreq);
284 if (unlikely(retval < 0))
285 goto enqueue_err;
286 }
287 /* Send burst request to hw QP */
288 enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i);
289
290 for (j = enq; j < i; j++)
291 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
292
293 return enq;
294
295 enqueue_err:
296 for (j = 0; j < i; j++)
297 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
298
299 return enq;
300 }
301
bcmfs_sym_set_request_status(struct rte_crypto_op * op,struct bcmfs_sym_request * out)302 static void bcmfs_sym_set_request_status(struct rte_crypto_op *op,
303 struct bcmfs_sym_request *out)
304 {
305 if (*out->resp == BCMFS_SYM_RESPONSE_SUCCESS)
306 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
307 else if (*out->resp == BCMFS_SYM_RESPONSE_HASH_TAG_ERROR)
308 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
309 else
310 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
311 }
312
313 static uint16_t
bcmfs_sym_pmd_dequeue_op_burst(void * queue_pair,struct rte_crypto_op ** ops,uint16_t nb_ops)314 bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair,
315 struct rte_crypto_op **ops,
316 uint16_t nb_ops)
317 {
318 int i;
319 uint16_t deq = 0;
320 unsigned int pkts = 0;
321 struct bcmfs_sym_request *sreq;
322 struct bcmfs_qp *qp = queue_pair;
323
324 if (nb_ops > BCMFS_MAX_REQS_BUFF)
325 nb_ops = BCMFS_MAX_REQS_BUFF;
326
327 deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops);
328 /* get rte_crypto_ops */
329 for (i = 0; i < deq; i++) {
330 sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx;
331
332 /* set the status based on the response from the crypto h/w */
333 bcmfs_sym_set_request_status(sreq->op, sreq);
334
335 ops[pkts++] = sreq->op;
336
337 rte_mempool_put(qp->sr_mp, sreq);
338 }
339
340 return pkts;
341 }
342
343 /*
344 * An rte_driver is needed in the registration of both the
345 * device and the driver with cryptodev.
346 */
347 static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD);
348 static const struct rte_driver cryptodev_bcmfs_sym_driver = {
349 .name = bcmfs_sym_drv_name,
350 .alias = bcmfs_sym_drv_name
351 };
352
353 int
bcmfs_sym_dev_create(struct bcmfs_device * fsdev)354 bcmfs_sym_dev_create(struct bcmfs_device *fsdev)
355 {
356 struct rte_cryptodev_pmd_init_params init_params = {
357 .name = "",
358 .socket_id = rte_socket_id(),
359 .private_data_size = sizeof(struct bcmfs_sym_dev_private)
360 };
361 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
362 struct rte_cryptodev *cryptodev;
363 struct bcmfs_sym_dev_private *internals;
364
365 snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
366 fsdev->name, "sym");
367
368 /* Populate subset device to use in cryptodev device creation */
369 fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver;
370 fsdev->sym_rte_dev.numa_node = 0;
371 fsdev->sym_rte_dev.devargs = NULL;
372
373 cryptodev = rte_cryptodev_pmd_create(name,
374 &fsdev->sym_rte_dev,
375 &init_params);
376 if (cryptodev == NULL)
377 return -ENODEV;
378
379 fsdev->sym_rte_dev.name = cryptodev->data->name;
380 cryptodev->driver_id = cryptodev_bcmfs_driver_id;
381 cryptodev->dev_ops = &crypto_bcmfs_ops;
382
383 cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst;
384 cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst;
385
386 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
387 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
388 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
389
390 internals = cryptodev->data->dev_private;
391 internals->fsdev = fsdev;
392 fsdev->sym_dev = internals;
393
394 internals->sym_dev_id = cryptodev->data->dev_id;
395 internals->fsdev_capabilities = bcmfs_sym_get_capabilities();
396
397 BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d",
398 cryptodev->data->name, internals->sym_dev_id);
399 return 0;
400 }
401
402 int
bcmfs_sym_dev_destroy(struct bcmfs_device * fsdev)403 bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev)
404 {
405 struct rte_cryptodev *cryptodev;
406
407 if (fsdev == NULL)
408 return -ENODEV;
409 if (fsdev->sym_dev == NULL)
410 return 0;
411
412 /* free crypto device */
413 cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id);
414 rte_cryptodev_pmd_destroy(cryptodev);
415 fsdev->sym_rte_dev.name = NULL;
416 fsdev->sym_dev = NULL;
417
418 return 0;
419 }
420
421 static struct cryptodev_driver bcmfs_crypto_drv;
422 RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv,
423 cryptodev_bcmfs_sym_driver,
424 cryptodev_bcmfs_driver_id);
425