1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4 #include <rte_common.h>
5 #include <rte_hexdump.h>
6 #include <rte_cryptodev.h>
7 #include <cryptodev_pmd.h>
8 #include <rte_bus_vdev.h>
9 #include <rte_malloc.h>
10 #include <rte_cpuflags.h>
11 #include <rte_reorder.h>
12 #include <rte_string_fns.h>
13
14 #include "rte_cryptodev_scheduler.h"
15 #include "scheduler_pmd_private.h"
16
17 uint8_t cryptodev_scheduler_driver_id;
18
19 struct scheduler_init_params {
20 struct rte_cryptodev_pmd_init_params def_p;
21 uint32_t nb_workers;
22 enum rte_cryptodev_scheduler_mode mode;
23 char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
24 uint32_t enable_ordering;
25 uint16_t wc_pool[RTE_MAX_LCORE];
26 uint16_t nb_wc;
27 char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS]
28 [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
29 };
30
31 #define RTE_CRYPTODEV_VDEV_NAME ("name")
32 #define RTE_CRYPTODEV_VDEV_WORKER ("worker")
33 #define RTE_CRYPTODEV_VDEV_MODE ("mode")
34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM ("mode_param")
35 #define RTE_CRYPTODEV_VDEV_ORDERING ("ordering")
36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG ("max_nb_queue_pairs")
37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID ("socket_id")
38 #define RTE_CRYPTODEV_VDEV_COREMASK ("coremask")
39 #define RTE_CRYPTODEV_VDEV_CORELIST ("corelist")
40
41 static const char * const scheduler_valid_params[] = {
42 RTE_CRYPTODEV_VDEV_NAME,
43 RTE_CRYPTODEV_VDEV_WORKER,
44 RTE_CRYPTODEV_VDEV_MODE,
45 RTE_CRYPTODEV_VDEV_MODE_PARAM,
46 RTE_CRYPTODEV_VDEV_ORDERING,
47 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
48 RTE_CRYPTODEV_VDEV_SOCKET_ID,
49 RTE_CRYPTODEV_VDEV_COREMASK,
50 RTE_CRYPTODEV_VDEV_CORELIST
51 };
52
53 struct scheduler_parse_map {
54 const char *name;
55 uint32_t val;
56 };
57
58 const struct scheduler_parse_map scheduler_mode_map[] = {
59 {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
60 CDEV_SCHED_MODE_ROUNDROBIN},
61 {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
62 CDEV_SCHED_MODE_PKT_SIZE_DISTR},
63 {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
64 CDEV_SCHED_MODE_FAILOVER},
65 {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
66 CDEV_SCHED_MODE_MULTICORE}
67 };
68
69 const struct scheduler_parse_map scheduler_ordering_map[] = {
70 {"enable", 1},
71 {"disable", 0}
72 };
73
74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR ':'
75
76 static int
cryptodev_scheduler_create(const char * name,struct rte_vdev_device * vdev,struct scheduler_init_params * init_params)77 cryptodev_scheduler_create(const char *name,
78 struct rte_vdev_device *vdev,
79 struct scheduler_init_params *init_params)
80 {
81 struct rte_cryptodev *dev;
82 struct scheduler_ctx *sched_ctx;
83 uint32_t i;
84 int ret;
85
86 dev = rte_cryptodev_pmd_create(name, &vdev->device,
87 &init_params->def_p);
88 if (dev == NULL) {
89 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
90 name);
91 return -EFAULT;
92 }
93
94 dev->driver_id = cryptodev_scheduler_driver_id;
95 dev->dev_ops = rte_crypto_scheduler_pmd_ops;
96
97 sched_ctx = dev->data->dev_private;
98 sched_ctx->max_nb_queue_pairs =
99 init_params->def_p.max_nb_queue_pairs;
100
101 if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
102 uint16_t i;
103
104 sched_ctx->nb_wc = init_params->nb_wc;
105
106 for (i = 0; i < sched_ctx->nb_wc; i++) {
107 sched_ctx->wc_pool[i] = init_params->wc_pool[i];
108 CR_SCHED_LOG(INFO, " Worker core[%u]=%u added",
109 i, sched_ctx->wc_pool[i]);
110 }
111 }
112
113 if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
114 init_params->mode < CDEV_SCHED_MODE_COUNT) {
115 union {
116 struct rte_cryptodev_scheduler_threshold_option
117 threshold_option;
118 } option;
119 enum rte_cryptodev_schedule_option_type option_type;
120 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
121 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
122 char *s, *end;
123
124 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
125 init_params->mode);
126 if (ret < 0) {
127 rte_cryptodev_pmd_release_device(dev);
128 return ret;
129 }
130
131 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
132 if (scheduler_mode_map[i].val != sched_ctx->mode)
133 continue;
134
135 CR_SCHED_LOG(INFO, " Scheduling mode = %s",
136 scheduler_mode_map[i].name);
137 break;
138 }
139
140 if (strlen(init_params->mode_param_str) > 0) {
141 s = strchr(init_params->mode_param_str,
142 CDEV_SCHED_MODE_PARAM_SEP_CHAR);
143 if (s == NULL) {
144 CR_SCHED_LOG(ERR, "Invalid mode param");
145 return -EINVAL;
146 }
147
148 strlcpy(param_name, init_params->mode_param_str,
149 s - init_params->mode_param_str + 1);
150 s++;
151 strlcpy(param_val, s,
152 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
153
154 switch (init_params->mode) {
155 case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
156 if (strcmp(param_name,
157 RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
158 != 0) {
159 CR_SCHED_LOG(ERR, "Invalid mode param");
160 return -EINVAL;
161 }
162 option_type = CDEV_SCHED_OPTION_THRESHOLD;
163
164 option.threshold_option.threshold =
165 strtoul(param_val, &end, 0);
166 break;
167 default:
168 CR_SCHED_LOG(ERR, "Invalid mode param");
169 return -EINVAL;
170 }
171
172 if (sched_ctx->ops.option_set(dev, option_type,
173 (void *)&option) < 0) {
174 CR_SCHED_LOG(ERR, "Invalid mode param");
175 return -EINVAL;
176 }
177
178 RTE_LOG(INFO, PMD, " Sched mode param (%s = %s)\n",
179 param_name, param_val);
180 }
181 }
182
183 sched_ctx->reordering_enabled = init_params->enable_ordering;
184
185 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
186 if (scheduler_ordering_map[i].val !=
187 sched_ctx->reordering_enabled)
188 continue;
189
190 CR_SCHED_LOG(INFO, " Packet ordering = %s",
191 scheduler_ordering_map[i].name);
192
193 break;
194 }
195
196 for (i = 0; i < init_params->nb_workers; i++) {
197 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] =
198 rte_zmalloc_socket(
199 NULL,
200 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
201 SOCKET_ID_ANY);
202
203 if (!sched_ctx->init_worker_names[
204 sched_ctx->nb_init_workers]) {
205 CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
206 name);
207 return -ENOMEM;
208 }
209
210 strncpy(sched_ctx->init_worker_names[
211 sched_ctx->nb_init_workers],
212 init_params->worker_names[i],
213 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
214
215 sched_ctx->nb_init_workers++;
216 }
217
218 /*
219 * Initialize capabilities structure as an empty structure,
220 * in case device information is requested when no workers are attached
221 */
222 sched_ctx->capabilities = rte_zmalloc_socket(NULL,
223 sizeof(struct rte_cryptodev_capabilities),
224 0, SOCKET_ID_ANY);
225
226 if (!sched_ctx->capabilities) {
227 CR_SCHED_LOG(ERR, "Not enough memory for capability "
228 "information");
229 return -ENOMEM;
230 }
231
232 rte_cryptodev_pmd_probing_finish(dev);
233
234 return 0;
235 }
236
237 static int
cryptodev_scheduler_remove(struct rte_vdev_device * vdev)238 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
239 {
240 const char *name;
241 struct rte_cryptodev *dev;
242 struct scheduler_ctx *sched_ctx;
243
244 if (vdev == NULL)
245 return -EINVAL;
246
247 name = rte_vdev_device_name(vdev);
248 dev = rte_cryptodev_pmd_get_named_dev(name);
249 if (dev == NULL)
250 return -EINVAL;
251
252 sched_ctx = dev->data->dev_private;
253
254 if (sched_ctx->nb_workers) {
255 uint32_t i;
256
257 for (i = 0; i < sched_ctx->nb_workers; i++)
258 rte_cryptodev_scheduler_worker_detach(dev->data->dev_id,
259 sched_ctx->workers[i].dev_id);
260 }
261
262 return rte_cryptodev_pmd_destroy(dev);
263 }
264
265 /** Parse integer from integer argument */
266 static int
parse_integer_arg(const char * key __rte_unused,const char * value,void * extra_args)267 parse_integer_arg(const char *key __rte_unused,
268 const char *value, void *extra_args)
269 {
270 int *i = (int *) extra_args;
271
272 *i = atoi(value);
273 if (*i < 0) {
274 CR_SCHED_LOG(ERR, "Argument has to be positive.");
275 return -EINVAL;
276 }
277
278 return 0;
279 }
280
281 /** Parse integer from hexadecimal integer argument */
282 static int
parse_coremask_arg(const char * key __rte_unused,const char * value,void * extra_args)283 parse_coremask_arg(const char *key __rte_unused,
284 const char *value, void *extra_args)
285 {
286 int i, j, val;
287 uint16_t idx = 0;
288 char c;
289 struct scheduler_init_params *params = extra_args;
290
291 params->nb_wc = 0;
292
293 if (value == NULL)
294 return -1;
295 /* Remove all blank characters ahead and after .
296 * Remove 0x/0X if exists.
297 */
298 while (isblank(*value))
299 value++;
300 if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
301 value += 2;
302 i = strlen(value);
303 while ((i > 0) && isblank(value[i - 1]))
304 i--;
305
306 if (i == 0)
307 return -1;
308
309 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
310 c = value[i];
311 if (isxdigit(c) == 0) {
312 /* invalid characters */
313 return -1;
314 }
315 if (isdigit(c))
316 val = c - '0';
317 else if (isupper(c))
318 val = c - 'A' + 10;
319 else
320 val = c - 'a' + 10;
321
322 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
323 if ((1 << j) & val)
324 params->wc_pool[params->nb_wc++] = idx;
325 }
326 }
327
328 return 0;
329 }
330
331 /** Parse integer from list of integers argument */
332 static int
parse_corelist_arg(const char * key __rte_unused,const char * value,void * extra_args)333 parse_corelist_arg(const char *key __rte_unused,
334 const char *value, void *extra_args)
335 {
336 struct scheduler_init_params *params = extra_args;
337
338 params->nb_wc = 0;
339
340 const char *token = value;
341
342 while (isdigit(token[0])) {
343 char *rval;
344 unsigned int core = strtoul(token, &rval, 10);
345
346 if (core >= RTE_MAX_LCORE) {
347 CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
348 "than %u.", core, RTE_MAX_LCORE);
349 }
350 params->wc_pool[params->nb_wc++] = (uint16_t)core;
351 token = (const char *)rval;
352 if (token[0] == '\0')
353 break;
354 token++;
355 }
356
357 return 0;
358 }
359
360 /** Parse name */
361 static int
parse_name_arg(const char * key __rte_unused,const char * value,void * extra_args)362 parse_name_arg(const char *key __rte_unused,
363 const char *value, void *extra_args)
364 {
365 struct rte_cryptodev_pmd_init_params *params = extra_args;
366
367 if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
368 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
369 "%u bytes.", value,
370 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
371 return -EINVAL;
372 }
373
374 strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
375
376 return 0;
377 }
378
379 /** Parse worker */
380 static int
parse_worker_arg(const char * key __rte_unused,const char * value,void * extra_args)381 parse_worker_arg(const char *key __rte_unused,
382 const char *value, void *extra_args)
383 {
384 struct scheduler_init_params *param = extra_args;
385
386 if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) {
387 CR_SCHED_LOG(ERR, "Too many workers.");
388 return -ENOMEM;
389 }
390
391 strncpy(param->worker_names[param->nb_workers++], value,
392 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
393
394 return 0;
395 }
396
397 static int
parse_mode_arg(const char * key __rte_unused,const char * value,void * extra_args)398 parse_mode_arg(const char *key __rte_unused,
399 const char *value, void *extra_args)
400 {
401 struct scheduler_init_params *param = extra_args;
402 uint32_t i;
403
404 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
405 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
406 param->mode = (enum rte_cryptodev_scheduler_mode)
407 scheduler_mode_map[i].val;
408
409 break;
410 }
411 }
412
413 if (i == RTE_DIM(scheduler_mode_map)) {
414 CR_SCHED_LOG(ERR, "Unrecognized input.");
415 return -EINVAL;
416 }
417
418 return 0;
419 }
420
421 static int
parse_mode_param_arg(const char * key __rte_unused,const char * value,void * extra_args)422 parse_mode_param_arg(const char *key __rte_unused,
423 const char *value, void *extra_args)
424 {
425 struct scheduler_init_params *param = extra_args;
426
427 strlcpy(param->mode_param_str, value,
428 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
429
430 return 0;
431 }
432
433 static int
parse_ordering_arg(const char * key __rte_unused,const char * value,void * extra_args)434 parse_ordering_arg(const char *key __rte_unused,
435 const char *value, void *extra_args)
436 {
437 struct scheduler_init_params *param = extra_args;
438 uint32_t i;
439
440 for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
441 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
442 param->enable_ordering =
443 scheduler_ordering_map[i].val;
444 break;
445 }
446 }
447
448 if (i == RTE_DIM(scheduler_ordering_map)) {
449 CR_SCHED_LOG(ERR, "Unrecognized input.");
450 return -EINVAL;
451 }
452
453 return 0;
454 }
455
456 static int
scheduler_parse_init_params(struct scheduler_init_params * params,const char * input_args)457 scheduler_parse_init_params(struct scheduler_init_params *params,
458 const char *input_args)
459 {
460 struct rte_kvargs *kvlist = NULL;
461 int ret = 0;
462
463 if (params == NULL)
464 return -EINVAL;
465
466 if (input_args) {
467 kvlist = rte_kvargs_parse(input_args,
468 scheduler_valid_params);
469 if (kvlist == NULL)
470 return -1;
471
472 ret = rte_kvargs_process(kvlist,
473 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
474 &parse_integer_arg,
475 ¶ms->def_p.max_nb_queue_pairs);
476 if (ret < 0)
477 goto free_kvlist;
478
479 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
480 &parse_integer_arg,
481 ¶ms->def_p.socket_id);
482 if (ret < 0)
483 goto free_kvlist;
484
485 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
486 &parse_coremask_arg,
487 params);
488 if (ret < 0)
489 goto free_kvlist;
490
491 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
492 &parse_corelist_arg,
493 params);
494 if (ret < 0)
495 goto free_kvlist;
496
497 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
498 &parse_name_arg,
499 ¶ms->def_p);
500 if (ret < 0)
501 goto free_kvlist;
502
503 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER,
504 &parse_worker_arg, params);
505 if (ret < 0)
506 goto free_kvlist;
507
508 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
509 &parse_mode_arg, params);
510 if (ret < 0)
511 goto free_kvlist;
512
513 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
514 &parse_mode_param_arg, params);
515 if (ret < 0)
516 goto free_kvlist;
517
518 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
519 &parse_ordering_arg, params);
520 if (ret < 0)
521 goto free_kvlist;
522 }
523
524 free_kvlist:
525 rte_kvargs_free(kvlist);
526 return ret;
527 }
528
529 static int
cryptodev_scheduler_probe(struct rte_vdev_device * vdev)530 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
531 {
532 struct scheduler_init_params init_params = {
533 .def_p = {
534 "",
535 sizeof(struct scheduler_ctx),
536 rte_socket_id(),
537 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
538 },
539 .nb_workers = 0,
540 .mode = CDEV_SCHED_MODE_NOT_SET,
541 .enable_ordering = 0,
542 .worker_names = { {0} }
543 };
544 const char *name;
545
546 name = rte_vdev_device_name(vdev);
547 if (name == NULL)
548 return -EINVAL;
549
550 scheduler_parse_init_params(&init_params,
551 rte_vdev_device_args(vdev));
552
553
554 return cryptodev_scheduler_create(name,
555 vdev,
556 &init_params);
557 }
558
559 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
560 .probe = cryptodev_scheduler_probe,
561 .remove = cryptodev_scheduler_remove
562 };
563
564 static struct cryptodev_driver scheduler_crypto_drv;
565
566 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
567 cryptodev_scheduler_pmd_drv);
568 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
569 "max_nb_queue_pairs=<int> "
570 "socket_id=<int> "
571 "worker=<name>");
572 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
573 cryptodev_scheduler_pmd_drv.driver,
574 cryptodev_scheduler_driver_id);
575