1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Intel Corporation nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32 #include <rte_common.h>
33 #include <rte_hexdump.h>
34 #include <rte_cryptodev.h>
35 #include <rte_cryptodev_pmd.h>
36 #include <rte_bus_vdev.h>
37 #include <rte_malloc.h>
38 #include <rte_cpuflags.h>
39 #include <rte_reorder.h>
40 
41 #include "rte_cryptodev_scheduler.h"
42 #include "scheduler_pmd_private.h"
43 
44 uint8_t cryptodev_driver_id;
45 
46 struct scheduler_init_params {
47 	struct rte_cryptodev_pmd_init_params def_p;
48 	uint32_t nb_slaves;
49 	enum rte_cryptodev_scheduler_mode mode;
50 	uint32_t enable_ordering;
51 	uint16_t wc_pool[RTE_MAX_LCORE];
52 	uint16_t nb_wc;
53 	char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
54 			[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
55 };
56 
57 #define RTE_CRYPTODEV_VDEV_NAME			("name")
58 #define RTE_CRYPTODEV_VDEV_SLAVE		("slave")
59 #define RTE_CRYPTODEV_VDEV_MODE			("mode")
60 #define RTE_CRYPTODEV_VDEV_ORDERING		("ordering")
61 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG	("max_nb_queue_pairs")
62 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG	("max_nb_sessions")
63 #define RTE_CRYPTODEV_VDEV_SOCKET_ID		("socket_id")
64 #define RTE_CRYPTODEV_VDEV_COREMASK		("coremask")
65 #define RTE_CRYPTODEV_VDEV_CORELIST		("corelist")
66 
67 const char *scheduler_valid_params[] = {
68 	RTE_CRYPTODEV_VDEV_NAME,
69 	RTE_CRYPTODEV_VDEV_SLAVE,
70 	RTE_CRYPTODEV_VDEV_MODE,
71 	RTE_CRYPTODEV_VDEV_ORDERING,
72 	RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
73 	RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
74 	RTE_CRYPTODEV_VDEV_SOCKET_ID,
75 	RTE_CRYPTODEV_VDEV_COREMASK,
76 	RTE_CRYPTODEV_VDEV_CORELIST
77 };
78 
79 struct scheduler_parse_map {
80 	const char *name;
81 	uint32_t val;
82 };
83 
84 const struct scheduler_parse_map scheduler_mode_map[] = {
85 	{RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
86 			CDEV_SCHED_MODE_ROUNDROBIN},
87 	{RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
88 			CDEV_SCHED_MODE_PKT_SIZE_DISTR},
89 	{RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
90 			CDEV_SCHED_MODE_FAILOVER},
91 	{RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
92 			CDEV_SCHED_MODE_MULTICORE}
93 };
94 
95 const struct scheduler_parse_map scheduler_ordering_map[] = {
96 		{"enable", 1},
97 		{"disable", 0}
98 };
99 
100 static int
101 cryptodev_scheduler_create(const char *name,
102 		struct rte_vdev_device *vdev,
103 		struct scheduler_init_params *init_params)
104 {
105 	struct rte_cryptodev *dev;
106 	struct scheduler_ctx *sched_ctx;
107 	uint32_t i;
108 	int ret;
109 
110 	dev = rte_cryptodev_pmd_create(name, &vdev->device,
111 			&init_params->def_p);
112 	if (dev == NULL) {
113 		CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
114 			name);
115 		return -EFAULT;
116 	}
117 
118 	dev->driver_id = cryptodev_driver_id;
119 	dev->dev_ops = rte_crypto_scheduler_pmd_ops;
120 
121 	sched_ctx = dev->data->dev_private;
122 	sched_ctx->max_nb_queue_pairs =
123 			init_params->def_p.max_nb_queue_pairs;
124 
125 	if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
126 		uint16_t i;
127 
128 		sched_ctx->nb_wc = init_params->nb_wc;
129 
130 		for (i = 0; i < sched_ctx->nb_wc; i++) {
131 			sched_ctx->wc_pool[i] = init_params->wc_pool[i];
132 			RTE_LOG(INFO, PMD, "  Worker core[%u]=%u added\n",
133 				i, sched_ctx->wc_pool[i]);
134 		}
135 	}
136 
137 	if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
138 			init_params->mode < CDEV_SCHED_MODE_COUNT) {
139 		ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
140 			init_params->mode);
141 		if (ret < 0) {
142 			rte_cryptodev_pmd_release_device(dev);
143 			return ret;
144 		}
145 
146 		for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
147 			if (scheduler_mode_map[i].val != sched_ctx->mode)
148 				continue;
149 
150 			RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
151 					scheduler_mode_map[i].name);
152 			break;
153 		}
154 	}
155 
156 	sched_ctx->reordering_enabled = init_params->enable_ordering;
157 
158 	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
159 		if (scheduler_ordering_map[i].val !=
160 				sched_ctx->reordering_enabled)
161 			continue;
162 
163 		RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
164 				scheduler_ordering_map[i].name);
165 
166 		break;
167 	}
168 
169 	for (i = 0; i < init_params->nb_slaves; i++) {
170 		sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
171 			rte_zmalloc_socket(
172 				NULL,
173 				RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
174 				SOCKET_ID_ANY);
175 
176 		if (!sched_ctx->init_slave_names[
177 				sched_ctx->nb_init_slaves]) {
178 			CS_LOG_ERR("driver %s: Insufficient memory",
179 					name);
180 			return -ENOMEM;
181 		}
182 
183 		strncpy(sched_ctx->init_slave_names[
184 					sched_ctx->nb_init_slaves],
185 				init_params->slave_names[i],
186 				RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
187 
188 		sched_ctx->nb_init_slaves++;
189 	}
190 
191 	/*
192 	 * Initialize capabilities structure as an empty structure,
193 	 * in case device information is requested when no slaves are attached
194 	 */
195 	sched_ctx->capabilities = rte_zmalloc_socket(NULL,
196 			sizeof(struct rte_cryptodev_capabilities),
197 			0, SOCKET_ID_ANY);
198 
199 	if (!sched_ctx->capabilities) {
200 		RTE_LOG(ERR, PMD, "Not enough memory for capability "
201 				"information\n");
202 		return -ENOMEM;
203 	}
204 
205 	return 0;
206 }
207 
208 static int
209 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
210 {
211 	const char *name;
212 	struct rte_cryptodev *dev;
213 	struct scheduler_ctx *sched_ctx;
214 
215 	if (vdev == NULL)
216 		return -EINVAL;
217 
218 	name = rte_vdev_device_name(vdev);
219 	dev = rte_cryptodev_pmd_get_named_dev(name);
220 	if (dev == NULL)
221 		return -EINVAL;
222 
223 	sched_ctx = dev->data->dev_private;
224 
225 	if (sched_ctx->nb_slaves) {
226 		uint32_t i;
227 
228 		for (i = 0; i < sched_ctx->nb_slaves; i++)
229 			rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
230 					sched_ctx->slaves[i].dev_id);
231 	}
232 
233 	return rte_cryptodev_pmd_destroy(dev);
234 }
235 
236 /** Parse integer from integer argument */
237 static int
238 parse_integer_arg(const char *key __rte_unused,
239 		const char *value, void *extra_args)
240 {
241 	int *i = (int *) extra_args;
242 
243 	*i = atoi(value);
244 	if (*i < 0) {
245 		CS_LOG_ERR("Argument has to be positive.\n");
246 		return -EINVAL;
247 	}
248 
249 	return 0;
250 }
251 
252 /** Parse integer from hexadecimal integer argument */
253 static int
254 parse_coremask_arg(const char *key __rte_unused,
255 		const char *value, void *extra_args)
256 {
257 	int i, j, val;
258 	uint16_t idx = 0;
259 	char c;
260 	struct scheduler_init_params *params = extra_args;
261 
262 	params->nb_wc = 0;
263 
264 	if (value == NULL)
265 		return -1;
266 	/* Remove all blank characters ahead and after .
267 	 * Remove 0x/0X if exists.
268 	 */
269 	while (isblank(*value))
270 		value++;
271 	if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
272 		value += 2;
273 	i = strlen(value);
274 	while ((i > 0) && isblank(value[i - 1]))
275 		i--;
276 
277 	if (i == 0)
278 		return -1;
279 
280 	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
281 		c = value[i];
282 		if (isxdigit(c) == 0) {
283 			/* invalid characters */
284 			return -1;
285 		}
286 		if (isdigit(c))
287 			val = c - '0';
288 		else if (isupper(c))
289 			val = c - 'A' + 10;
290 		else
291 			val = c - 'a' + 10;
292 
293 		for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
294 			if ((1 << j) & val)
295 				params->wc_pool[params->nb_wc++] = idx;
296 		}
297 	}
298 
299 	return 0;
300 }
301 
302 /** Parse integer from list of integers argument */
303 static int
304 parse_corelist_arg(const char *key __rte_unused,
305 		const char *value, void *extra_args)
306 {
307 	struct scheduler_init_params *params = extra_args;
308 
309 	params->nb_wc = 0;
310 
311 	const char *token = value;
312 
313 	while (isdigit(token[0])) {
314 		char *rval;
315 		unsigned int core = strtoul(token, &rval, 10);
316 
317 		if (core >= RTE_MAX_LCORE) {
318 			CS_LOG_ERR("Invalid worker core %u, should be smaller "
319 				   "than %u.\n", core, RTE_MAX_LCORE);
320 		}
321 		params->wc_pool[params->nb_wc++] = (uint16_t)core;
322 		token = (const char *)rval;
323 		if (token[0] == '\0')
324 			break;
325 		token++;
326 	}
327 
328 	return 0;
329 }
330 
331 /** Parse name */
332 static int
333 parse_name_arg(const char *key __rte_unused,
334 		const char *value, void *extra_args)
335 {
336 	struct rte_cryptodev_pmd_init_params *params = extra_args;
337 
338 	if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
339 		CS_LOG_ERR("Invalid name %s, should be less than "
340 				"%u bytes.\n", value,
341 				RTE_CRYPTODEV_NAME_MAX_LEN - 1);
342 		return -EINVAL;
343 	}
344 
345 	strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
346 
347 	return 0;
348 }
349 
350 /** Parse slave */
351 static int
352 parse_slave_arg(const char *key __rte_unused,
353 		const char *value, void *extra_args)
354 {
355 	struct scheduler_init_params *param = extra_args;
356 
357 	if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
358 		CS_LOG_ERR("Too many slaves.\n");
359 		return -ENOMEM;
360 	}
361 
362 	strncpy(param->slave_names[param->nb_slaves++], value,
363 			RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
364 
365 	return 0;
366 }
367 
368 static int
369 parse_mode_arg(const char *key __rte_unused,
370 		const char *value, void *extra_args)
371 {
372 	struct scheduler_init_params *param = extra_args;
373 	uint32_t i;
374 
375 	for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
376 		if (strcmp(value, scheduler_mode_map[i].name) == 0) {
377 			param->mode = (enum rte_cryptodev_scheduler_mode)
378 					scheduler_mode_map[i].val;
379 			break;
380 		}
381 	}
382 
383 	if (i == RTE_DIM(scheduler_mode_map)) {
384 		CS_LOG_ERR("Unrecognized input.\n");
385 		return -EINVAL;
386 	}
387 
388 	return 0;
389 }
390 
391 static int
392 parse_ordering_arg(const char *key __rte_unused,
393 		const char *value, void *extra_args)
394 {
395 	struct scheduler_init_params *param = extra_args;
396 	uint32_t i;
397 
398 	for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
399 		if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
400 			param->enable_ordering =
401 					scheduler_ordering_map[i].val;
402 			break;
403 		}
404 	}
405 
406 	if (i == RTE_DIM(scheduler_ordering_map)) {
407 		CS_LOG_ERR("Unrecognized input.\n");
408 		return -EINVAL;
409 	}
410 
411 	return 0;
412 }
413 
414 static int
415 scheduler_parse_init_params(struct scheduler_init_params *params,
416 		const char *input_args)
417 {
418 	struct rte_kvargs *kvlist = NULL;
419 	int ret = 0;
420 
421 	if (params == NULL)
422 		return -EINVAL;
423 
424 	if (input_args) {
425 		kvlist = rte_kvargs_parse(input_args,
426 				scheduler_valid_params);
427 		if (kvlist == NULL)
428 			return -1;
429 
430 		ret = rte_kvargs_process(kvlist,
431 				RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
432 				&parse_integer_arg,
433 				&params->def_p.max_nb_queue_pairs);
434 		if (ret < 0)
435 			goto free_kvlist;
436 
437 		ret = rte_kvargs_process(kvlist,
438 				RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
439 				&parse_integer_arg,
440 				&params->def_p.max_nb_sessions);
441 		if (ret < 0)
442 			goto free_kvlist;
443 
444 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
445 				&parse_integer_arg,
446 				&params->def_p.socket_id);
447 		if (ret < 0)
448 			goto free_kvlist;
449 
450 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
451 				&parse_coremask_arg,
452 				params);
453 		if (ret < 0)
454 			goto free_kvlist;
455 
456 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
457 				&parse_corelist_arg,
458 				params);
459 		if (ret < 0)
460 			goto free_kvlist;
461 
462 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
463 				&parse_name_arg,
464 				&params->def_p);
465 		if (ret < 0)
466 			goto free_kvlist;
467 
468 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
469 				&parse_slave_arg, params);
470 		if (ret < 0)
471 			goto free_kvlist;
472 
473 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
474 				&parse_mode_arg, params);
475 		if (ret < 0)
476 			goto free_kvlist;
477 
478 		ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
479 				&parse_ordering_arg, params);
480 		if (ret < 0)
481 			goto free_kvlist;
482 	}
483 
484 free_kvlist:
485 	rte_kvargs_free(kvlist);
486 	return ret;
487 }
488 
489 static int
490 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
491 {
492 	struct scheduler_init_params init_params = {
493 		.def_p = {
494 			"",
495 			sizeof(struct scheduler_ctx),
496 			rte_socket_id(),
497 			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
498 			RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
499 		},
500 		.nb_slaves = 0,
501 		.mode = CDEV_SCHED_MODE_NOT_SET,
502 		.enable_ordering = 0,
503 		.slave_names = { {0} }
504 	};
505 	const char *name;
506 
507 	name = rte_vdev_device_name(vdev);
508 	if (name == NULL)
509 		return -EINVAL;
510 
511 	scheduler_parse_init_params(&init_params,
512 				    rte_vdev_device_args(vdev));
513 
514 
515 	return cryptodev_scheduler_create(name,
516 					vdev,
517 					&init_params);
518 }
519 
520 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
521 	.probe = cryptodev_scheduler_probe,
522 	.remove = cryptodev_scheduler_remove
523 };
524 
525 static struct cryptodev_driver scheduler_crypto_drv;
526 
527 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
528 	cryptodev_scheduler_pmd_drv);
529 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
530 	"max_nb_queue_pairs=<int> "
531 	"max_nb_sessions=<int> "
532 	"socket_id=<int> "
533 	"slave=<name>");
534 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
535 		cryptodev_scheduler_pmd_drv,
536 		cryptodev_driver_id);
537