1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <stdbool.h>
10 #include <assert.h>
11 #include <getopt.h>
12
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
19
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
23 #include <cmdline.h>
24
25 #define NB_VIRTIO_QUEUES (1)
26 #define MAX_PKT_BURST (64)
27 #define MAX_IV_LEN (32)
28 #define NB_MEMPOOL_OBJS (8192)
29 #define NB_CRYPTO_DESCRIPTORS (4096)
30 #define NB_CACHE_OBJS (128)
31 #define SESSION_MAP_ENTRIES (1024)
32 #define REFRESH_TIME_SEC (3)
33
34 #define MAX_NB_SOCKETS (4)
35 #define MAX_NB_WORKER_CORES (16)
36
37 struct lcore_option {
38 uint32_t lcore_id;
39 char *socket_files[MAX_NB_SOCKETS];
40 uint32_t nb_sockets;
41 uint8_t cid;
42 uint16_t qid;
43 };
44
45 struct vhost_crypto_info {
46 int vids[MAX_NB_SOCKETS];
47 uint32_t nb_vids;
48 struct rte_mempool *sess_pool;
49 struct rte_mempool *sess_priv_pool;
50 struct rte_mempool *cop_pool;
51 uint8_t cid;
52 uint32_t qid;
53 uint32_t nb_inflight_ops;
54 volatile uint32_t initialized[MAX_NB_SOCKETS];
55 } __rte_cache_aligned;
56
57 struct vhost_crypto_options {
58 struct lcore_option los[MAX_NB_WORKER_CORES];
59 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
60 uint32_t nb_los;
61 uint32_t zero_copy;
62 uint32_t guest_polling;
63 } options;
64
65 enum {
66 #define OPT_CONFIG "config"
67 OPT_CONFIG_NUM = 256,
68 #define OPT_SOCKET_FILE "socket-file"
69 OPT_SOCKET_FILE_NUM,
70 #define OPT_ZERO_COPY "zero-copy"
71 OPT_ZERO_COPY_NUM,
72 #define OPT_POLLING "guest-polling"
73 OPT_POLLING_NUM,
74 };
75
76 #define NB_SOCKET_FIELDS (2)
77
78 static uint32_t
find_lo(uint32_t lcore_id)79 find_lo(uint32_t lcore_id)
80 {
81 uint32_t i;
82
83 for (i = 0; i < options.nb_los; i++)
84 if (options.los[i].lcore_id == lcore_id)
85 return i;
86
87 return UINT32_MAX;
88 }
89
90 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
91 static int
parse_socket_arg(char * arg)92 parse_socket_arg(char *arg)
93 {
94 uint32_t nb_sockets;
95 uint32_t lcore_id;
96 char *str_fld[NB_SOCKET_FIELDS];
97 struct lcore_option *lo;
98 uint32_t idx;
99 char *end;
100
101 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
102 NB_SOCKET_FIELDS) {
103 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
104 return -EINVAL;
105 }
106
107 errno = 0;
108 lcore_id = strtoul(str_fld[0], &end, 0);
109 if (errno != 0 || end == str_fld[0] || lcore_id > 255)
110 return -EINVAL;
111
112 idx = find_lo(lcore_id);
113 if (idx == UINT32_MAX) {
114 if (options.nb_los == MAX_NB_WORKER_CORES)
115 return -ENOMEM;
116 lo = &options.los[options.nb_los];
117 lo->lcore_id = lcore_id;
118 options.nb_los++;
119 } else
120 lo = &options.los[idx];
121
122 nb_sockets = lo->nb_sockets;
123
124 if (nb_sockets >= MAX_NB_SOCKETS) {
125 RTE_LOG(ERR, USER1, "Too many socket files!\n");
126 return -ENOMEM;
127 }
128
129 lo->socket_files[nb_sockets] = strdup(str_fld[1]);
130 if (!lo->socket_files[nb_sockets]) {
131 RTE_LOG(ERR, USER1, "Insufficient memory\n");
132 return -ENOMEM;
133 }
134
135 lo->nb_sockets++;
136
137 return 0;
138 }
139
140 static int
parse_config(char * q_arg)141 parse_config(char *q_arg)
142 {
143 struct lcore_option *lo;
144 char s[256];
145 const char *p, *p0 = q_arg;
146 char *end;
147 enum fieldnames {
148 FLD_LCORE = 0,
149 FLD_CID,
150 FLD_QID,
151 _NUM_FLD
152 };
153 uint32_t flds[_NUM_FLD];
154 char *str_fld[_NUM_FLD];
155 uint32_t i;
156 uint32_t size;
157
158 while ((p = strchr(p0, '(')) != NULL) {
159 ++p;
160 p0 = strchr(p, ')');
161 if (p0 == NULL)
162 return -1;
163
164 size = p0 - p;
165 if (size >= sizeof(s))
166 return -1;
167
168 snprintf(s, sizeof(s), "%.*s", size, p);
169 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
170 _NUM_FLD)
171 return -1;
172 for (i = 0; i < _NUM_FLD; i++) {
173 errno = 0;
174 flds[i] = strtoul(str_fld[i], &end, 0);
175 if (errno != 0 || end == str_fld[i] || flds[i] > 255)
176 return -EINVAL;
177 }
178
179 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
180 return -EINVAL;
181
182 i = find_lo(flds[FLD_LCORE]);
183 if (i == UINT32_MAX) {
184 if (options.nb_los == MAX_NB_WORKER_CORES)
185 return -ENOMEM;
186 lo = &options.los[options.nb_los];
187 options.nb_los++;
188 } else
189 lo = &options.los[i];
190
191 lo->lcore_id = flds[FLD_LCORE];
192 lo->cid = flds[FLD_CID];
193 lo->qid = flds[FLD_QID];
194 }
195
196 return 0;
197 }
198
199 static void
vhost_crypto_usage(const char * prgname)200 vhost_crypto_usage(const char *prgname)
201 {
202 printf("%s [EAL options] --\n"
203 " --%s <lcore>,SOCKET-FILE-PATH\n"
204 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
205 " --%s: zero copy\n"
206 " --%s: guest polling\n",
207 prgname, OPT_SOCKET_FILE, OPT_CONFIG,
208 OPT_ZERO_COPY, OPT_POLLING);
209 }
210
211 static int
vhost_crypto_parse_args(int argc,char ** argv)212 vhost_crypto_parse_args(int argc, char **argv)
213 {
214 int opt, ret;
215 char *prgname = argv[0];
216 char **argvopt;
217 int option_index;
218 struct option lgopts[] = {
219 {OPT_SOCKET_FILE, required_argument,
220 NULL, OPT_SOCKET_FILE_NUM},
221 {OPT_CONFIG, required_argument,
222 NULL, OPT_CONFIG_NUM},
223 {OPT_ZERO_COPY, no_argument,
224 NULL, OPT_ZERO_COPY_NUM},
225 {OPT_POLLING, no_argument,
226 NULL, OPT_POLLING_NUM},
227 {NULL, 0, 0, 0}
228 };
229
230 argvopt = argv;
231
232 while ((opt = getopt_long(argc, argvopt, "",
233 lgopts, &option_index)) != EOF) {
234
235 if (opt == '?') {
236 vhost_crypto_usage(prgname);
237 return -1;
238 }
239
240 switch (opt) {
241 case OPT_SOCKET_FILE_NUM:
242 ret = parse_socket_arg(optarg);
243 if (ret < 0) {
244 vhost_crypto_usage(prgname);
245 return ret;
246 }
247 break;
248
249 case OPT_CONFIG_NUM:
250 ret = parse_config(optarg);
251 if (ret < 0) {
252 vhost_crypto_usage(prgname);
253 return ret;
254 }
255 break;
256
257 case OPT_ZERO_COPY_NUM:
258 options.zero_copy =
259 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
260 break;
261
262 case OPT_POLLING_NUM:
263 options.guest_polling = 1;
264 break;
265
266 default:
267 vhost_crypto_usage(prgname);
268 return -EINVAL;
269 }
270 }
271
272 return 0;
273 }
274
275 static int
new_device(int vid)276 new_device(int vid)
277 {
278 struct vhost_crypto_info *info = NULL;
279 char path[PATH_MAX];
280 uint32_t i, j;
281 int ret;
282
283 ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
284 if (ret) {
285 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
286 return ret;
287 }
288
289 for (i = 0; i < options.nb_los; i++) {
290 for (j = 0; j < options.los[i].nb_sockets; j++) {
291 if (strcmp(path, options.los[i].socket_files[j]) == 0) {
292 info = options.infos[i];
293 break;
294 }
295 }
296
297 if (info)
298 break;
299 }
300
301 if (!info) {
302 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
303 return -ENOENT;
304 }
305
306 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
307 info->sess_priv_pool,
308 rte_lcore_to_socket_id(options.los[i].lcore_id));
309 if (ret) {
310 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
311 return ret;
312 }
313
314 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
315 if (ret) {
316 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
317 options.zero_copy == 1 ? "enable" : "disable");
318 return ret;
319 }
320
321 info->vids[j] = vid;
322 info->initialized[j] = 1;
323
324 rte_wmb();
325
326 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
327 vid);
328 return 0;
329 }
330
331 static void
destroy_device(int vid)332 destroy_device(int vid)
333 {
334 struct vhost_crypto_info *info = NULL;
335 uint32_t i, j;
336
337 for (i = 0; i < options.nb_los; i++) {
338 for (j = 0; j < options.los[i].nb_sockets; j++) {
339 if (options.infos[i]->vids[j] == vid) {
340 info = options.infos[i];
341 break;
342 }
343 }
344 if (info)
345 break;
346 }
347
348 if (!info) {
349 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
350 return;
351 }
352
353 do {
354
355 } while (info->nb_inflight_ops);
356
357 info->initialized[j] = 0;
358
359 rte_wmb();
360
361 rte_vhost_crypto_free(vid);
362
363 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
364 }
365
366 static const struct rte_vhost_device_ops virtio_crypto_device_ops = {
367 .new_device = new_device,
368 .destroy_device = destroy_device,
369 };
370
371 static int
vhost_crypto_worker(void * arg)372 vhost_crypto_worker(void *arg)
373 {
374 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
375 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
376 struct vhost_crypto_info *info = arg;
377 uint16_t nb_callfds;
378 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
379 uint32_t lcore_id = rte_lcore_id();
380 uint32_t burst_size = MAX_PKT_BURST;
381 uint32_t i, j, k;
382 uint32_t to_fetch, fetched;
383
384 int ret = 0;
385
386 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
387
388 for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
389 if (rte_crypto_op_bulk_alloc(info->cop_pool,
390 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
391 burst_size) < burst_size) {
392 RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
393 ret = -1;
394 goto exit;
395 }
396 }
397
398 while (1) {
399 for (i = 0; i < info->nb_vids; i++) {
400 if (unlikely(info->initialized[i] == 0))
401 continue;
402
403 for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
404 to_fetch = RTE_MIN(burst_size,
405 (NB_CRYPTO_DESCRIPTORS -
406 info->nb_inflight_ops));
407 fetched = rte_vhost_crypto_fetch_requests(
408 info->vids[i], j, ops[j],
409 to_fetch);
410 info->nb_inflight_ops +=
411 rte_cryptodev_enqueue_burst(
412 info->cid, info->qid, ops[j],
413 fetched);
414 if (unlikely(rte_crypto_op_bulk_alloc(
415 info->cop_pool,
416 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
417 ops[j], fetched) < fetched)) {
418 RTE_LOG(ERR, USER1, "Failed realloc\n");
419 return -1;
420 }
421
422 fetched = rte_cryptodev_dequeue_burst(
423 info->cid, info->qid,
424 ops_deq[j], RTE_MIN(burst_size,
425 info->nb_inflight_ops));
426 fetched = rte_vhost_crypto_finalize_requests(
427 ops_deq[j], fetched, callfds,
428 &nb_callfds);
429
430 info->nb_inflight_ops -= fetched;
431
432 if (!options.guest_polling) {
433 for (k = 0; k < nb_callfds; k++)
434 eventfd_write(callfds[k],
435 (eventfd_t)1);
436 }
437
438 rte_mempool_put_bulk(info->cop_pool,
439 (void **)ops_deq[j], fetched);
440 }
441 }
442 }
443 exit:
444 return ret;
445 }
446
447 static void
free_resource(void)448 free_resource(void)
449 {
450 uint32_t i, j;
451
452 for (i = 0; i < options.nb_los; i++) {
453 struct lcore_option *lo = &options.los[i];
454 struct vhost_crypto_info *info = options.infos[i];
455
456 if (!info)
457 continue;
458
459 rte_mempool_free(info->cop_pool);
460 rte_mempool_free(info->sess_pool);
461 rte_mempool_free(info->sess_priv_pool);
462
463 for (j = 0; j < lo->nb_sockets; j++) {
464 rte_vhost_driver_unregister(lo->socket_files[i]);
465 free(lo->socket_files[i]);
466 }
467
468 rte_free(info);
469 }
470
471 memset(&options, 0, sizeof(options));
472
473 /* clean up the EAL */
474 rte_eal_cleanup();
475 }
476
477 int
main(int argc,char * argv[])478 main(int argc, char *argv[])
479 {
480 struct rte_cryptodev_qp_conf qp_conf;
481 struct rte_cryptodev_config config;
482 struct rte_cryptodev_info dev_info;
483 char name[128];
484 uint32_t i, j, lcore;
485 int ret;
486
487 ret = rte_eal_init(argc, argv);
488 if (ret < 0)
489 return -1;
490 argc -= ret;
491 argv += ret;
492
493 ret = vhost_crypto_parse_args(argc, argv);
494 if (ret < 0)
495 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
496
497 for (i = 0; i < options.nb_los; i++) {
498 struct lcore_option *lo = &options.los[i];
499 struct vhost_crypto_info *info;
500
501 info = rte_zmalloc_socket(NULL, sizeof(*info),
502 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
503 lo->lcore_id));
504 if (!info) {
505 ret = -ENOMEM;
506 goto error_exit;
507 }
508
509 info->cid = lo->cid;
510 info->qid = lo->qid;
511 info->nb_vids = lo->nb_sockets;
512
513 rte_cryptodev_info_get(info->cid, &dev_info);
514 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
515 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
516 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
517 if (strstr(dev_info.driver_name,
518 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
519 strstr(dev_info.driver_name,
520 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
521 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
522 dev_info.driver_name);
523 ret = -EPERM;
524 goto error_exit;
525 }
526 }
527
528 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
529 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
530 dev_info.max_nb_queue_pairs);
531 goto error_exit;
532 }
533
534 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
535 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
536 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
537
538 ret = rte_cryptodev_configure(info->cid, &config);
539 if (ret < 0) {
540 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
541 info->cid);
542 goto error_exit;
543 }
544
545 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
546 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
547 SESSION_MAP_ENTRIES, 0, 0, 0,
548 rte_lcore_to_socket_id(lo->lcore_id));
549
550 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
551 info->sess_priv_pool = rte_mempool_create(name,
552 SESSION_MAP_ENTRIES,
553 rte_cryptodev_sym_get_private_session_size(
554 info->cid), 64, 0, NULL, NULL, NULL, NULL,
555 rte_lcore_to_socket_id(lo->lcore_id), 0);
556 if (!info->sess_priv_pool || !info->sess_pool) {
557 RTE_LOG(ERR, USER1, "Failed to create mempool");
558 goto error_exit;
559 }
560
561 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
562 info->cop_pool = rte_crypto_op_pool_create(name,
563 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
564 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
565 rte_lcore_to_socket_id(lo->lcore_id));
566
567 if (!info->cop_pool) {
568 RTE_LOG(ERR, USER1, "Failed to create crypto pool");
569 ret = -ENOMEM;
570 goto error_exit;
571 }
572
573 options.infos[i] = info;
574
575 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
576 qp_conf.mp_session = info->sess_pool;
577 qp_conf.mp_session_private = info->sess_priv_pool;
578
579 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
580 ret = rte_cryptodev_queue_pair_setup(info->cid, j,
581 &qp_conf, rte_lcore_to_socket_id(
582 lo->lcore_id));
583 if (ret < 0) {
584 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
585 goto error_exit;
586 }
587 }
588 }
589
590 for (i = 0; i < options.nb_los; i++) {
591 struct lcore_option *lo = &options.los[i];
592 struct vhost_crypto_info *info = options.infos[i];
593
594 ret = rte_cryptodev_start(lo->cid);
595 if (ret < 0) {
596 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
597 goto error_exit;
598 }
599
600 if (rte_eal_remote_launch(vhost_crypto_worker, info,
601 lo->lcore_id) < 0) {
602 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
603 goto error_exit;
604 }
605
606 for (j = 0; j < lo->nb_sockets; j++) {
607 ret = rte_vhost_driver_register(lo->socket_files[j],
608 RTE_VHOST_USER_ASYNC_COPY);
609 if (ret < 0) {
610 RTE_LOG(ERR, USER1, "socket %s already exists\n",
611 lo->socket_files[j]);
612 goto error_exit;
613 }
614
615 rte_vhost_driver_callback_register(lo->socket_files[j],
616 &virtio_crypto_device_ops);
617
618 ret = rte_vhost_crypto_driver_start(
619 lo->socket_files[j]);
620 if (ret < 0) {
621 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
622 goto error_exit;
623 }
624 }
625 }
626
627 RTE_LCORE_FOREACH(lcore)
628 rte_eal_wait_lcore(lcore);
629
630 free_resource();
631
632 return 0;
633
634 error_exit:
635
636 free_resource();
637
638 return -1;
639 }
640