1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
3 */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <stdbool.h>
10 #include <assert.h>
11 #include <getopt.h>
12
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
19
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
23 #include <cmdline.h>
24
25 #define NB_VIRTIO_QUEUES (1)
26 #define MAX_PKT_BURST (64)
27 #define MAX_IV_LEN (32)
28 #define NB_MEMPOOL_OBJS (8192)
29 #define NB_CRYPTO_DESCRIPTORS (4096)
30 #define NB_CACHE_OBJS (128)
31 #define SESSION_MAP_ENTRIES (1024)
32 #define REFRESH_TIME_SEC (3)
33
34 #define MAX_NB_SOCKETS (4)
35 #define MAX_NB_WORKER_CORES (16)
36
37 struct lcore_option {
38 uint32_t lcore_id;
39 char *socket_files[MAX_NB_SOCKETS];
40 uint32_t nb_sockets;
41 uint8_t cid;
42 uint16_t qid;
43 };
44
45 struct vhost_crypto_info {
46 int vids[MAX_NB_SOCKETS];
47 uint32_t nb_vids;
48 struct rte_mempool *sess_pool;
49 struct rte_mempool *sess_priv_pool;
50 struct rte_mempool *cop_pool;
51 uint8_t cid;
52 uint32_t qid;
53 uint32_t nb_inflight_ops;
54 volatile uint32_t initialized[MAX_NB_SOCKETS];
55 } __rte_cache_aligned;
56
57 struct vhost_crypto_options {
58 struct lcore_option los[MAX_NB_WORKER_CORES];
59 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
60 uint32_t nb_los;
61 uint32_t zero_copy;
62 uint32_t guest_polling;
63 } options;
64
65 #define CONFIG_KEYWORD "config"
66 #define SOCKET_FILE_KEYWORD "socket-file"
67 #define ZERO_COPY_KEYWORD "zero-copy"
68 #define POLLING_KEYWORD "guest-polling"
69
70 #define NB_SOCKET_FIELDS (2)
71
72 static uint32_t
find_lo(uint32_t lcore_id)73 find_lo(uint32_t lcore_id)
74 {
75 uint32_t i;
76
77 for (i = 0; i < options.nb_los; i++)
78 if (options.los[i].lcore_id == lcore_id)
79 return i;
80
81 return UINT32_MAX;
82 }
83
84 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
85 static int
parse_socket_arg(char * arg)86 parse_socket_arg(char *arg)
87 {
88 uint32_t nb_sockets;
89 uint32_t lcore_id;
90 char *str_fld[NB_SOCKET_FIELDS];
91 struct lcore_option *lo;
92 uint32_t idx;
93 char *end;
94
95 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
96 NB_SOCKET_FIELDS) {
97 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
98 return -EINVAL;
99 }
100
101 errno = 0;
102 lcore_id = strtoul(str_fld[0], &end, 0);
103 if (errno != 0 || end == str_fld[0] || lcore_id > 255)
104 return -EINVAL;
105
106 idx = find_lo(lcore_id);
107 if (idx == UINT32_MAX) {
108 if (options.nb_los == MAX_NB_WORKER_CORES)
109 return -ENOMEM;
110 lo = &options.los[options.nb_los];
111 lo->lcore_id = lcore_id;
112 options.nb_los++;
113 } else
114 lo = &options.los[idx];
115
116 nb_sockets = lo->nb_sockets;
117
118 if (nb_sockets >= MAX_NB_SOCKETS) {
119 RTE_LOG(ERR, USER1, "Too many socket files!\n");
120 return -ENOMEM;
121 }
122
123 lo->socket_files[nb_sockets] = strdup(str_fld[1]);
124 if (!lo->socket_files[nb_sockets]) {
125 RTE_LOG(ERR, USER1, "Insufficient memory\n");
126 return -ENOMEM;
127 }
128
129 lo->nb_sockets++;
130
131 return 0;
132 }
133
134 static int
parse_config(char * q_arg)135 parse_config(char *q_arg)
136 {
137 struct lcore_option *lo;
138 char s[256];
139 const char *p, *p0 = q_arg;
140 char *end;
141 enum fieldnames {
142 FLD_LCORE = 0,
143 FLD_CID,
144 FLD_QID,
145 _NUM_FLD
146 };
147 uint32_t flds[_NUM_FLD];
148 char *str_fld[_NUM_FLD];
149 uint32_t i;
150 uint32_t size;
151
152 while ((p = strchr(p0, '(')) != NULL) {
153 ++p;
154 p0 = strchr(p, ')');
155 if (p0 == NULL)
156 return -1;
157
158 size = p0 - p;
159 if (size >= sizeof(s))
160 return -1;
161
162 snprintf(s, sizeof(s), "%.*s", size, p);
163 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
164 _NUM_FLD)
165 return -1;
166 for (i = 0; i < _NUM_FLD; i++) {
167 errno = 0;
168 flds[i] = strtoul(str_fld[i], &end, 0);
169 if (errno != 0 || end == str_fld[i] || flds[i] > 255)
170 return -EINVAL;
171 }
172
173 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
174 return -EINVAL;
175
176 i = find_lo(flds[FLD_LCORE]);
177 if (i == UINT32_MAX) {
178 if (options.nb_los == MAX_NB_WORKER_CORES)
179 return -ENOMEM;
180 lo = &options.los[options.nb_los];
181 options.nb_los++;
182 } else
183 lo = &options.los[i];
184
185 lo->lcore_id = flds[FLD_LCORE];
186 lo->cid = flds[FLD_CID];
187 lo->qid = flds[FLD_QID];
188 }
189
190 return 0;
191 }
192
193 static void
vhost_crypto_usage(const char * prgname)194 vhost_crypto_usage(const char *prgname)
195 {
196 printf("%s [EAL options] --\n"
197 " --%s <lcore>,SOCKET-FILE-PATH\n"
198 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
199 " --%s: zero copy\n"
200 " --%s: guest polling\n",
201 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
202 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
203 }
204
205 static int
vhost_crypto_parse_args(int argc,char ** argv)206 vhost_crypto_parse_args(int argc, char **argv)
207 {
208 int opt, ret;
209 char *prgname = argv[0];
210 char **argvopt;
211 int option_index;
212 struct option lgopts[] = {
213 {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
214 {CONFIG_KEYWORD, required_argument, 0, 0},
215 {ZERO_COPY_KEYWORD, no_argument, 0, 0},
216 {POLLING_KEYWORD, no_argument, 0, 0},
217 {NULL, 0, 0, 0}
218 };
219
220 argvopt = argv;
221
222 while ((opt = getopt_long(argc, argvopt, "s:",
223 lgopts, &option_index)) != EOF) {
224
225 switch (opt) {
226 case 0:
227 if (strcmp(lgopts[option_index].name,
228 SOCKET_FILE_KEYWORD) == 0) {
229 ret = parse_socket_arg(optarg);
230 if (ret < 0) {
231 vhost_crypto_usage(prgname);
232 return ret;
233 }
234 } else if (strcmp(lgopts[option_index].name,
235 CONFIG_KEYWORD) == 0) {
236 ret = parse_config(optarg);
237 if (ret < 0) {
238 vhost_crypto_usage(prgname);
239 return ret;
240 }
241 } else if (strcmp(lgopts[option_index].name,
242 ZERO_COPY_KEYWORD) == 0) {
243 options.zero_copy =
244 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
245 } else if (strcmp(lgopts[option_index].name,
246 POLLING_KEYWORD) == 0) {
247 options.guest_polling = 1;
248 } else {
249 vhost_crypto_usage(prgname);
250 return -EINVAL;
251 }
252 break;
253 default:
254 return -1;
255 }
256 }
257
258 return 0;
259 }
260
261 static int
new_device(int vid)262 new_device(int vid)
263 {
264 struct vhost_crypto_info *info = NULL;
265 char path[PATH_MAX];
266 uint32_t i, j;
267 int ret;
268
269 ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
270 if (ret) {
271 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
272 return ret;
273 }
274
275 for (i = 0; i < options.nb_los; i++) {
276 for (j = 0; j < options.los[i].nb_sockets; j++) {
277 if (strcmp(path, options.los[i].socket_files[j]) == 0) {
278 info = options.infos[i];
279 break;
280 }
281 }
282
283 if (info)
284 break;
285 }
286
287 if (!info) {
288 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
289 return -ENOENT;
290 }
291
292 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
293 info->sess_priv_pool,
294 rte_lcore_to_socket_id(options.los[i].lcore_id));
295 if (ret) {
296 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
297 return ret;
298 }
299
300 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
301 if (ret) {
302 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
303 options.zero_copy == 1 ? "enable" : "disable");
304 return ret;
305 }
306
307 info->vids[j] = vid;
308 info->initialized[j] = 1;
309
310 rte_wmb();
311
312 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
313 vid);
314 return 0;
315 }
316
317 static void
destroy_device(int vid)318 destroy_device(int vid)
319 {
320 struct vhost_crypto_info *info = NULL;
321 uint32_t i, j;
322
323 for (i = 0; i < options.nb_los; i++) {
324 for (j = 0; j < options.los[i].nb_sockets; j++) {
325 if (options.infos[i]->vids[j] == vid) {
326 info = options.infos[i];
327 break;
328 }
329 }
330 if (info)
331 break;
332 }
333
334 if (!info) {
335 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
336 return;
337 }
338
339 do {
340
341 } while (info->nb_inflight_ops);
342
343 info->initialized[j] = 0;
344
345 rte_wmb();
346
347 rte_vhost_crypto_free(vid);
348
349 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
350 }
351
352 static const struct vhost_device_ops virtio_crypto_device_ops = {
353 .new_device = new_device,
354 .destroy_device = destroy_device,
355 };
356
357 static int
vhost_crypto_worker(void * arg)358 vhost_crypto_worker(void *arg)
359 {
360 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
361 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
362 struct vhost_crypto_info *info = arg;
363 uint16_t nb_callfds;
364 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
365 uint32_t lcore_id = rte_lcore_id();
366 uint32_t burst_size = MAX_PKT_BURST;
367 uint32_t i, j, k;
368 uint32_t to_fetch, fetched;
369
370 int ret = 0;
371
372 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
373
374 for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
375 if (rte_crypto_op_bulk_alloc(info->cop_pool,
376 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
377 burst_size) < burst_size) {
378 RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
379 ret = -1;
380 goto exit;
381 }
382 }
383
384 while (1) {
385 for (i = 0; i < info->nb_vids; i++) {
386 if (unlikely(info->initialized[i] == 0))
387 continue;
388
389 for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
390 to_fetch = RTE_MIN(burst_size,
391 (NB_CRYPTO_DESCRIPTORS -
392 info->nb_inflight_ops));
393 fetched = rte_vhost_crypto_fetch_requests(
394 info->vids[i], j, ops[j],
395 to_fetch);
396 info->nb_inflight_ops +=
397 rte_cryptodev_enqueue_burst(
398 info->cid, info->qid, ops[j],
399 fetched);
400 if (unlikely(rte_crypto_op_bulk_alloc(
401 info->cop_pool,
402 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
403 ops[j], fetched) < fetched)) {
404 RTE_LOG(ERR, USER1, "Failed realloc\n");
405 return -1;
406 }
407
408 fetched = rte_cryptodev_dequeue_burst(
409 info->cid, info->qid,
410 ops_deq[j], RTE_MIN(burst_size,
411 info->nb_inflight_ops));
412 fetched = rte_vhost_crypto_finalize_requests(
413 ops_deq[j], fetched, callfds,
414 &nb_callfds);
415
416 info->nb_inflight_ops -= fetched;
417
418 if (!options.guest_polling) {
419 for (k = 0; k < nb_callfds; k++)
420 eventfd_write(callfds[k],
421 (eventfd_t)1);
422 }
423
424 rte_mempool_put_bulk(info->cop_pool,
425 (void **)ops_deq[j], fetched);
426 }
427 }
428 }
429 exit:
430 return ret;
431 }
432
433 static void
free_resource(void)434 free_resource(void)
435 {
436 uint32_t i, j;
437
438 for (i = 0; i < options.nb_los; i++) {
439 struct lcore_option *lo = &options.los[i];
440 struct vhost_crypto_info *info = options.infos[i];
441
442 if (!info)
443 continue;
444
445 rte_mempool_free(info->cop_pool);
446 rte_mempool_free(info->sess_pool);
447 rte_mempool_free(info->sess_priv_pool);
448
449 for (j = 0; j < lo->nb_sockets; j++) {
450 rte_vhost_driver_unregister(lo->socket_files[i]);
451 free(lo->socket_files[i]);
452 }
453
454 rte_free(info);
455 }
456
457 memset(&options, 0, sizeof(options));
458 }
459
460 int
main(int argc,char * argv[])461 main(int argc, char *argv[])
462 {
463 struct rte_cryptodev_qp_conf qp_conf;
464 struct rte_cryptodev_config config;
465 struct rte_cryptodev_info dev_info;
466 char name[128];
467 uint32_t i, j, lcore;
468 int ret;
469
470 ret = rte_eal_init(argc, argv);
471 if (ret < 0)
472 return -1;
473 argc -= ret;
474 argv += ret;
475
476 ret = vhost_crypto_parse_args(argc, argv);
477 if (ret < 0)
478 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
479
480 for (i = 0; i < options.nb_los; i++) {
481 struct lcore_option *lo = &options.los[i];
482 struct vhost_crypto_info *info;
483
484 info = rte_zmalloc_socket(NULL, sizeof(*info),
485 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
486 lo->lcore_id));
487 if (!info) {
488 ret = -ENOMEM;
489 goto error_exit;
490 }
491
492 info->cid = lo->cid;
493 info->qid = lo->qid;
494 info->nb_vids = lo->nb_sockets;
495
496 rte_cryptodev_info_get(info->cid, &dev_info);
497 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
498 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb
499 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
500 if (strstr(dev_info.driver_name,
501 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
502 strstr(dev_info.driver_name,
503 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
504 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
505 dev_info.driver_name);
506 ret = -EPERM;
507 goto error_exit;
508 }
509 }
510
511 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
512 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
513 dev_info.max_nb_queue_pairs);
514 goto error_exit;
515 }
516
517 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
518 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
519 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
520
521 ret = rte_cryptodev_configure(info->cid, &config);
522 if (ret < 0) {
523 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
524 info->cid);
525 goto error_exit;
526 }
527
528 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
529 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
530 SESSION_MAP_ENTRIES, 0, 0, 0,
531 rte_lcore_to_socket_id(lo->lcore_id));
532
533 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
534 info->sess_priv_pool = rte_mempool_create(name,
535 SESSION_MAP_ENTRIES,
536 rte_cryptodev_sym_get_private_session_size(
537 info->cid), 64, 0, NULL, NULL, NULL, NULL,
538 rte_lcore_to_socket_id(lo->lcore_id), 0);
539 if (!info->sess_priv_pool || !info->sess_pool) {
540 RTE_LOG(ERR, USER1, "Failed to create mempool");
541 goto error_exit;
542 }
543
544 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
545 info->cop_pool = rte_crypto_op_pool_create(name,
546 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
547 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
548 rte_lcore_to_socket_id(lo->lcore_id));
549
550 if (!info->cop_pool) {
551 RTE_LOG(ERR, USER1, "Failed to create crypto pool");
552 ret = -ENOMEM;
553 goto error_exit;
554 }
555
556 options.infos[i] = info;
557
558 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
559 qp_conf.mp_session = info->sess_pool;
560 qp_conf.mp_session_private = info->sess_priv_pool;
561
562 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
563 ret = rte_cryptodev_queue_pair_setup(info->cid, j,
564 &qp_conf, rte_lcore_to_socket_id(
565 lo->lcore_id));
566 if (ret < 0) {
567 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
568 goto error_exit;
569 }
570 }
571 }
572
573 for (i = 0; i < options.nb_los; i++) {
574 struct lcore_option *lo = &options.los[i];
575 struct vhost_crypto_info *info = options.infos[i];
576
577 ret = rte_cryptodev_start(lo->cid);
578 if (ret < 0) {
579 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
580 goto error_exit;
581 }
582
583 if (rte_eal_remote_launch(vhost_crypto_worker, info,
584 lo->lcore_id) < 0) {
585 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
586 goto error_exit;
587 }
588
589 for (j = 0; j < lo->nb_sockets; j++) {
590 ret = rte_vhost_driver_register(lo->socket_files[j],
591 RTE_VHOST_USER_ASYNC_COPY);
592 if (ret < 0) {
593 RTE_LOG(ERR, USER1, "socket %s already exists\n",
594 lo->socket_files[j]);
595 goto error_exit;
596 }
597
598 rte_vhost_driver_callback_register(lo->socket_files[j],
599 &virtio_crypto_device_ops);
600
601 ret = rte_vhost_crypto_driver_start(
602 lo->socket_files[j]);
603 if (ret < 0) {
604 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
605 goto error_exit;
606 }
607 }
608 }
609
610 RTE_LCORE_FOREACH(lcore)
611 rte_eal_wait_lcore(lcore);
612
613 free_resource();
614
615 return 0;
616
617 error_exit:
618
619 free_resource();
620
621 return -1;
622 }
623