1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017-2018 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <stdlib.h> 7 #include <string.h> 8 #include <unistd.h> 9 #include <stdbool.h> 10 #include <assert.h> 11 #include <getopt.h> 12 13 #include <rte_malloc.h> 14 #include <rte_cycles.h> 15 #include <rte_vhost.h> 16 #include <rte_cryptodev.h> 17 #include <rte_vhost_crypto.h> 18 #include <rte_string_fns.h> 19 20 #include <cmdline_rdline.h> 21 #include <cmdline_parse.h> 22 #include <cmdline_parse_string.h> 23 #include <cmdline.h> 24 25 #define NB_VIRTIO_QUEUES (1) 26 #define MAX_PKT_BURST (64) 27 #define MAX_IV_LEN (32) 28 #define NB_MEMPOOL_OBJS (8192) 29 #define NB_CRYPTO_DESCRIPTORS (4096) 30 #define NB_CACHE_OBJS (128) 31 #define SESSION_MAP_ENTRIES (1024) 32 #define REFRESH_TIME_SEC (3) 33 34 #define MAX_NB_SOCKETS (4) 35 #define MAX_NB_WORKER_CORES (16) 36 37 struct lcore_option { 38 uint32_t lcore_id; 39 char *socket_files[MAX_NB_SOCKETS]; 40 uint32_t nb_sockets; 41 uint8_t cid; 42 uint16_t qid; 43 }; 44 45 struct vhost_crypto_info { 46 int vids[MAX_NB_SOCKETS]; 47 uint32_t nb_vids; 48 struct rte_mempool *sess_pool; 49 struct rte_mempool *cop_pool; 50 uint8_t cid; 51 uint32_t qid; 52 uint32_t nb_inflight_ops; 53 volatile uint32_t initialized[MAX_NB_SOCKETS]; 54 } __rte_cache_aligned; 55 56 struct vhost_crypto_options { 57 struct lcore_option los[MAX_NB_WORKER_CORES]; 58 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES]; 59 uint32_t nb_los; 60 uint32_t zero_copy; 61 uint32_t guest_polling; 62 } options; 63 64 #define CONFIG_KEYWORD "config" 65 #define SOCKET_FILE_KEYWORD "socket-file" 66 #define ZERO_COPY_KEYWORD "zero-copy" 67 #define POLLING_KEYWORD "guest-polling" 68 69 #define NB_SOCKET_FIELDS (2) 70 71 static uint32_t 72 find_lo(uint32_t lcore_id) 73 { 74 uint32_t i; 75 76 for (i = 0; i < options.nb_los; i++) 77 if (options.los[i].lcore_id == lcore_id) 78 return i; 79 80 return UINT32_MAX; 81 } 82 83 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */ 84 static int 85 parse_socket_arg(char *arg) 86 { 87 uint32_t nb_sockets; 88 uint32_t lcore_id; 89 char *str_fld[NB_SOCKET_FIELDS]; 90 struct lcore_option *lo; 91 uint32_t idx; 92 char *end; 93 94 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') != 95 NB_SOCKET_FIELDS) { 96 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg); 97 return -EINVAL; 98 } 99 100 errno = 0; 101 lcore_id = strtoul(str_fld[0], &end, 0); 102 if (errno != 0 || end == str_fld[0] || lcore_id > 255) 103 return -EINVAL; 104 105 idx = find_lo(lcore_id); 106 if (idx == UINT32_MAX) { 107 if (options.nb_los == MAX_NB_WORKER_CORES) 108 return -ENOMEM; 109 lo = &options.los[options.nb_los]; 110 lo->lcore_id = lcore_id; 111 options.nb_los++; 112 } else 113 lo = &options.los[idx]; 114 115 nb_sockets = lo->nb_sockets; 116 117 if (nb_sockets >= MAX_NB_SOCKETS) { 118 RTE_LOG(ERR, USER1, "Too many socket files!\n"); 119 return -ENOMEM; 120 } 121 122 lo->socket_files[nb_sockets] = strdup(str_fld[1]); 123 if (!lo->socket_files[nb_sockets]) { 124 RTE_LOG(ERR, USER1, "Insufficient memory\n"); 125 return -ENOMEM; 126 } 127 128 lo->nb_sockets++; 129 130 return 0; 131 } 132 133 static int 134 parse_config(char *q_arg) 135 { 136 struct lcore_option *lo; 137 char s[256]; 138 const char *p, *p0 = q_arg; 139 char *end; 140 enum fieldnames { 141 FLD_LCORE = 0, 142 FLD_CID, 143 FLD_QID, 144 _NUM_FLD 145 }; 146 uint32_t flds[_NUM_FLD]; 147 char *str_fld[_NUM_FLD]; 148 uint32_t i; 149 uint32_t size; 150 151 while ((p = strchr(p0, '(')) != NULL) { 152 ++p; 153 p0 = strchr(p, ')'); 154 if (p0 == NULL) 155 return -1; 156 157 size = p0 - p; 158 if (size >= sizeof(s)) 159 return -1; 160 161 snprintf(s, sizeof(s), "%.*s", size, p); 162 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != 163 _NUM_FLD) 164 return -1; 165 for (i = 0; i < _NUM_FLD; i++) { 166 errno = 0; 167 flds[i] = strtoul(str_fld[i], &end, 0); 168 if (errno != 0 || end == str_fld[i] || flds[i] > 255) 169 return -EINVAL; 170 } 171 172 if (flds[FLD_LCORE] > RTE_MAX_LCORE) 173 return -EINVAL; 174 175 i = find_lo(flds[FLD_LCORE]); 176 if (i == UINT32_MAX) { 177 if (options.nb_los == MAX_NB_WORKER_CORES) 178 return -ENOMEM; 179 lo = &options.los[options.nb_los]; 180 options.nb_los++; 181 } else 182 lo = &options.los[i]; 183 184 lo->lcore_id = flds[FLD_LCORE]; 185 lo->cid = flds[FLD_CID]; 186 lo->qid = flds[FLD_QID]; 187 } 188 189 return 0; 190 } 191 192 static void 193 vhost_crypto_usage(const char *prgname) 194 { 195 printf("%s [EAL options] --\n" 196 " --%s <lcore>,SOCKET-FILE-PATH\n" 197 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]" 198 " --%s: zero copy\n" 199 " --%s: guest polling\n", 200 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD, 201 ZERO_COPY_KEYWORD, POLLING_KEYWORD); 202 } 203 204 static int 205 vhost_crypto_parse_args(int argc, char **argv) 206 { 207 int opt, ret; 208 char *prgname = argv[0]; 209 char **argvopt; 210 int option_index; 211 struct option lgopts[] = { 212 {SOCKET_FILE_KEYWORD, required_argument, 0, 0}, 213 {CONFIG_KEYWORD, required_argument, 0, 0}, 214 {ZERO_COPY_KEYWORD, no_argument, 0, 0}, 215 {POLLING_KEYWORD, no_argument, 0, 0}, 216 {NULL, 0, 0, 0} 217 }; 218 219 argvopt = argv; 220 221 while ((opt = getopt_long(argc, argvopt, "s:", 222 lgopts, &option_index)) != EOF) { 223 224 switch (opt) { 225 case 0: 226 if (strcmp(lgopts[option_index].name, 227 SOCKET_FILE_KEYWORD) == 0) { 228 ret = parse_socket_arg(optarg); 229 if (ret < 0) { 230 vhost_crypto_usage(prgname); 231 return ret; 232 } 233 } else if (strcmp(lgopts[option_index].name, 234 CONFIG_KEYWORD) == 0) { 235 ret = parse_config(optarg); 236 if (ret < 0) { 237 vhost_crypto_usage(prgname); 238 return ret; 239 } 240 } else if (strcmp(lgopts[option_index].name, 241 ZERO_COPY_KEYWORD) == 0) { 242 options.zero_copy = 243 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE; 244 } else if (strcmp(lgopts[option_index].name, 245 POLLING_KEYWORD) == 0) { 246 options.guest_polling = 1; 247 } else { 248 vhost_crypto_usage(prgname); 249 return -EINVAL; 250 } 251 break; 252 default: 253 return -1; 254 } 255 } 256 257 return 0; 258 } 259 260 static int 261 new_device(int vid) 262 { 263 struct vhost_crypto_info *info = NULL; 264 char path[PATH_MAX]; 265 uint32_t i, j; 266 int ret; 267 268 ret = rte_vhost_get_ifname(vid, path, PATH_MAX); 269 if (ret) { 270 RTE_LOG(ERR, USER1, "Cannot find matched socket\n"); 271 return ret; 272 } 273 274 for (i = 0; i < options.nb_los; i++) { 275 for (j = 0; j < options.los[i].nb_sockets; j++) { 276 if (strcmp(path, options.los[i].socket_files[j]) == 0) { 277 info = options.infos[i]; 278 break; 279 } 280 } 281 282 if (info) 283 break; 284 } 285 286 if (!info) { 287 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n"); 288 return -ENOENT; 289 } 290 291 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool, 292 rte_lcore_to_socket_id(options.los[i].lcore_id)); 293 if (ret) { 294 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n"); 295 return ret; 296 } 297 298 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy); 299 if (ret) { 300 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n", 301 options.zero_copy == 1 ? "enable" : "disable"); 302 return ret; 303 } 304 305 info->vids[j] = vid; 306 info->initialized[j] = 1; 307 308 rte_wmb(); 309 310 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path, 311 vid); 312 return 0; 313 } 314 315 static void 316 destroy_device(int vid) 317 { 318 struct vhost_crypto_info *info = NULL; 319 uint32_t i, j; 320 321 for (i = 0; i < options.nb_los; i++) { 322 for (j = 0; j < options.los[i].nb_sockets; j++) { 323 if (options.infos[i]->vids[j] == vid) { 324 info = options.infos[i]; 325 break; 326 } 327 } 328 if (info) 329 break; 330 } 331 332 if (!info) { 333 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n"); 334 return; 335 } 336 337 do { 338 339 } while (info->nb_inflight_ops); 340 341 info->initialized[j] = 0; 342 343 rte_wmb(); 344 345 rte_vhost_crypto_free(vid); 346 347 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid); 348 } 349 350 static const struct vhost_device_ops virtio_crypto_device_ops = { 351 .new_device = new_device, 352 .destroy_device = destroy_device, 353 }; 354 355 static int 356 vhost_crypto_worker(void *arg) 357 { 358 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 359 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1]; 360 struct vhost_crypto_info *info = arg; 361 uint16_t nb_callfds; 362 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS]; 363 uint32_t lcore_id = rte_lcore_id(); 364 uint32_t burst_size = MAX_PKT_BURST; 365 uint32_t i, j, k; 366 uint32_t to_fetch, fetched; 367 368 int ret = 0; 369 370 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id); 371 372 for (i = 0; i < NB_VIRTIO_QUEUES; i++) { 373 if (rte_crypto_op_bulk_alloc(info->cop_pool, 374 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i], 375 burst_size) < burst_size) { 376 RTE_LOG(ERR, USER1, "Failed to alloc cops\n"); 377 ret = -1; 378 goto exit; 379 } 380 } 381 382 while (1) { 383 for (i = 0; i < info->nb_vids; i++) { 384 if (unlikely(info->initialized[i] == 0)) 385 continue; 386 387 for (j = 0; j < NB_VIRTIO_QUEUES; j++) { 388 to_fetch = RTE_MIN(burst_size, 389 (NB_CRYPTO_DESCRIPTORS - 390 info->nb_inflight_ops)); 391 fetched = rte_vhost_crypto_fetch_requests( 392 info->vids[i], j, ops[j], 393 to_fetch); 394 info->nb_inflight_ops += 395 rte_cryptodev_enqueue_burst( 396 info->cid, info->qid, ops[j], 397 fetched); 398 if (unlikely(rte_crypto_op_bulk_alloc( 399 info->cop_pool, 400 RTE_CRYPTO_OP_TYPE_SYMMETRIC, 401 ops[j], fetched) < fetched)) { 402 RTE_LOG(ERR, USER1, "Failed realloc\n"); 403 return -1; 404 } 405 406 fetched = rte_cryptodev_dequeue_burst( 407 info->cid, info->qid, 408 ops_deq[j], RTE_MIN(burst_size, 409 info->nb_inflight_ops)); 410 fetched = rte_vhost_crypto_finalize_requests( 411 ops_deq[j], fetched, callfds, 412 &nb_callfds); 413 414 info->nb_inflight_ops -= fetched; 415 416 if (!options.guest_polling) { 417 for (k = 0; k < nb_callfds; k++) 418 eventfd_write(callfds[k], 419 (eventfd_t)1); 420 } 421 422 rte_mempool_put_bulk(info->cop_pool, 423 (void **)ops_deq[j], fetched); 424 } 425 } 426 } 427 exit: 428 return ret; 429 } 430 431 static void 432 free_resource(void) 433 { 434 uint32_t i, j; 435 436 for (i = 0; i < options.nb_los; i++) { 437 struct lcore_option *lo = &options.los[i]; 438 struct vhost_crypto_info *info = options.infos[i]; 439 440 if (!info) 441 continue; 442 443 rte_mempool_free(info->cop_pool); 444 rte_mempool_free(info->sess_pool); 445 446 for (j = 0; j < lo->nb_sockets; j++) { 447 rte_vhost_driver_unregister(lo->socket_files[i]); 448 free(lo->socket_files[i]); 449 } 450 451 rte_free(info); 452 } 453 454 memset(&options, 0, sizeof(options)); 455 } 456 457 int 458 main(int argc, char *argv[]) 459 { 460 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS}; 461 struct rte_cryptodev_config config; 462 struct rte_cryptodev_info dev_info; 463 char name[128]; 464 uint32_t i, j, lcore; 465 int ret; 466 467 ret = rte_eal_init(argc, argv); 468 if (ret < 0) 469 return -1; 470 argc -= ret; 471 argv += ret; 472 473 ret = vhost_crypto_parse_args(argc, argv); 474 if (ret < 0) 475 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n"); 476 477 for (i = 0; i < options.nb_los; i++) { 478 struct lcore_option *lo = &options.los[i]; 479 struct vhost_crypto_info *info; 480 481 info = rte_zmalloc_socket(NULL, sizeof(*info), 482 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id( 483 lo->lcore_id)); 484 if (!info) { 485 ret = -ENOMEM; 486 goto error_exit; 487 } 488 489 info->cid = lo->cid; 490 info->qid = lo->qid; 491 info->nb_vids = lo->nb_sockets; 492 493 rte_cryptodev_info_get(info->cid, &dev_info); 494 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) { 495 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD crypto_aesni_mb 496 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm 497 if (strstr(dev_info.driver_name, 498 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) || 499 strstr(dev_info.driver_name, 500 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) { 501 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n", 502 dev_info.driver_name); 503 ret = -EPERM; 504 goto error_exit; 505 } 506 } 507 508 if (dev_info.max_nb_queue_pairs < info->qid + 1) { 509 RTE_LOG(ERR, USER1, "Number of queues cannot over %u", 510 dev_info.max_nb_queue_pairs); 511 goto error_exit; 512 } 513 514 config.nb_queue_pairs = dev_info.max_nb_queue_pairs; 515 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id); 516 517 ret = rte_cryptodev_configure(info->cid, &config); 518 if (ret < 0) { 519 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u", 520 info->cid); 521 goto error_exit; 522 } 523 524 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id); 525 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES, 526 rte_cryptodev_sym_get_private_session_size( 527 info->cid), 64, 0, NULL, NULL, NULL, NULL, 528 rte_lcore_to_socket_id(lo->lcore_id), 0); 529 if (!info->sess_pool) { 530 RTE_LOG(ERR, USER1, "Failed to create mempool"); 531 goto error_exit; 532 } 533 534 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id); 535 info->cop_pool = rte_crypto_op_pool_create(name, 536 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS, 537 NB_CACHE_OBJS, 0, 538 rte_lcore_to_socket_id(lo->lcore_id)); 539 540 if (!info->cop_pool) { 541 RTE_LOG(ERR, USER1, "Failed to create crypto pool"); 542 ret = -ENOMEM; 543 goto error_exit; 544 } 545 546 options.infos[i] = info; 547 548 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) { 549 ret = rte_cryptodev_queue_pair_setup(info->cid, j, 550 &qp_conf, rte_lcore_to_socket_id( 551 lo->lcore_id), 552 info->sess_pool); 553 if (ret < 0) { 554 RTE_LOG(ERR, USER1, "Failed to configure qp\n"); 555 goto error_exit; 556 } 557 } 558 } 559 560 for (i = 0; i < options.nb_los; i++) { 561 struct lcore_option *lo = &options.los[i]; 562 struct vhost_crypto_info *info = options.infos[i]; 563 564 ret = rte_cryptodev_start(lo->cid); 565 if (ret < 0) { 566 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n"); 567 goto error_exit; 568 } 569 570 if (rte_eal_remote_launch(vhost_crypto_worker, info, 571 lo->lcore_id) < 0) { 572 RTE_LOG(ERR, USER1, "Failed to start worker lcore"); 573 goto error_exit; 574 } 575 576 for (j = 0; j < lo->nb_sockets; j++) { 577 ret = rte_vhost_driver_register(lo->socket_files[j], 578 RTE_VHOST_USER_DEQUEUE_ZERO_COPY); 579 if (ret < 0) { 580 RTE_LOG(ERR, USER1, "socket %s already exists\n", 581 lo->socket_files[j]); 582 goto error_exit; 583 } 584 585 rte_vhost_driver_callback_register(lo->socket_files[j], 586 &virtio_crypto_device_ops); 587 588 ret = rte_vhost_driver_start(lo->socket_files[j]); 589 if (ret < 0) { 590 RTE_LOG(ERR, USER1, "failed to start vhost.\n"); 591 goto error_exit; 592 } 593 } 594 } 595 596 RTE_LCORE_FOREACH(lcore) 597 rte_eal_wait_lcore(lcore); 598 599 free_resource(); 600 601 return 0; 602 603 error_exit: 604 605 free_resource(); 606 607 return -1; 608 } 609