1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 #include <rte_reorder.h> 33 #include <rte_cryptodev.h> 34 #include <rte_cryptodev_pmd.h> 35 #include <rte_malloc.h> 36 37 #include "rte_cryptodev_scheduler.h" 38 #include "scheduler_pmd_private.h" 39 40 /** update the scheduler pmd's capability with attaching device's 41 * capability. 42 * For each device to be attached, the scheduler's capability should be 43 * the common capability set of all slaves 44 **/ 45 static uint32_t 46 sync_caps(struct rte_cryptodev_capabilities *caps, 47 uint32_t nb_caps, 48 const struct rte_cryptodev_capabilities *slave_caps) 49 { 50 uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0; 51 uint32_t i; 52 53 while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED) 54 nb_slave_caps++; 55 56 if (nb_caps == 0) { 57 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps); 58 return nb_slave_caps; 59 } 60 61 for (i = 0; i < sync_nb_caps; i++) { 62 struct rte_cryptodev_capabilities *cap = &caps[i]; 63 uint32_t j; 64 65 for (j = 0; j < nb_slave_caps; j++) { 66 const struct rte_cryptodev_capabilities *s_cap = 67 &slave_caps[j]; 68 69 if (s_cap->op != cap->op || s_cap->sym.xform_type != 70 cap->sym.xform_type) 71 continue; 72 73 if (s_cap->sym.xform_type == 74 RTE_CRYPTO_SYM_XFORM_AUTH) { 75 if (s_cap->sym.auth.algo != 76 cap->sym.auth.algo) 77 continue; 78 79 cap->sym.auth.digest_size.min = 80 s_cap->sym.auth.digest_size.min < 81 cap->sym.auth.digest_size.min ? 82 s_cap->sym.auth.digest_size.min : 83 cap->sym.auth.digest_size.min; 84 cap->sym.auth.digest_size.max = 85 s_cap->sym.auth.digest_size.max < 86 cap->sym.auth.digest_size.max ? 87 s_cap->sym.auth.digest_size.max : 88 cap->sym.auth.digest_size.max; 89 90 } 91 92 if (s_cap->sym.xform_type == 93 RTE_CRYPTO_SYM_XFORM_CIPHER) 94 if (s_cap->sym.cipher.algo != 95 cap->sym.cipher.algo) 96 continue; 97 98 /* no common cap found */ 99 break; 100 } 101 102 if (j < nb_slave_caps) 103 continue; 104 105 /* remove a uncommon cap from the array */ 106 for (j = i; j < sync_nb_caps - 1; j++) 107 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap)); 108 109 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap)); 110 sync_nb_caps--; 111 } 112 113 return sync_nb_caps; 114 } 115 116 static int 117 update_scheduler_capability(struct scheduler_ctx *sched_ctx) 118 { 119 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; 120 uint32_t nb_caps = 0, i; 121 122 if (sched_ctx->capabilities) { 123 rte_free(sched_ctx->capabilities); 124 sched_ctx->capabilities = NULL; 125 } 126 127 for (i = 0; i < sched_ctx->nb_slaves; i++) { 128 struct rte_cryptodev_info dev_info; 129 130 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 131 132 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities); 133 if (nb_caps == 0) 134 return -1; 135 } 136 137 sched_ctx->capabilities = rte_zmalloc_socket(NULL, 138 sizeof(struct rte_cryptodev_capabilities) * 139 (nb_caps + 1), 0, SOCKET_ID_ANY); 140 if (!sched_ctx->capabilities) 141 return -ENOMEM; 142 143 rte_memcpy(sched_ctx->capabilities, tmp_caps, 144 sizeof(struct rte_cryptodev_capabilities) * nb_caps); 145 146 return 0; 147 } 148 149 static void 150 update_scheduler_feature_flag(struct rte_cryptodev *dev) 151 { 152 struct scheduler_ctx *sched_ctx = dev->data->dev_private; 153 uint32_t i; 154 155 dev->feature_flags = 0; 156 157 for (i = 0; i < sched_ctx->nb_slaves; i++) { 158 struct rte_cryptodev_info dev_info; 159 160 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 161 162 dev->feature_flags |= dev_info.feature_flags; 163 } 164 } 165 166 static void 167 update_max_nb_qp(struct scheduler_ctx *sched_ctx) 168 { 169 uint32_t i; 170 uint32_t max_nb_qp; 171 172 if (!sched_ctx->nb_slaves) 173 return; 174 175 max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0; 176 177 for (i = 0; i < sched_ctx->nb_slaves; i++) { 178 struct rte_cryptodev_info dev_info; 179 180 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info); 181 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ? 182 dev_info.max_nb_queue_pairs : max_nb_qp; 183 } 184 185 sched_ctx->max_nb_queue_pairs = max_nb_qp; 186 } 187 188 /** Attach a device to the scheduler. */ 189 int 190 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id) 191 { 192 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 193 struct scheduler_ctx *sched_ctx; 194 struct scheduler_slave *slave; 195 struct rte_cryptodev_info dev_info; 196 uint32_t i; 197 198 if (!dev) { 199 CS_LOG_ERR("Operation not supported"); 200 return -ENOTSUP; 201 } 202 203 if (dev->driver_id != cryptodev_driver_id) { 204 CS_LOG_ERR("Operation not supported"); 205 return -ENOTSUP; 206 } 207 208 if (dev->data->dev_started) { 209 CS_LOG_ERR("Illegal operation"); 210 return -EBUSY; 211 } 212 213 sched_ctx = dev->data->dev_private; 214 if (sched_ctx->nb_slaves >= 215 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) { 216 CS_LOG_ERR("Too many slaves attached"); 217 return -ENOMEM; 218 } 219 220 for (i = 0; i < sched_ctx->nb_slaves; i++) 221 if (sched_ctx->slaves[i].dev_id == slave_id) { 222 CS_LOG_ERR("Slave already added"); 223 return -ENOTSUP; 224 } 225 226 slave = &sched_ctx->slaves[sched_ctx->nb_slaves]; 227 228 rte_cryptodev_info_get(slave_id, &dev_info); 229 230 slave->dev_id = slave_id; 231 slave->driver_id = dev_info.driver_id; 232 sched_ctx->nb_slaves++; 233 234 if (update_scheduler_capability(sched_ctx) < 0) { 235 slave->dev_id = 0; 236 slave->driver_id = 0; 237 sched_ctx->nb_slaves--; 238 239 CS_LOG_ERR("capabilities update failed"); 240 return -ENOTSUP; 241 } 242 243 update_scheduler_feature_flag(dev); 244 245 update_max_nb_qp(sched_ctx); 246 247 return 0; 248 } 249 250 int 251 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id) 252 { 253 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 254 struct scheduler_ctx *sched_ctx; 255 uint32_t i, slave_pos; 256 257 if (!dev) { 258 CS_LOG_ERR("Operation not supported"); 259 return -ENOTSUP; 260 } 261 262 if (dev->driver_id != cryptodev_driver_id) { 263 CS_LOG_ERR("Operation not supported"); 264 return -ENOTSUP; 265 } 266 267 if (dev->data->dev_started) { 268 CS_LOG_ERR("Illegal operation"); 269 return -EBUSY; 270 } 271 272 sched_ctx = dev->data->dev_private; 273 274 for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++) 275 if (sched_ctx->slaves[slave_pos].dev_id == slave_id) 276 break; 277 if (slave_pos == sched_ctx->nb_slaves) { 278 CS_LOG_ERR("Cannot find slave"); 279 return -ENOTSUP; 280 } 281 282 if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) { 283 CS_LOG_ERR("Failed to detach slave"); 284 return -ENOTSUP; 285 } 286 287 for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) { 288 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1], 289 sizeof(struct scheduler_slave)); 290 } 291 memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0, 292 sizeof(struct scheduler_slave)); 293 sched_ctx->nb_slaves--; 294 295 if (update_scheduler_capability(sched_ctx) < 0) { 296 CS_LOG_ERR("capabilities update failed"); 297 return -ENOTSUP; 298 } 299 300 update_scheduler_feature_flag(dev); 301 302 update_max_nb_qp(sched_ctx); 303 304 return 0; 305 } 306 307 int 308 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id, 309 enum rte_cryptodev_scheduler_mode mode) 310 { 311 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 312 struct scheduler_ctx *sched_ctx; 313 314 if (!dev) { 315 CS_LOG_ERR("Operation not supported"); 316 return -ENOTSUP; 317 } 318 319 if (dev->driver_id != cryptodev_driver_id) { 320 CS_LOG_ERR("Operation not supported"); 321 return -ENOTSUP; 322 } 323 324 if (dev->data->dev_started) { 325 CS_LOG_ERR("Illegal operation"); 326 return -EBUSY; 327 } 328 329 sched_ctx = dev->data->dev_private; 330 331 if (mode == sched_ctx->mode) 332 return 0; 333 334 switch (mode) { 335 case CDEV_SCHED_MODE_ROUNDROBIN: 336 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 337 roundrobin_scheduler) < 0) { 338 CS_LOG_ERR("Failed to load scheduler"); 339 return -1; 340 } 341 break; 342 case CDEV_SCHED_MODE_PKT_SIZE_DISTR: 343 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 344 pkt_size_based_distr_scheduler) < 0) { 345 CS_LOG_ERR("Failed to load scheduler"); 346 return -1; 347 } 348 break; 349 case CDEV_SCHED_MODE_FAILOVER: 350 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 351 failover_scheduler) < 0) { 352 CS_LOG_ERR("Failed to load scheduler"); 353 return -1; 354 } 355 break; 356 case CDEV_SCHED_MODE_MULTICORE: 357 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id, 358 multicore_scheduler) < 0) { 359 CS_LOG_ERR("Failed to load scheduler"); 360 return -1; 361 } 362 break; 363 default: 364 CS_LOG_ERR("Not yet supported"); 365 return -ENOTSUP; 366 } 367 368 return 0; 369 } 370 371 enum rte_cryptodev_scheduler_mode 372 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id) 373 { 374 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 375 struct scheduler_ctx *sched_ctx; 376 377 if (!dev) { 378 CS_LOG_ERR("Operation not supported"); 379 return -ENOTSUP; 380 } 381 382 if (dev->driver_id != cryptodev_driver_id) { 383 CS_LOG_ERR("Operation not supported"); 384 return -ENOTSUP; 385 } 386 387 sched_ctx = dev->data->dev_private; 388 389 return sched_ctx->mode; 390 } 391 392 int 393 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id, 394 uint32_t enable_reorder) 395 { 396 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 397 struct scheduler_ctx *sched_ctx; 398 399 if (!dev) { 400 CS_LOG_ERR("Operation not supported"); 401 return -ENOTSUP; 402 } 403 404 if (dev->driver_id != cryptodev_driver_id) { 405 CS_LOG_ERR("Operation not supported"); 406 return -ENOTSUP; 407 } 408 409 if (dev->data->dev_started) { 410 CS_LOG_ERR("Illegal operation"); 411 return -EBUSY; 412 } 413 414 sched_ctx = dev->data->dev_private; 415 416 sched_ctx->reordering_enabled = enable_reorder; 417 418 return 0; 419 } 420 421 int 422 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id) 423 { 424 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 425 struct scheduler_ctx *sched_ctx; 426 427 if (!dev) { 428 CS_LOG_ERR("Operation not supported"); 429 return -ENOTSUP; 430 } 431 432 if (dev->driver_id != cryptodev_driver_id) { 433 CS_LOG_ERR("Operation not supported"); 434 return -ENOTSUP; 435 } 436 437 sched_ctx = dev->data->dev_private; 438 439 return (int)sched_ctx->reordering_enabled; 440 } 441 442 int 443 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, 444 struct rte_cryptodev_scheduler *scheduler) { 445 446 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 447 struct scheduler_ctx *sched_ctx; 448 449 if (!dev) { 450 CS_LOG_ERR("Operation not supported"); 451 return -ENOTSUP; 452 } 453 454 if (dev->driver_id != cryptodev_driver_id) { 455 CS_LOG_ERR("Operation not supported"); 456 return -ENOTSUP; 457 } 458 459 if (dev->data->dev_started) { 460 CS_LOG_ERR("Illegal operation"); 461 return -EBUSY; 462 } 463 464 sched_ctx = dev->data->dev_private; 465 466 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) { 467 CS_LOG_ERR("Invalid name %s, should be less than " 468 "%u bytes.\n", scheduler->name, 469 RTE_CRYPTODEV_NAME_MAX_LEN); 470 return -EINVAL; 471 } 472 snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s", 473 scheduler->name); 474 475 if (strlen(scheduler->description) > 476 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) { 477 CS_LOG_ERR("Invalid description %s, should be less than " 478 "%u bytes.\n", scheduler->description, 479 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1); 480 return -EINVAL; 481 } 482 snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s", 483 scheduler->description); 484 485 /* load scheduler instance operations functions */ 486 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair; 487 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx; 488 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start; 489 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop; 490 sched_ctx->ops.slave_attach = scheduler->ops->slave_attach; 491 sched_ctx->ops.slave_detach = scheduler->ops->slave_detach; 492 sched_ctx->ops.option_set = scheduler->ops->option_set; 493 sched_ctx->ops.option_get = scheduler->ops->option_get; 494 495 if (sched_ctx->private_ctx) { 496 rte_free(sched_ctx->private_ctx); 497 sched_ctx->private_ctx = NULL; 498 } 499 500 if (sched_ctx->ops.create_private_ctx) { 501 int ret = (*sched_ctx->ops.create_private_ctx)(dev); 502 503 if (ret < 0) { 504 CS_LOG_ERR("Unable to create scheduler private " 505 "context"); 506 return ret; 507 } 508 } 509 510 sched_ctx->mode = scheduler->mode; 511 512 return 0; 513 } 514 515 int 516 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves) 517 { 518 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 519 struct scheduler_ctx *sched_ctx; 520 uint32_t nb_slaves = 0; 521 522 if (!dev) { 523 CS_LOG_ERR("Operation not supported"); 524 return -ENOTSUP; 525 } 526 527 if (dev->driver_id != cryptodev_driver_id) { 528 CS_LOG_ERR("Operation not supported"); 529 return -ENOTSUP; 530 } 531 532 sched_ctx = dev->data->dev_private; 533 534 nb_slaves = sched_ctx->nb_slaves; 535 536 if (slaves && nb_slaves) { 537 uint32_t i; 538 539 for (i = 0; i < nb_slaves; i++) 540 slaves[i] = sched_ctx->slaves[i].dev_id; 541 } 542 543 return (int)nb_slaves; 544 } 545 546 int 547 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id, 548 enum rte_cryptodev_schedule_option_type option_type, 549 void *option) 550 { 551 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 552 struct scheduler_ctx *sched_ctx; 553 554 if (option_type == CDEV_SCHED_OPTION_NOT_SET || 555 option_type >= CDEV_SCHED_OPTION_COUNT) { 556 CS_LOG_ERR("Invalid option parameter"); 557 return -EINVAL; 558 } 559 560 if (!option) { 561 CS_LOG_ERR("Invalid option parameter"); 562 return -EINVAL; 563 } 564 565 if (dev->data->dev_started) { 566 CS_LOG_ERR("Illegal operation"); 567 return -EBUSY; 568 } 569 570 sched_ctx = dev->data->dev_private; 571 572 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP); 573 574 return (*sched_ctx->ops.option_set)(dev, option_type, option); 575 } 576 577 int 578 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id, 579 enum rte_cryptodev_schedule_option_type option_type, 580 void *option) 581 { 582 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id); 583 struct scheduler_ctx *sched_ctx; 584 585 if (!dev) { 586 CS_LOG_ERR("Operation not supported"); 587 return -ENOTSUP; 588 } 589 590 if (!option) { 591 CS_LOG_ERR("Invalid option parameter"); 592 return -EINVAL; 593 } 594 595 if (dev->driver_id != cryptodev_driver_id) { 596 CS_LOG_ERR("Operation not supported"); 597 return -ENOTSUP; 598 } 599 600 sched_ctx = dev->data->dev_private; 601 602 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP); 603 604 return (*sched_ctx->ops.option_get)(dev, option_type, option); 605 } 606