1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <inttypes.h> 6 #include <string.h> 7 8 #include <rte_bus_vdev.h> 9 #include <rte_lcore.h> 10 #include <rte_memzone.h> 11 #include <rte_kvargs.h> 12 #include <rte_errno.h> 13 #include <rte_cycles.h> 14 15 #include "opdl_evdev.h" 16 #include "opdl_ring.h" 17 #include "opdl_log.h" 18 19 #define EVENTDEV_NAME_OPDL_PMD event_opdl 20 #define NUMA_NODE_ARG "numa_node" 21 #define DO_VALIDATION_ARG "do_validation" 22 #define DO_TEST_ARG "self_test" 23 24 25 static void 26 opdl_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); 27 28 uint16_t 29 opdl_event_enqueue_burst(void *port, 30 const struct rte_event ev[], 31 uint16_t num) 32 { 33 struct opdl_port *p = port; 34 35 if (unlikely(!p->opdl->data->dev_started)) 36 return 0; 37 38 39 /* either rx_enqueue or disclaim*/ 40 return p->enq(p, ev, num); 41 } 42 43 uint16_t 44 opdl_event_enqueue(void *port, const struct rte_event *ev) 45 { 46 struct opdl_port *p = port; 47 48 if (unlikely(!p->opdl->data->dev_started)) 49 return 0; 50 51 52 return p->enq(p, ev, 1); 53 } 54 55 uint16_t 56 opdl_event_dequeue_burst(void *port, 57 struct rte_event *ev, 58 uint16_t num, 59 uint64_t wait) 60 { 61 struct opdl_port *p = (void *)port; 62 63 RTE_SET_USED(wait); 64 65 if (unlikely(!p->opdl->data->dev_started)) 66 return 0; 67 68 /* This function pointer can point to tx_dequeue or claim*/ 69 return p->deq(p, ev, num); 70 } 71 72 uint16_t 73 opdl_event_dequeue(void *port, 74 struct rte_event *ev, 75 uint64_t wait) 76 { 77 struct opdl_port *p = (void *)port; 78 79 if (unlikely(!p->opdl->data->dev_started)) 80 return 0; 81 82 RTE_SET_USED(wait); 83 84 return p->deq(p, ev, 1); 85 } 86 87 static int 88 opdl_port_link(struct rte_eventdev *dev, 89 void *port, 90 const uint8_t queues[], 91 const uint8_t priorities[], 92 uint16_t num) 93 { 94 struct opdl_port *p = port; 95 96 RTE_SET_USED(priorities); 97 RTE_SET_USED(dev); 98 99 if (unlikely(dev->data->dev_started)) { 100 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 101 "Attempt to link queue (%u) to port %d while device started\n", 102 dev->data->dev_id, 103 queues[0], 104 p->id); 105 rte_errno = -EINVAL; 106 return 0; 107 } 108 109 /* Max of 1 queue per port */ 110 if (num > 1) { 111 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 112 "Attempt to link more than one queue (%u) to port %d requested\n", 113 dev->data->dev_id, 114 num, 115 p->id); 116 rte_errno = -EDQUOT; 117 return 0; 118 } 119 120 if (!p->configured) { 121 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 122 "port %d not configured, cannot link to %u\n", 123 dev->data->dev_id, 124 p->id, 125 queues[0]); 126 rte_errno = -EINVAL; 127 return 0; 128 } 129 130 if (p->external_qid != OPDL_INVALID_QID) { 131 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 132 "port %d already linked to queue %u, cannot link to %u\n", 133 dev->data->dev_id, 134 p->id, 135 p->external_qid, 136 queues[0]); 137 rte_errno = -EINVAL; 138 return 0; 139 } 140 141 p->external_qid = queues[0]; 142 143 return 1; 144 } 145 146 static int 147 opdl_port_unlink(struct rte_eventdev *dev, 148 void *port, 149 uint8_t queues[], 150 uint16_t nb_unlinks) 151 { 152 struct opdl_port *p = port; 153 154 RTE_SET_USED(queues); 155 RTE_SET_USED(nb_unlinks); 156 157 if (unlikely(dev->data->dev_started)) { 158 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 159 "Attempt to unlink queue (%u) to port %d while device started\n", 160 dev->data->dev_id, 161 queues[0], 162 p->id); 163 rte_errno = -EINVAL; 164 return 0; 165 } 166 RTE_SET_USED(nb_unlinks); 167 168 /* Port Stuff */ 169 p->queue_id = OPDL_INVALID_QID; 170 p->p_type = OPDL_INVALID_PORT; 171 p->external_qid = OPDL_INVALID_QID; 172 173 /* always unlink 0 queue due to statice pipeline */ 174 return 0; 175 } 176 177 static int 178 opdl_port_setup(struct rte_eventdev *dev, 179 uint8_t port_id, 180 const struct rte_event_port_conf *conf) 181 { 182 struct opdl_evdev *device = opdl_pmd_priv(dev); 183 struct opdl_port *p = &device->ports[port_id]; 184 185 RTE_SET_USED(conf); 186 187 /* Check if port already configured */ 188 if (p->configured) { 189 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 190 "Attempt to setup port %d which is already setup\n", 191 dev->data->dev_id, 192 p->id); 193 return -EDQUOT; 194 } 195 196 *p = (struct opdl_port){0}; /* zero entire structure */ 197 p->id = port_id; 198 p->opdl = device; 199 p->queue_id = OPDL_INVALID_QID; 200 p->external_qid = OPDL_INVALID_QID; 201 dev->data->ports[port_id] = p; 202 rte_smp_wmb(); 203 p->configured = 1; 204 device->nb_ports++; 205 return 0; 206 } 207 208 static void 209 opdl_port_release(void *port) 210 { 211 struct opdl_port *p = (void *)port; 212 213 if (p == NULL || 214 p->opdl->data->dev_started) { 215 return; 216 } 217 218 p->configured = 0; 219 p->initialized = 0; 220 } 221 222 static void 223 opdl_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 224 struct rte_event_port_conf *port_conf) 225 { 226 RTE_SET_USED(dev); 227 RTE_SET_USED(port_id); 228 229 port_conf->new_event_threshold = MAX_OPDL_CONS_Q_DEPTH; 230 port_conf->dequeue_depth = MAX_OPDL_CONS_Q_DEPTH; 231 port_conf->enqueue_depth = MAX_OPDL_CONS_Q_DEPTH; 232 } 233 234 static int 235 opdl_queue_setup(struct rte_eventdev *dev, 236 uint8_t queue_id, 237 const struct rte_event_queue_conf *conf) 238 { 239 enum queue_type type; 240 241 struct opdl_evdev *device = opdl_pmd_priv(dev); 242 243 /* Extra sanity check, probably not needed */ 244 if (queue_id == OPDL_INVALID_QID) { 245 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 246 "Invalid queue id %u requested\n", 247 dev->data->dev_id, 248 queue_id); 249 return -EINVAL; 250 } 251 252 if (device->nb_q_md > device->max_queue_nb) { 253 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 254 "Max number of queues %u exceeded by request %u\n", 255 dev->data->dev_id, 256 device->max_queue_nb, 257 device->nb_q_md); 258 return -EINVAL; 259 } 260 261 if (RTE_EVENT_QUEUE_CFG_ALL_TYPES 262 & conf->event_queue_cfg) { 263 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 264 "QUEUE_CFG_ALL_TYPES not supported\n", 265 dev->data->dev_id); 266 return -ENOTSUP; 267 } else if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK 268 & conf->event_queue_cfg) { 269 type = OPDL_Q_TYPE_SINGLE_LINK; 270 } else { 271 switch (conf->schedule_type) { 272 case RTE_SCHED_TYPE_ORDERED: 273 type = OPDL_Q_TYPE_ORDERED; 274 break; 275 case RTE_SCHED_TYPE_ATOMIC: 276 type = OPDL_Q_TYPE_ATOMIC; 277 break; 278 case RTE_SCHED_TYPE_PARALLEL: 279 type = OPDL_Q_TYPE_ORDERED; 280 break; 281 default: 282 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 283 "Unknown queue type %d requested\n", 284 dev->data->dev_id, 285 conf->event_queue_cfg); 286 return -EINVAL; 287 } 288 } 289 /* Check if queue id has been setup already */ 290 uint32_t i; 291 for (i = 0; i < device->nb_q_md; i++) { 292 if (device->q_md[i].ext_id == queue_id) { 293 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 294 "queue id %u already setup\n", 295 dev->data->dev_id, 296 queue_id); 297 return -EINVAL; 298 } 299 } 300 301 device->q_md[device->nb_q_md].ext_id = queue_id; 302 device->q_md[device->nb_q_md].type = type; 303 device->q_md[device->nb_q_md].setup = 1; 304 device->nb_q_md++; 305 306 return 1; 307 } 308 309 static void 310 opdl_queue_release(struct rte_eventdev *dev, uint8_t queue_id) 311 { 312 struct opdl_evdev *device = opdl_pmd_priv(dev); 313 314 RTE_SET_USED(queue_id); 315 316 if (device->data->dev_started) 317 return; 318 319 } 320 321 static void 322 opdl_queue_def_conf(struct rte_eventdev *dev, 323 uint8_t queue_id, 324 struct rte_event_queue_conf *conf) 325 { 326 RTE_SET_USED(dev); 327 RTE_SET_USED(queue_id); 328 329 static const struct rte_event_queue_conf default_conf = { 330 .nb_atomic_flows = 1024, 331 .nb_atomic_order_sequences = 1, 332 .event_queue_cfg = 0, 333 .schedule_type = RTE_SCHED_TYPE_ORDERED, 334 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 335 }; 336 337 *conf = default_conf; 338 } 339 340 341 static int 342 opdl_dev_configure(const struct rte_eventdev *dev) 343 { 344 struct opdl_evdev *opdl = opdl_pmd_priv(dev); 345 const struct rte_eventdev_data *data = dev->data; 346 const struct rte_event_dev_config *conf = &data->dev_conf; 347 348 opdl->max_queue_nb = conf->nb_event_queues; 349 opdl->max_port_nb = conf->nb_event_ports; 350 opdl->nb_events_limit = conf->nb_events_limit; 351 352 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { 353 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : " 354 "DEQUEUE_TIMEOUT not supported\n", 355 dev->data->dev_id); 356 return -ENOTSUP; 357 } 358 359 return 0; 360 } 361 362 static void 363 opdl_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) 364 { 365 RTE_SET_USED(dev); 366 367 static const struct rte_event_dev_info evdev_opdl_info = { 368 .driver_name = OPDL_PMD_NAME, 369 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, 370 .max_event_queue_flows = OPDL_QID_NUM_FIDS, 371 .max_event_queue_priority_levels = OPDL_Q_PRIORITY_MAX, 372 .max_event_priority_levels = OPDL_IQS_MAX, 373 .max_event_ports = OPDL_PORTS_MAX, 374 .max_event_port_dequeue_depth = MAX_OPDL_CONS_Q_DEPTH, 375 .max_event_port_enqueue_depth = MAX_OPDL_CONS_Q_DEPTH, 376 .max_num_events = OPDL_INFLIGHT_EVENTS_TOTAL, 377 .event_dev_cap = RTE_EVENT_DEV_CAP_BURST_MODE, 378 }; 379 380 *info = evdev_opdl_info; 381 } 382 383 static void 384 opdl_dump(struct rte_eventdev *dev, FILE *f) 385 { 386 struct opdl_evdev *device = opdl_pmd_priv(dev); 387 388 if (!device->do_validation) 389 return; 390 391 fprintf(f, 392 "\n\n -- RING STATISTICS --\n"); 393 uint32_t i; 394 for (i = 0; i < device->nb_opdls; i++) 395 opdl_ring_dump(device->opdl[i], f); 396 397 fprintf(f, 398 "\n\n -- PORT STATISTICS --\n" 399 "Type Port Index Port Id Queue Id Av. Req Size " 400 "Av. Grant Size Av. Cycles PP" 401 " Empty DEQs Non Empty DEQs Pkts Processed\n"); 402 403 for (i = 0; i < device->max_port_nb; i++) { 404 char queue_id[64]; 405 char total_cyc[64]; 406 const char *p_type; 407 408 uint64_t cne, cpg; 409 struct opdl_port *port = &device->ports[i]; 410 411 if (port->initialized) { 412 cne = port->port_stat[claim_non_empty]; 413 cpg = port->port_stat[claim_pkts_granted]; 414 if (port->p_type == OPDL_REGULAR_PORT) 415 p_type = "REG"; 416 else if (port->p_type == OPDL_PURE_RX_PORT) 417 p_type = " RX"; 418 else if (port->p_type == OPDL_PURE_TX_PORT) 419 p_type = " TX"; 420 else if (port->p_type == OPDL_ASYNC_PORT) 421 p_type = "SYNC"; 422 else 423 p_type = "????"; 424 425 sprintf(queue_id, "%02u", port->external_qid); 426 if (port->p_type == OPDL_REGULAR_PORT || 427 port->p_type == OPDL_ASYNC_PORT) 428 sprintf(total_cyc, 429 " %'16"PRIu64"", 430 (cpg != 0 ? 431 port->port_stat[total_cycles] / cpg 432 : 0)); 433 else 434 sprintf(total_cyc, 435 " ----"); 436 fprintf(f, 437 "%4s %10u %8u %9s %'16"PRIu64" %'16"PRIu64" %s " 438 "%'16"PRIu64" %'16"PRIu64" %'16"PRIu64"\n", 439 p_type, 440 i, 441 port->id, 442 (port->external_qid == OPDL_INVALID_QID ? "---" 443 : queue_id), 444 (cne != 0 ? 445 port->port_stat[claim_pkts_requested] / cne 446 : 0), 447 (cne != 0 ? 448 port->port_stat[claim_pkts_granted] / cne 449 : 0), 450 total_cyc, 451 port->port_stat[claim_empty], 452 port->port_stat[claim_non_empty], 453 port->port_stat[claim_pkts_granted]); 454 } 455 } 456 fprintf(f, "\n"); 457 } 458 459 460 static void 461 opdl_stop(struct rte_eventdev *dev) 462 { 463 struct opdl_evdev *device = opdl_pmd_priv(dev); 464 465 opdl_xstats_uninit(dev); 466 467 destroy_queues_and_rings(dev); 468 469 470 device->started = 0; 471 472 rte_smp_wmb(); 473 } 474 475 static int 476 opdl_start(struct rte_eventdev *dev) 477 { 478 int err = 0; 479 480 if (!err) 481 err = create_queues_and_rings(dev); 482 483 484 if (!err) 485 err = assign_internal_queue_ids(dev); 486 487 488 if (!err) 489 err = initialise_queue_zero_ports(dev); 490 491 492 if (!err) 493 err = initialise_all_other_ports(dev); 494 495 496 if (!err) 497 err = check_queues_linked(dev); 498 499 500 if (!err) 501 err = opdl_add_event_handlers(dev); 502 503 504 if (!err) 505 err = build_all_dependencies(dev); 506 507 if (!err) { 508 opdl_xstats_init(dev); 509 510 struct opdl_evdev *device = opdl_pmd_priv(dev); 511 512 PMD_DRV_LOG(INFO, "DEV_ID:[%02d] : " 513 "SUCCESS : Created %u total queues (%u ex, %u in)," 514 " %u opdls, %u event_dev ports, %u input ports", 515 opdl_pmd_dev_id(device), 516 device->nb_queues, 517 (device->nb_queues - device->nb_opdls), 518 device->nb_opdls, 519 device->nb_opdls, 520 device->nb_ports, 521 device->queue[0].nb_ports); 522 } else 523 opdl_stop(dev); 524 525 return err; 526 } 527 528 static int 529 opdl_close(struct rte_eventdev *dev) 530 { 531 struct opdl_evdev *device = opdl_pmd_priv(dev); 532 uint32_t i; 533 534 for (i = 0; i < device->max_port_nb; i++) { 535 memset(&device->ports[i], 536 0, 537 sizeof(struct opdl_port)); 538 } 539 540 memset(&device->s_md, 541 0x0, 542 sizeof(struct opdl_stage_meta_data)*OPDL_PORTS_MAX); 543 544 memset(&device->q_md, 545 0xFF, 546 sizeof(struct opdl_queue_meta_data)*OPDL_MAX_QUEUES); 547 548 549 memset(device->q_map_ex_to_in, 550 0, 551 sizeof(uint8_t)*OPDL_INVALID_QID); 552 553 opdl_xstats_uninit(dev); 554 555 device->max_port_nb = 0; 556 557 device->max_queue_nb = 0; 558 559 device->nb_opdls = 0; 560 561 device->nb_queues = 0; 562 563 device->nb_ports = 0; 564 565 device->nb_q_md = 0; 566 567 dev->data->nb_queues = 0; 568 569 dev->data->nb_ports = 0; 570 571 572 return 0; 573 } 574 575 static int 576 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) 577 { 578 int *socket_id = opaque; 579 *socket_id = atoi(value); 580 if (*socket_id >= RTE_MAX_NUMA_NODES) 581 return -1; 582 return 0; 583 } 584 585 static int 586 set_do_validation(const char *key __rte_unused, const char *value, void *opaque) 587 { 588 int *do_val = opaque; 589 *do_val = atoi(value); 590 if (*do_val != 0) 591 *do_val = 1; 592 593 return 0; 594 } 595 static int 596 set_do_test(const char *key __rte_unused, const char *value, void *opaque) 597 { 598 int *do_test = opaque; 599 600 *do_test = atoi(value); 601 602 if (*do_test != 0) 603 *do_test = 1; 604 return 0; 605 } 606 607 static int 608 opdl_probe(struct rte_vdev_device *vdev) 609 { 610 static const struct rte_eventdev_ops evdev_opdl_ops = { 611 .dev_configure = opdl_dev_configure, 612 .dev_infos_get = opdl_info_get, 613 .dev_close = opdl_close, 614 .dev_start = opdl_start, 615 .dev_stop = opdl_stop, 616 .dump = opdl_dump, 617 618 .queue_def_conf = opdl_queue_def_conf, 619 .queue_setup = opdl_queue_setup, 620 .queue_release = opdl_queue_release, 621 .port_def_conf = opdl_port_def_conf, 622 .port_setup = opdl_port_setup, 623 .port_release = opdl_port_release, 624 .port_link = opdl_port_link, 625 .port_unlink = opdl_port_unlink, 626 627 628 .xstats_get = opdl_xstats_get, 629 .xstats_get_names = opdl_xstats_get_names, 630 .xstats_get_by_name = opdl_xstats_get_by_name, 631 .xstats_reset = opdl_xstats_reset, 632 }; 633 634 static const char *const args[] = { 635 NUMA_NODE_ARG, 636 DO_VALIDATION_ARG, 637 DO_TEST_ARG, 638 NULL 639 }; 640 const char *name; 641 const char *params; 642 struct rte_eventdev *dev; 643 struct opdl_evdev *opdl; 644 int socket_id = rte_socket_id(); 645 int do_validation = 0; 646 int do_test = 0; 647 int str_len; 648 int test_result = 0; 649 650 name = rte_vdev_device_name(vdev); 651 params = rte_vdev_device_args(vdev); 652 if (params != NULL && params[0] != '\0') { 653 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); 654 655 if (!kvlist) { 656 PMD_DRV_LOG(INFO, 657 "Ignoring unsupported parameters when creating device '%s'\n", 658 name); 659 } else { 660 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, 661 assign_numa_node, &socket_id); 662 if (ret != 0) { 663 PMD_DRV_LOG(ERR, 664 "%s: Error parsing numa node parameter", 665 name); 666 667 rte_kvargs_free(kvlist); 668 return ret; 669 } 670 671 ret = rte_kvargs_process(kvlist, DO_VALIDATION_ARG, 672 set_do_validation, &do_validation); 673 if (ret != 0) { 674 PMD_DRV_LOG(ERR, 675 "%s: Error parsing do validation parameter", 676 name); 677 rte_kvargs_free(kvlist); 678 return ret; 679 } 680 681 ret = rte_kvargs_process(kvlist, DO_TEST_ARG, 682 set_do_test, &do_test); 683 if (ret != 0) { 684 PMD_DRV_LOG(ERR, 685 "%s: Error parsing do test parameter", 686 name); 687 rte_kvargs_free(kvlist); 688 return ret; 689 } 690 691 rte_kvargs_free(kvlist); 692 } 693 } 694 dev = rte_event_pmd_vdev_init(name, 695 sizeof(struct opdl_evdev), socket_id); 696 697 if (dev == NULL) { 698 PMD_DRV_LOG(ERR, "eventdev vdev init() failed"); 699 return -EFAULT; 700 } 701 702 PMD_DRV_LOG(INFO, "DEV_ID:[%02d] : " 703 "Success - creating eventdev device %s, numa_node:[%d], do_valdation:[%s]" 704 " , self_test:[%s]\n", 705 dev->data->dev_id, 706 name, 707 socket_id, 708 (do_validation ? "true" : "false"), 709 (do_test ? "true" : "false")); 710 711 dev->dev_ops = &evdev_opdl_ops; 712 713 dev->enqueue = opdl_event_enqueue; 714 dev->enqueue_burst = opdl_event_enqueue_burst; 715 dev->enqueue_new_burst = opdl_event_enqueue_burst; 716 dev->enqueue_forward_burst = opdl_event_enqueue_burst; 717 dev->dequeue = opdl_event_dequeue; 718 dev->dequeue_burst = opdl_event_dequeue_burst; 719 720 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 721 return 0; 722 723 opdl = dev->data->dev_private; 724 opdl->data = dev->data; 725 opdl->socket = socket_id; 726 opdl->do_validation = do_validation; 727 opdl->do_test = do_test; 728 str_len = strlen(name); 729 memcpy(opdl->service_name, name, str_len); 730 731 if (do_test == 1) 732 test_result = opdl_selftest(); 733 734 return test_result; 735 } 736 737 static int 738 opdl_remove(struct rte_vdev_device *vdev) 739 { 740 const char *name; 741 742 name = rte_vdev_device_name(vdev); 743 if (name == NULL) 744 return -EINVAL; 745 746 PMD_DRV_LOG(INFO, "Closing eventdev opdl device %s\n", name); 747 748 return rte_event_pmd_vdev_uninit(name); 749 } 750 751 static struct rte_vdev_driver evdev_opdl_pmd_drv = { 752 .probe = opdl_probe, 753 .remove = opdl_remove 754 }; 755 756 RTE_INIT(opdl_init_log); 757 758 static void 759 opdl_init_log(void) 760 { 761 opdl_logtype_driver = rte_log_register("pmd.event.opdl.driver"); 762 if (opdl_logtype_driver >= 0) 763 rte_log_set_level(opdl_logtype_driver, RTE_LOG_INFO); 764 } 765 766 767 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_OPDL_PMD, evdev_opdl_pmd_drv); 768 RTE_PMD_REGISTER_PARAM_STRING(event_opdl, NUMA_NODE_ARG "=<int>" 769 DO_VALIDATION_ARG "=<int>" DO_TEST_ARG "=<int>"); 770