1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include "rte_eth_ring.h" 6 #include <rte_mbuf.h> 7 #include <rte_ethdev_driver.h> 8 #include <rte_malloc.h> 9 #include <rte_memcpy.h> 10 #include <rte_string_fns.h> 11 #include <rte_bus_vdev.h> 12 #include <rte_kvargs.h> 13 #include <rte_errno.h> 14 15 #define ETH_RING_NUMA_NODE_ACTION_ARG "nodeaction" 16 #define ETH_RING_ACTION_CREATE "CREATE" 17 #define ETH_RING_ACTION_ATTACH "ATTACH" 18 #define ETH_RING_INTERNAL_ARG "internal" 19 20 static const char *valid_arguments[] = { 21 ETH_RING_NUMA_NODE_ACTION_ARG, 22 ETH_RING_INTERNAL_ARG, 23 NULL 24 }; 25 26 struct ring_internal_args { 27 struct rte_ring * const *rx_queues; 28 const unsigned int nb_rx_queues; 29 struct rte_ring * const *tx_queues; 30 const unsigned int nb_tx_queues; 31 const unsigned int numa_node; 32 void *addr; /* self addr for sanity check */ 33 }; 34 35 enum dev_action { 36 DEV_CREATE, 37 DEV_ATTACH 38 }; 39 40 struct ring_queue { 41 struct rte_ring *rng; 42 rte_atomic64_t rx_pkts; 43 rte_atomic64_t tx_pkts; 44 rte_atomic64_t err_pkts; 45 }; 46 47 struct pmd_internals { 48 unsigned max_rx_queues; 49 unsigned max_tx_queues; 50 51 struct ring_queue rx_ring_queues[RTE_PMD_RING_MAX_RX_RINGS]; 52 struct ring_queue tx_ring_queues[RTE_PMD_RING_MAX_TX_RINGS]; 53 54 struct ether_addr address; 55 enum dev_action action; 56 }; 57 58 59 static struct rte_eth_link pmd_link = { 60 .link_speed = ETH_SPEED_NUM_10G, 61 .link_duplex = ETH_LINK_FULL_DUPLEX, 62 .link_status = ETH_LINK_DOWN, 63 .link_autoneg = ETH_LINK_FIXED, 64 }; 65 66 static int eth_ring_logtype; 67 68 #define PMD_LOG(level, fmt, args...) \ 69 rte_log(RTE_LOG_ ## level, eth_ring_logtype, \ 70 "%s(): " fmt "\n", __func__, ##args) 71 72 static uint16_t 73 eth_ring_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 74 { 75 void **ptrs = (void *)&bufs[0]; 76 struct ring_queue *r = q; 77 const uint16_t nb_rx = (uint16_t)rte_ring_dequeue_burst(r->rng, 78 ptrs, nb_bufs, NULL); 79 if (r->rng->flags & RING_F_SC_DEQ) 80 r->rx_pkts.cnt += nb_rx; 81 else 82 rte_atomic64_add(&(r->rx_pkts), nb_rx); 83 return nb_rx; 84 } 85 86 static uint16_t 87 eth_ring_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 88 { 89 void **ptrs = (void *)&bufs[0]; 90 struct ring_queue *r = q; 91 const uint16_t nb_tx = (uint16_t)rte_ring_enqueue_burst(r->rng, 92 ptrs, nb_bufs, NULL); 93 if (r->rng->flags & RING_F_SP_ENQ) { 94 r->tx_pkts.cnt += nb_tx; 95 r->err_pkts.cnt += nb_bufs - nb_tx; 96 } else { 97 rte_atomic64_add(&(r->tx_pkts), nb_tx); 98 rte_atomic64_add(&(r->err_pkts), nb_bufs - nb_tx); 99 } 100 return nb_tx; 101 } 102 103 static int 104 eth_dev_configure(struct rte_eth_dev *dev __rte_unused) { return 0; } 105 106 static int 107 eth_dev_start(struct rte_eth_dev *dev) 108 { 109 dev->data->dev_link.link_status = ETH_LINK_UP; 110 return 0; 111 } 112 113 static void 114 eth_dev_stop(struct rte_eth_dev *dev) 115 { 116 dev->data->dev_link.link_status = ETH_LINK_DOWN; 117 } 118 119 static int 120 eth_dev_set_link_down(struct rte_eth_dev *dev) 121 { 122 dev->data->dev_link.link_status = ETH_LINK_DOWN; 123 return 0; 124 } 125 126 static int 127 eth_dev_set_link_up(struct rte_eth_dev *dev) 128 { 129 dev->data->dev_link.link_status = ETH_LINK_UP; 130 return 0; 131 } 132 133 static int 134 eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, 135 uint16_t nb_rx_desc __rte_unused, 136 unsigned int socket_id __rte_unused, 137 const struct rte_eth_rxconf *rx_conf __rte_unused, 138 struct rte_mempool *mb_pool __rte_unused) 139 { 140 struct pmd_internals *internals = dev->data->dev_private; 141 dev->data->rx_queues[rx_queue_id] = &internals->rx_ring_queues[rx_queue_id]; 142 return 0; 143 } 144 145 static int 146 eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, 147 uint16_t nb_tx_desc __rte_unused, 148 unsigned int socket_id __rte_unused, 149 const struct rte_eth_txconf *tx_conf __rte_unused) 150 { 151 struct pmd_internals *internals = dev->data->dev_private; 152 dev->data->tx_queues[tx_queue_id] = &internals->tx_ring_queues[tx_queue_id]; 153 return 0; 154 } 155 156 157 static void 158 eth_dev_info(struct rte_eth_dev *dev, 159 struct rte_eth_dev_info *dev_info) 160 { 161 struct pmd_internals *internals = dev->data->dev_private; 162 dev_info->max_mac_addrs = 1; 163 dev_info->max_rx_pktlen = (uint32_t)-1; 164 dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues; 165 dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues; 166 dev_info->min_rx_bufsize = 0; 167 } 168 169 static int 170 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 171 { 172 unsigned i; 173 unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0; 174 const struct pmd_internals *internal = dev->data->dev_private; 175 176 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 177 i < dev->data->nb_rx_queues; i++) { 178 stats->q_ipackets[i] = internal->rx_ring_queues[i].rx_pkts.cnt; 179 rx_total += stats->q_ipackets[i]; 180 } 181 182 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 183 i < dev->data->nb_tx_queues; i++) { 184 stats->q_opackets[i] = internal->tx_ring_queues[i].tx_pkts.cnt; 185 stats->q_errors[i] = internal->tx_ring_queues[i].err_pkts.cnt; 186 tx_total += stats->q_opackets[i]; 187 tx_err_total += stats->q_errors[i]; 188 } 189 190 stats->ipackets = rx_total; 191 stats->opackets = tx_total; 192 stats->oerrors = tx_err_total; 193 194 return 0; 195 } 196 197 static void 198 eth_stats_reset(struct rte_eth_dev *dev) 199 { 200 unsigned i; 201 struct pmd_internals *internal = dev->data->dev_private; 202 for (i = 0; i < dev->data->nb_rx_queues; i++) 203 internal->rx_ring_queues[i].rx_pkts.cnt = 0; 204 for (i = 0; i < dev->data->nb_tx_queues; i++) { 205 internal->tx_ring_queues[i].tx_pkts.cnt = 0; 206 internal->tx_ring_queues[i].err_pkts.cnt = 0; 207 } 208 } 209 210 static void 211 eth_mac_addr_remove(struct rte_eth_dev *dev __rte_unused, 212 uint32_t index __rte_unused) 213 { 214 } 215 216 static int 217 eth_mac_addr_add(struct rte_eth_dev *dev __rte_unused, 218 struct ether_addr *mac_addr __rte_unused, 219 uint32_t index __rte_unused, 220 uint32_t vmdq __rte_unused) 221 { 222 return 0; 223 } 224 225 static void 226 eth_queue_release(void *q __rte_unused) { ; } 227 static int 228 eth_link_update(struct rte_eth_dev *dev __rte_unused, 229 int wait_to_complete __rte_unused) { return 0; } 230 231 static const struct eth_dev_ops ops = { 232 .dev_start = eth_dev_start, 233 .dev_stop = eth_dev_stop, 234 .dev_set_link_up = eth_dev_set_link_up, 235 .dev_set_link_down = eth_dev_set_link_down, 236 .dev_configure = eth_dev_configure, 237 .dev_infos_get = eth_dev_info, 238 .rx_queue_setup = eth_rx_queue_setup, 239 .tx_queue_setup = eth_tx_queue_setup, 240 .rx_queue_release = eth_queue_release, 241 .tx_queue_release = eth_queue_release, 242 .link_update = eth_link_update, 243 .stats_get = eth_stats_get, 244 .stats_reset = eth_stats_reset, 245 .mac_addr_remove = eth_mac_addr_remove, 246 .mac_addr_add = eth_mac_addr_add, 247 }; 248 249 static struct rte_vdev_driver pmd_ring_drv; 250 251 static int 252 do_eth_dev_ring_create(const char *name, 253 struct rte_ring * const rx_queues[], const unsigned nb_rx_queues, 254 struct rte_ring *const tx_queues[], const unsigned nb_tx_queues, 255 const unsigned int numa_node, enum dev_action action, 256 struct rte_eth_dev **eth_dev_p) 257 { 258 struct rte_eth_dev_data *data = NULL; 259 struct pmd_internals *internals = NULL; 260 struct rte_eth_dev *eth_dev = NULL; 261 void **rx_queues_local = NULL; 262 void **tx_queues_local = NULL; 263 unsigned i; 264 265 PMD_LOG(INFO, "Creating rings-backed ethdev on numa socket %u", 266 numa_node); 267 268 rx_queues_local = rte_zmalloc_socket(name, 269 sizeof(void *) * nb_rx_queues, 0, numa_node); 270 if (rx_queues_local == NULL) { 271 rte_errno = ENOMEM; 272 goto error; 273 } 274 275 tx_queues_local = rte_zmalloc_socket(name, 276 sizeof(void *) * nb_tx_queues, 0, numa_node); 277 if (tx_queues_local == NULL) { 278 rte_errno = ENOMEM; 279 goto error; 280 } 281 282 internals = rte_zmalloc_socket(name, sizeof(*internals), 0, numa_node); 283 if (internals == NULL) { 284 rte_errno = ENOMEM; 285 goto error; 286 } 287 288 /* reserve an ethdev entry */ 289 eth_dev = rte_eth_dev_allocate(name); 290 if (eth_dev == NULL) { 291 rte_errno = ENOSPC; 292 goto error; 293 } 294 295 /* now put it all together 296 * - store queue data in internals, 297 * - store numa_node info in eth_dev_data 298 * - point eth_dev_data to internals 299 * - and point eth_dev structure to new eth_dev_data structure 300 */ 301 302 data = eth_dev->data; 303 data->rx_queues = rx_queues_local; 304 data->tx_queues = tx_queues_local; 305 306 internals->action = action; 307 internals->max_rx_queues = nb_rx_queues; 308 internals->max_tx_queues = nb_tx_queues; 309 for (i = 0; i < nb_rx_queues; i++) { 310 internals->rx_ring_queues[i].rng = rx_queues[i]; 311 data->rx_queues[i] = &internals->rx_ring_queues[i]; 312 } 313 for (i = 0; i < nb_tx_queues; i++) { 314 internals->tx_ring_queues[i].rng = tx_queues[i]; 315 data->tx_queues[i] = &internals->tx_ring_queues[i]; 316 } 317 318 data->dev_private = internals; 319 data->nb_rx_queues = (uint16_t)nb_rx_queues; 320 data->nb_tx_queues = (uint16_t)nb_tx_queues; 321 data->dev_link = pmd_link; 322 data->mac_addrs = &internals->address; 323 324 eth_dev->dev_ops = &ops; 325 data->kdrv = RTE_KDRV_NONE; 326 data->numa_node = numa_node; 327 328 /* finally assign rx and tx ops */ 329 eth_dev->rx_pkt_burst = eth_ring_rx; 330 eth_dev->tx_pkt_burst = eth_ring_tx; 331 332 rte_eth_dev_probing_finish(eth_dev); 333 *eth_dev_p = eth_dev; 334 335 return data->port_id; 336 337 error: 338 rte_free(rx_queues_local); 339 rte_free(tx_queues_local); 340 rte_free(internals); 341 342 return -1; 343 } 344 345 int 346 rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[], 347 const unsigned nb_rx_queues, 348 struct rte_ring *const tx_queues[], 349 const unsigned nb_tx_queues, 350 const unsigned numa_node) 351 { 352 struct ring_internal_args args = { 353 .rx_queues = rx_queues, 354 .nb_rx_queues = nb_rx_queues, 355 .tx_queues = tx_queues, 356 .nb_tx_queues = nb_tx_queues, 357 .numa_node = numa_node, 358 .addr = &args, 359 }; 360 char args_str[32] = { 0 }; 361 char ring_name[32] = { 0 }; 362 uint16_t port_id = RTE_MAX_ETHPORTS; 363 int ret; 364 365 /* do some parameter checking */ 366 if (rx_queues == NULL && nb_rx_queues > 0) { 367 rte_errno = EINVAL; 368 return -1; 369 } 370 if (tx_queues == NULL && nb_tx_queues > 0) { 371 rte_errno = EINVAL; 372 return -1; 373 } 374 if (nb_rx_queues > RTE_PMD_RING_MAX_RX_RINGS) { 375 rte_errno = EINVAL; 376 return -1; 377 } 378 379 snprintf(args_str, 32, "%s=%p", ETH_RING_INTERNAL_ARG, &args); 380 snprintf(ring_name, 32, "net_ring_%s", name); 381 382 ret = rte_vdev_init(ring_name, args_str); 383 if (ret) { 384 rte_errno = EINVAL; 385 return -1; 386 } 387 388 rte_eth_dev_get_port_by_name(ring_name, &port_id); 389 390 return port_id; 391 } 392 393 int 394 rte_eth_from_ring(struct rte_ring *r) 395 { 396 return rte_eth_from_rings(r->name, &r, 1, &r, 1, 397 r->memzone ? r->memzone->socket_id : SOCKET_ID_ANY); 398 } 399 400 static int 401 eth_dev_ring_create(const char *name, const unsigned numa_node, 402 enum dev_action action, struct rte_eth_dev **eth_dev) 403 { 404 /* rx and tx are so-called from point of view of first port. 405 * They are inverted from the point of view of second port 406 */ 407 struct rte_ring *rxtx[RTE_PMD_RING_MAX_RX_RINGS]; 408 unsigned i; 409 char rng_name[RTE_RING_NAMESIZE]; 410 unsigned num_rings = RTE_MIN(RTE_PMD_RING_MAX_RX_RINGS, 411 RTE_PMD_RING_MAX_TX_RINGS); 412 413 for (i = 0; i < num_rings; i++) { 414 snprintf(rng_name, sizeof(rng_name), "ETH_RXTX%u_%s", i, name); 415 rxtx[i] = (action == DEV_CREATE) ? 416 rte_ring_create(rng_name, 1024, numa_node, 417 RING_F_SP_ENQ|RING_F_SC_DEQ) : 418 rte_ring_lookup(rng_name); 419 if (rxtx[i] == NULL) 420 return -1; 421 } 422 423 if (do_eth_dev_ring_create(name, rxtx, num_rings, rxtx, num_rings, 424 numa_node, action, eth_dev) < 0) 425 return -1; 426 427 return 0; 428 } 429 430 struct node_action_pair { 431 char name[PATH_MAX]; 432 unsigned node; 433 enum dev_action action; 434 }; 435 436 struct node_action_list { 437 unsigned total; 438 unsigned count; 439 struct node_action_pair *list; 440 }; 441 442 static int parse_kvlist (const char *key __rte_unused, const char *value, void *data) 443 { 444 struct node_action_list *info = data; 445 int ret; 446 char *name; 447 char *action; 448 char *node; 449 char *end; 450 451 name = strdup(value); 452 453 ret = -EINVAL; 454 455 if (!name) { 456 PMD_LOG(WARNING, "command line parameter is empty for ring pmd!"); 457 goto out; 458 } 459 460 node = strchr(name, ':'); 461 if (!node) { 462 PMD_LOG(WARNING, "could not parse node value from %s", 463 name); 464 goto out; 465 } 466 467 *node = '\0'; 468 node++; 469 470 action = strchr(node, ':'); 471 if (!action) { 472 PMD_LOG(WARNING, "could not parse action value from %s", 473 node); 474 goto out; 475 } 476 477 *action = '\0'; 478 action++; 479 480 /* 481 * Need to do some sanity checking here 482 */ 483 484 if (strcmp(action, ETH_RING_ACTION_ATTACH) == 0) 485 info->list[info->count].action = DEV_ATTACH; 486 else if (strcmp(action, ETH_RING_ACTION_CREATE) == 0) 487 info->list[info->count].action = DEV_CREATE; 488 else 489 goto out; 490 491 errno = 0; 492 info->list[info->count].node = strtol(node, &end, 10); 493 494 if ((errno != 0) || (*end != '\0')) { 495 PMD_LOG(WARNING, 496 "node value %s is unparseable as a number", node); 497 goto out; 498 } 499 500 snprintf(info->list[info->count].name, sizeof(info->list[info->count].name), "%s", name); 501 502 info->count++; 503 504 ret = 0; 505 out: 506 free(name); 507 return ret; 508 } 509 510 static int 511 parse_internal_args(const char *key __rte_unused, const char *value, 512 void *data) 513 { 514 struct ring_internal_args **internal_args = data; 515 void *args; 516 517 sscanf(value, "%p", &args); 518 519 *internal_args = args; 520 521 if ((*internal_args)->addr != args) 522 return -1; 523 524 return 0; 525 } 526 527 static int 528 rte_pmd_ring_probe(struct rte_vdev_device *dev) 529 { 530 const char *name, *params; 531 struct rte_kvargs *kvlist = NULL; 532 int ret = 0; 533 struct node_action_list *info = NULL; 534 struct rte_eth_dev *eth_dev = NULL; 535 struct ring_internal_args *internal_args; 536 537 name = rte_vdev_device_name(dev); 538 params = rte_vdev_device_args(dev); 539 540 PMD_LOG(INFO, "Initializing pmd_ring for %s", name); 541 542 if (params == NULL || params[0] == '\0') { 543 ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE, 544 ð_dev); 545 if (ret == -1) { 546 PMD_LOG(INFO, 547 "Attach to pmd_ring for %s", name); 548 ret = eth_dev_ring_create(name, rte_socket_id(), 549 DEV_ATTACH, ð_dev); 550 } 551 } else { 552 kvlist = rte_kvargs_parse(params, valid_arguments); 553 554 if (!kvlist) { 555 PMD_LOG(INFO, "Ignoring unsupported parameters when creating" 556 " rings-backed ethernet device"); 557 ret = eth_dev_ring_create(name, rte_socket_id(), 558 DEV_CREATE, ð_dev); 559 if (ret == -1) { 560 PMD_LOG(INFO, 561 "Attach to pmd_ring for %s", 562 name); 563 ret = eth_dev_ring_create(name, rte_socket_id(), 564 DEV_ATTACH, ð_dev); 565 } 566 567 if (eth_dev) 568 eth_dev->device = &dev->device; 569 570 return ret; 571 } 572 573 if (rte_kvargs_count(kvlist, ETH_RING_INTERNAL_ARG) == 1) { 574 ret = rte_kvargs_process(kvlist, ETH_RING_INTERNAL_ARG, 575 parse_internal_args, 576 &internal_args); 577 if (ret < 0) 578 goto out_free; 579 580 ret = do_eth_dev_ring_create(name, 581 internal_args->rx_queues, 582 internal_args->nb_rx_queues, 583 internal_args->tx_queues, 584 internal_args->nb_tx_queues, 585 internal_args->numa_node, 586 DEV_ATTACH, 587 ð_dev); 588 if (ret >= 0) 589 ret = 0; 590 } else { 591 ret = rte_kvargs_count(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG); 592 info = rte_zmalloc("struct node_action_list", 593 sizeof(struct node_action_list) + 594 (sizeof(struct node_action_pair) * ret), 595 0); 596 if (!info) 597 goto out_free; 598 599 info->total = ret; 600 info->list = (struct node_action_pair*)(info + 1); 601 602 ret = rte_kvargs_process(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG, 603 parse_kvlist, info); 604 605 if (ret < 0) 606 goto out_free; 607 608 for (info->count = 0; info->count < info->total; info->count++) { 609 ret = eth_dev_ring_create(info->list[info->count].name, 610 info->list[info->count].node, 611 info->list[info->count].action, 612 ð_dev); 613 if ((ret == -1) && 614 (info->list[info->count].action == DEV_CREATE)) { 615 PMD_LOG(INFO, 616 "Attach to pmd_ring for %s", 617 name); 618 ret = eth_dev_ring_create(name, 619 info->list[info->count].node, 620 DEV_ATTACH, 621 ð_dev); 622 } 623 } 624 } 625 } 626 627 if (eth_dev) 628 eth_dev->device = &dev->device; 629 630 out_free: 631 rte_kvargs_free(kvlist); 632 rte_free(info); 633 return ret; 634 } 635 636 static int 637 rte_pmd_ring_remove(struct rte_vdev_device *dev) 638 { 639 const char *name = rte_vdev_device_name(dev); 640 struct rte_eth_dev *eth_dev = NULL; 641 struct pmd_internals *internals = NULL; 642 struct ring_queue *r = NULL; 643 uint16_t i; 644 645 PMD_LOG(INFO, "Un-Initializing pmd_ring for %s", name); 646 647 if (name == NULL) 648 return -EINVAL; 649 650 /* find an ethdev entry */ 651 eth_dev = rte_eth_dev_allocated(name); 652 if (eth_dev == NULL) 653 return -ENODEV; 654 655 eth_dev_stop(eth_dev); 656 657 internals = eth_dev->data->dev_private; 658 if (internals->action == DEV_CREATE) { 659 /* 660 * it is only necessary to delete the rings in rx_queues because 661 * they are the same used in tx_queues 662 */ 663 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 664 r = eth_dev->data->rx_queues[i]; 665 rte_ring_free(r->rng); 666 } 667 } 668 669 /* mac_addrs must not be freed alone because part of dev_private */ 670 eth_dev->data->mac_addrs = NULL; 671 rte_eth_dev_release_port(eth_dev); 672 return 0; 673 } 674 675 static struct rte_vdev_driver pmd_ring_drv = { 676 .probe = rte_pmd_ring_probe, 677 .remove = rte_pmd_ring_remove, 678 }; 679 680 RTE_PMD_REGISTER_VDEV(net_ring, pmd_ring_drv); 681 RTE_PMD_REGISTER_ALIAS(net_ring, eth_ring); 682 RTE_PMD_REGISTER_PARAM_STRING(net_ring, 683 ETH_RING_NUMA_NODE_ACTION_ARG "=name:node:action(ATTACH|CREATE)"); 684 685 RTE_INIT(eth_ring_init_log) 686 { 687 eth_ring_logtype = rte_log_register("pmd.net.ring"); 688 if (eth_ring_logtype >= 0) 689 rte_log_set_level(eth_ring_logtype, RTE_LOG_NOTICE); 690 } 691