1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <inttypes.h> 34 #include <string.h> 35 36 #include <rte_bus_vdev.h> 37 #include <rte_kvargs.h> 38 #include <rte_ring.h> 39 #include <rte_errno.h> 40 #include <rte_event_ring.h> 41 #include <rte_service_component.h> 42 43 #include "sw_evdev.h" 44 #include "iq_ring.h" 45 46 #define EVENTDEV_NAME_SW_PMD event_sw 47 #define NUMA_NODE_ARG "numa_node" 48 #define SCHED_QUANTA_ARG "sched_quanta" 49 #define CREDIT_QUANTA_ARG "credit_quanta" 50 51 static void 52 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info); 53 54 static int 55 sw_port_link(struct rte_eventdev *dev, void *port, const uint8_t queues[], 56 const uint8_t priorities[], uint16_t num) 57 { 58 struct sw_port *p = port; 59 struct sw_evdev *sw = sw_pmd_priv(dev); 60 int i; 61 62 RTE_SET_USED(priorities); 63 for (i = 0; i < num; i++) { 64 struct sw_qid *q = &sw->qids[queues[i]]; 65 unsigned int j; 66 67 /* check for qid map overflow */ 68 if (q->cq_num_mapped_cqs >= RTE_DIM(q->cq_map)) { 69 rte_errno = -EDQUOT; 70 break; 71 } 72 73 if (p->is_directed && p->num_qids_mapped > 0) { 74 rte_errno = -EDQUOT; 75 break; 76 } 77 78 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 79 if (q->cq_map[j] == p->id) 80 break; 81 } 82 83 /* check if port is already linked */ 84 if (j < q->cq_num_mapped_cqs) 85 continue; 86 87 if (q->type == SW_SCHED_TYPE_DIRECT) { 88 /* check directed qids only map to one port */ 89 if (p->num_qids_mapped > 0) { 90 rte_errno = -EDQUOT; 91 break; 92 } 93 /* check port only takes a directed flow */ 94 if (num > 1) { 95 rte_errno = -EDQUOT; 96 break; 97 } 98 99 p->is_directed = 1; 100 p->num_qids_mapped = 1; 101 } else if (q->type == RTE_SCHED_TYPE_ORDERED) { 102 p->num_ordered_qids++; 103 p->num_qids_mapped++; 104 } else if (q->type == RTE_SCHED_TYPE_ATOMIC || 105 q->type == RTE_SCHED_TYPE_PARALLEL) { 106 p->num_qids_mapped++; 107 } 108 109 q->cq_map[q->cq_num_mapped_cqs] = p->id; 110 rte_smp_wmb(); 111 q->cq_num_mapped_cqs++; 112 } 113 return i; 114 } 115 116 static int 117 sw_port_unlink(struct rte_eventdev *dev, void *port, uint8_t queues[], 118 uint16_t nb_unlinks) 119 { 120 struct sw_port *p = port; 121 struct sw_evdev *sw = sw_pmd_priv(dev); 122 unsigned int i, j; 123 124 int unlinked = 0; 125 for (i = 0; i < nb_unlinks; i++) { 126 struct sw_qid *q = &sw->qids[queues[i]]; 127 for (j = 0; j < q->cq_num_mapped_cqs; j++) { 128 if (q->cq_map[j] == p->id) { 129 q->cq_map[j] = 130 q->cq_map[q->cq_num_mapped_cqs - 1]; 131 rte_smp_wmb(); 132 q->cq_num_mapped_cqs--; 133 unlinked++; 134 135 p->num_qids_mapped--; 136 137 if (q->type == RTE_SCHED_TYPE_ORDERED) 138 p->num_ordered_qids--; 139 140 continue; 141 } 142 } 143 } 144 return unlinked; 145 } 146 147 static int 148 sw_port_setup(struct rte_eventdev *dev, uint8_t port_id, 149 const struct rte_event_port_conf *conf) 150 { 151 struct sw_evdev *sw = sw_pmd_priv(dev); 152 struct sw_port *p = &sw->ports[port_id]; 153 char buf[RTE_RING_NAMESIZE]; 154 unsigned int i; 155 156 struct rte_event_dev_info info; 157 sw_info_get(dev, &info); 158 159 /* detect re-configuring and return credits to instance if needed */ 160 if (p->initialized) { 161 /* taking credits from pool is done one quanta at a time, and 162 * credits may be spend (counted in p->inflights) or still 163 * available in the port (p->inflight_credits). We must return 164 * the sum to no leak credits 165 */ 166 int possible_inflights = p->inflight_credits + p->inflights; 167 rte_atomic32_sub(&sw->inflights, possible_inflights); 168 } 169 170 *p = (struct sw_port){0}; /* zero entire structure */ 171 p->id = port_id; 172 p->sw = sw; 173 174 /* check to see if rings exists - port_setup() can be called multiple 175 * times legally (assuming device is stopped). If ring exists, free it 176 * to so it gets re-created with the correct size 177 */ 178 snprintf(buf, sizeof(buf), "sw%d_p%u_%s", dev->data->dev_id, 179 port_id, "rx_worker_ring"); 180 struct rte_event_ring *existing_ring = rte_event_ring_lookup(buf); 181 if (existing_ring) 182 rte_event_ring_free(existing_ring); 183 184 p->rx_worker_ring = rte_event_ring_create(buf, MAX_SW_PROD_Q_DEPTH, 185 dev->data->socket_id, 186 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 187 if (p->rx_worker_ring == NULL) { 188 SW_LOG_ERR("Error creating RX worker ring for port %d\n", 189 port_id); 190 return -1; 191 } 192 193 p->inflight_max = conf->new_event_threshold; 194 195 /* check if ring exists, same as rx_worker above */ 196 snprintf(buf, sizeof(buf), "sw%d_p%u, %s", dev->data->dev_id, 197 port_id, "cq_worker_ring"); 198 existing_ring = rte_event_ring_lookup(buf); 199 if (existing_ring) 200 rte_event_ring_free(existing_ring); 201 202 p->cq_worker_ring = rte_event_ring_create(buf, conf->dequeue_depth, 203 dev->data->socket_id, 204 RING_F_SP_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ); 205 if (p->cq_worker_ring == NULL) { 206 rte_event_ring_free(p->rx_worker_ring); 207 SW_LOG_ERR("Error creating CQ worker ring for port %d\n", 208 port_id); 209 return -1; 210 } 211 sw->cq_ring_space[port_id] = conf->dequeue_depth; 212 213 /* set hist list contents to empty */ 214 for (i = 0; i < SW_PORT_HIST_LIST; i++) { 215 p->hist_list[i].fid = -1; 216 p->hist_list[i].qid = -1; 217 } 218 dev->data->ports[port_id] = p; 219 220 rte_smp_wmb(); 221 p->initialized = 1; 222 return 0; 223 } 224 225 static void 226 sw_port_release(void *port) 227 { 228 struct sw_port *p = (void *)port; 229 if (p == NULL) 230 return; 231 232 rte_event_ring_free(p->rx_worker_ring); 233 rte_event_ring_free(p->cq_worker_ring); 234 memset(p, 0, sizeof(*p)); 235 } 236 237 static int32_t 238 qid_init(struct sw_evdev *sw, unsigned int idx, int type, 239 const struct rte_event_queue_conf *queue_conf) 240 { 241 unsigned int i; 242 int dev_id = sw->data->dev_id; 243 int socket_id = sw->data->socket_id; 244 char buf[IQ_RING_NAMESIZE]; 245 struct sw_qid *qid = &sw->qids[idx]; 246 247 for (i = 0; i < SW_IQS_MAX; i++) { 248 snprintf(buf, sizeof(buf), "q_%u_iq_%d", idx, i); 249 qid->iq[i] = iq_ring_create(buf, socket_id); 250 if (!qid->iq[i]) { 251 SW_LOG_DBG("ring create failed"); 252 goto cleanup; 253 } 254 } 255 256 /* Initialize the FID structures to no pinning (-1), and zero packets */ 257 const struct sw_fid_t fid = {.cq = -1, .pcount = 0}; 258 for (i = 0; i < RTE_DIM(qid->fids); i++) 259 qid->fids[i] = fid; 260 261 qid->id = idx; 262 qid->type = type; 263 qid->priority = queue_conf->priority; 264 265 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 266 char ring_name[RTE_RING_NAMESIZE]; 267 uint32_t window_size; 268 269 /* rte_ring and window_size_mask require require window_size to 270 * be a power-of-2. 271 */ 272 window_size = rte_align32pow2( 273 queue_conf->nb_atomic_order_sequences); 274 275 qid->window_size = window_size - 1; 276 277 if (!window_size) { 278 SW_LOG_DBG( 279 "invalid reorder_window_size for ordered queue\n" 280 ); 281 goto cleanup; 282 } 283 284 snprintf(buf, sizeof(buf), "sw%d_iq_%d_rob", dev_id, i); 285 qid->reorder_buffer = rte_zmalloc_socket(buf, 286 window_size * sizeof(qid->reorder_buffer[0]), 287 0, socket_id); 288 if (!qid->reorder_buffer) { 289 SW_LOG_DBG("reorder_buffer malloc failed\n"); 290 goto cleanup; 291 } 292 293 memset(&qid->reorder_buffer[0], 294 0, 295 window_size * sizeof(qid->reorder_buffer[0])); 296 297 snprintf(ring_name, sizeof(ring_name), "sw%d_q%d_freelist", 298 dev_id, idx); 299 300 /* lookup the ring, and if it already exists, free it */ 301 struct rte_ring *cleanup = rte_ring_lookup(ring_name); 302 if (cleanup) 303 rte_ring_free(cleanup); 304 305 qid->reorder_buffer_freelist = rte_ring_create(ring_name, 306 window_size, 307 socket_id, 308 RING_F_SP_ENQ | RING_F_SC_DEQ); 309 if (!qid->reorder_buffer_freelist) { 310 SW_LOG_DBG("freelist ring create failed"); 311 goto cleanup; 312 } 313 314 /* Populate the freelist with reorder buffer entries. Enqueue 315 * 'window_size - 1' entries because the rte_ring holds only 316 * that many. 317 */ 318 for (i = 0; i < window_size - 1; i++) { 319 if (rte_ring_sp_enqueue(qid->reorder_buffer_freelist, 320 &qid->reorder_buffer[i]) < 0) 321 goto cleanup; 322 } 323 324 qid->reorder_buffer_index = 0; 325 qid->cq_next_tx = 0; 326 } 327 328 qid->initialized = 1; 329 330 return 0; 331 332 cleanup: 333 for (i = 0; i < SW_IQS_MAX; i++) { 334 if (qid->iq[i]) 335 iq_ring_destroy(qid->iq[i]); 336 } 337 338 if (qid->reorder_buffer) { 339 rte_free(qid->reorder_buffer); 340 qid->reorder_buffer = NULL; 341 } 342 343 if (qid->reorder_buffer_freelist) { 344 rte_ring_free(qid->reorder_buffer_freelist); 345 qid->reorder_buffer_freelist = NULL; 346 } 347 348 return -EINVAL; 349 } 350 351 static void 352 sw_queue_release(struct rte_eventdev *dev, uint8_t id) 353 { 354 struct sw_evdev *sw = sw_pmd_priv(dev); 355 struct sw_qid *qid = &sw->qids[id]; 356 uint32_t i; 357 358 for (i = 0; i < SW_IQS_MAX; i++) 359 iq_ring_destroy(qid->iq[i]); 360 361 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 362 rte_free(qid->reorder_buffer); 363 rte_ring_free(qid->reorder_buffer_freelist); 364 } 365 memset(qid, 0, sizeof(*qid)); 366 } 367 368 static int 369 sw_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 370 const struct rte_event_queue_conf *conf) 371 { 372 int type; 373 374 type = conf->schedule_type; 375 376 if (RTE_EVENT_QUEUE_CFG_SINGLE_LINK & conf->event_queue_cfg) { 377 type = SW_SCHED_TYPE_DIRECT; 378 } else if (RTE_EVENT_QUEUE_CFG_ALL_TYPES 379 & conf->event_queue_cfg) { 380 SW_LOG_ERR("QUEUE_CFG_ALL_TYPES not supported\n"); 381 return -ENOTSUP; 382 } 383 384 struct sw_evdev *sw = sw_pmd_priv(dev); 385 386 if (sw->qids[queue_id].initialized) 387 sw_queue_release(dev, queue_id); 388 389 return qid_init(sw, queue_id, type, conf); 390 } 391 392 static void 393 sw_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 394 struct rte_event_queue_conf *conf) 395 { 396 RTE_SET_USED(dev); 397 RTE_SET_USED(queue_id); 398 399 static const struct rte_event_queue_conf default_conf = { 400 .nb_atomic_flows = 4096, 401 .nb_atomic_order_sequences = 1, 402 .schedule_type = RTE_SCHED_TYPE_ATOMIC, 403 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, 404 }; 405 406 *conf = default_conf; 407 } 408 409 static void 410 sw_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 411 struct rte_event_port_conf *port_conf) 412 { 413 RTE_SET_USED(dev); 414 RTE_SET_USED(port_id); 415 416 port_conf->new_event_threshold = 1024; 417 port_conf->dequeue_depth = 16; 418 port_conf->enqueue_depth = 16; 419 } 420 421 static int 422 sw_dev_configure(const struct rte_eventdev *dev) 423 { 424 struct sw_evdev *sw = sw_pmd_priv(dev); 425 const struct rte_eventdev_data *data = dev->data; 426 const struct rte_event_dev_config *conf = &data->dev_conf; 427 428 sw->qid_count = conf->nb_event_queues; 429 sw->port_count = conf->nb_event_ports; 430 sw->nb_events_limit = conf->nb_events_limit; 431 rte_atomic32_set(&sw->inflights, 0); 432 433 if (conf->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) 434 return -ENOTSUP; 435 436 return 0; 437 } 438 439 struct rte_eth_dev; 440 441 static int 442 sw_eth_rx_adapter_caps_get(const struct rte_eventdev *dev, 443 const struct rte_eth_dev *eth_dev, 444 uint32_t *caps) 445 { 446 RTE_SET_USED(dev); 447 RTE_SET_USED(eth_dev); 448 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 449 return 0; 450 } 451 452 static void 453 sw_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *info) 454 { 455 RTE_SET_USED(dev); 456 457 static const struct rte_event_dev_info evdev_sw_info = { 458 .driver_name = SW_PMD_NAME, 459 .max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV, 460 .max_event_queue_flows = SW_QID_NUM_FIDS, 461 .max_event_queue_priority_levels = SW_Q_PRIORITY_MAX, 462 .max_event_priority_levels = SW_IQS_MAX, 463 .max_event_ports = SW_PORTS_MAX, 464 .max_event_port_dequeue_depth = MAX_SW_CONS_Q_DEPTH, 465 .max_event_port_enqueue_depth = MAX_SW_PROD_Q_DEPTH, 466 .max_num_events = SW_INFLIGHT_EVENTS_TOTAL, 467 .event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS | 468 RTE_EVENT_DEV_CAP_BURST_MODE | 469 RTE_EVENT_DEV_CAP_EVENT_QOS), 470 }; 471 472 *info = evdev_sw_info; 473 } 474 475 static void 476 sw_dump(struct rte_eventdev *dev, FILE *f) 477 { 478 const struct sw_evdev *sw = sw_pmd_priv(dev); 479 480 static const char * const q_type_strings[] = { 481 "Ordered", "Atomic", "Parallel", "Directed" 482 }; 483 uint32_t i; 484 fprintf(f, "EventDev %s: ports %d, qids %d\n", "todo-fix-name", 485 sw->port_count, sw->qid_count); 486 487 fprintf(f, "\trx %"PRIu64"\n\tdrop %"PRIu64"\n\ttx %"PRIu64"\n", 488 sw->stats.rx_pkts, sw->stats.rx_dropped, sw->stats.tx_pkts); 489 fprintf(f, "\tsched calls: %"PRIu64"\n", sw->sched_called); 490 fprintf(f, "\tsched cq/qid call: %"PRIu64"\n", sw->sched_cq_qid_called); 491 fprintf(f, "\tsched no IQ enq: %"PRIu64"\n", sw->sched_no_iq_enqueues); 492 fprintf(f, "\tsched no CQ enq: %"PRIu64"\n", sw->sched_no_cq_enqueues); 493 uint32_t inflights = rte_atomic32_read(&sw->inflights); 494 uint32_t credits = sw->nb_events_limit - inflights; 495 fprintf(f, "\tinflight %d, credits: %d\n", inflights, credits); 496 497 #define COL_RED "\x1b[31m" 498 #define COL_RESET "\x1b[0m" 499 500 for (i = 0; i < sw->port_count; i++) { 501 int max, j; 502 const struct sw_port *p = &sw->ports[i]; 503 if (!p->initialized) { 504 fprintf(f, " %sPort %d not initialized.%s\n", 505 COL_RED, i, COL_RESET); 506 continue; 507 } 508 fprintf(f, " Port %d %s\n", i, 509 p->is_directed ? " (SingleCons)" : ""); 510 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64 511 "\t%sinflight %d%s\n", sw->ports[i].stats.rx_pkts, 512 sw->ports[i].stats.rx_dropped, 513 sw->ports[i].stats.tx_pkts, 514 (p->inflights == p->inflight_max) ? 515 COL_RED : COL_RESET, 516 sw->ports[i].inflights, COL_RESET); 517 518 fprintf(f, "\tMax New: %u" 519 "\tAvg cycles PP: %"PRIu64"\tCredits: %u\n", 520 sw->ports[i].inflight_max, 521 sw->ports[i].avg_pkt_ticks, 522 sw->ports[i].inflight_credits); 523 fprintf(f, "\tReceive burst distribution:\n"); 524 float zp_percent = p->zero_polls * 100.0 / p->total_polls; 525 fprintf(f, zp_percent < 10 ? "\t\t0:%.02f%% " : "\t\t0:%.0f%% ", 526 zp_percent); 527 for (max = (int)RTE_DIM(p->poll_buckets); max-- > 0;) 528 if (p->poll_buckets[max] != 0) 529 break; 530 for (j = 0; j <= max; j++) { 531 if (p->poll_buckets[j] != 0) { 532 float poll_pc = p->poll_buckets[j] * 100.0 / 533 p->total_polls; 534 fprintf(f, "%u-%u:%.02f%% ", 535 ((j << SW_DEQ_STAT_BUCKET_SHIFT) + 1), 536 ((j+1) << SW_DEQ_STAT_BUCKET_SHIFT), 537 poll_pc); 538 } 539 } 540 fprintf(f, "\n"); 541 542 if (p->rx_worker_ring) { 543 uint64_t used = rte_event_ring_count(p->rx_worker_ring); 544 uint64_t space = rte_event_ring_free_count( 545 p->rx_worker_ring); 546 const char *col = (space == 0) ? COL_RED : COL_RESET; 547 fprintf(f, "\t%srx ring used: %4"PRIu64"\tfree: %4" 548 PRIu64 COL_RESET"\n", col, used, space); 549 } else 550 fprintf(f, "\trx ring not initialized.\n"); 551 552 if (p->cq_worker_ring) { 553 uint64_t used = rte_event_ring_count(p->cq_worker_ring); 554 uint64_t space = rte_event_ring_free_count( 555 p->cq_worker_ring); 556 const char *col = (space == 0) ? COL_RED : COL_RESET; 557 fprintf(f, "\t%scq ring used: %4"PRIu64"\tfree: %4" 558 PRIu64 COL_RESET"\n", col, used, space); 559 } else 560 fprintf(f, "\tcq ring not initialized.\n"); 561 } 562 563 for (i = 0; i < sw->qid_count; i++) { 564 const struct sw_qid *qid = &sw->qids[i]; 565 if (!qid->initialized) { 566 fprintf(f, " %sQueue %d not initialized.%s\n", 567 COL_RED, i, COL_RESET); 568 continue; 569 } 570 int affinities_per_port[SW_PORTS_MAX] = {0}; 571 uint32_t inflights = 0; 572 573 fprintf(f, " Queue %d (%s)\n", i, q_type_strings[qid->type]); 574 fprintf(f, "\trx %"PRIu64"\tdrop %"PRIu64"\ttx %"PRIu64"\n", 575 qid->stats.rx_pkts, qid->stats.rx_dropped, 576 qid->stats.tx_pkts); 577 if (qid->type == RTE_SCHED_TYPE_ORDERED) { 578 struct rte_ring *rob_buf_free = 579 qid->reorder_buffer_freelist; 580 if (rob_buf_free) 581 fprintf(f, "\tReorder entries in use: %u\n", 582 rte_ring_free_count(rob_buf_free)); 583 else 584 fprintf(f, 585 "\tReorder buffer not initialized\n"); 586 } 587 588 uint32_t flow; 589 for (flow = 0; flow < RTE_DIM(qid->fids); flow++) 590 if (qid->fids[flow].cq != -1) { 591 affinities_per_port[qid->fids[flow].cq]++; 592 inflights += qid->fids[flow].pcount; 593 } 594 595 uint32_t port; 596 fprintf(f, "\tPer Port Stats:\n"); 597 for (port = 0; port < sw->port_count; port++) { 598 fprintf(f, "\t Port %d: Pkts: %"PRIu64, port, 599 qid->to_port[port]); 600 fprintf(f, "\tFlows: %d\n", affinities_per_port[port]); 601 } 602 603 uint32_t iq; 604 uint32_t iq_printed = 0; 605 for (iq = 0; iq < SW_IQS_MAX; iq++) { 606 if (!qid->iq[iq]) { 607 fprintf(f, "\tiq %d is not initialized.\n", iq); 608 iq_printed = 1; 609 continue; 610 } 611 uint32_t used = iq_ring_count(qid->iq[iq]); 612 uint32_t free = iq_ring_free_count(qid->iq[iq]); 613 const char *col = (free == 0) ? COL_RED : COL_RESET; 614 if (used > 0) { 615 fprintf(f, "\t%siq %d: Used %d\tFree %d" 616 COL_RESET"\n", col, iq, used, free); 617 iq_printed = 1; 618 } 619 } 620 if (iq_printed == 0) 621 fprintf(f, "\t-- iqs empty --\n"); 622 } 623 } 624 625 static int 626 sw_start(struct rte_eventdev *dev) 627 { 628 unsigned int i, j; 629 struct sw_evdev *sw = sw_pmd_priv(dev); 630 631 rte_service_component_runstate_set(sw->service_id, 1); 632 633 /* check a service core is mapped to this service */ 634 if (!rte_service_runstate_get(sw->service_id)) { 635 SW_LOG_ERR("Warning: No Service core enabled on service %s\n", 636 sw->service_name); 637 return -ENOENT; 638 } 639 640 /* check all ports are set up */ 641 for (i = 0; i < sw->port_count; i++) 642 if (sw->ports[i].rx_worker_ring == NULL) { 643 SW_LOG_ERR("Port %d not configured\n", i); 644 return -ESTALE; 645 } 646 647 /* check all queues are configured and mapped to ports*/ 648 for (i = 0; i < sw->qid_count; i++) 649 if (sw->qids[i].iq[0] == NULL || 650 sw->qids[i].cq_num_mapped_cqs == 0) { 651 SW_LOG_ERR("Queue %d not configured\n", i); 652 return -ENOLINK; 653 } 654 655 /* build up our prioritized array of qids */ 656 /* We don't use qsort here, as if all/multiple entries have the same 657 * priority, the result is non-deterministic. From "man 3 qsort": 658 * "If two members compare as equal, their order in the sorted 659 * array is undefined." 660 */ 661 uint32_t qidx = 0; 662 for (j = 0; j <= RTE_EVENT_DEV_PRIORITY_LOWEST; j++) { 663 for (i = 0; i < sw->qid_count; i++) { 664 if (sw->qids[i].priority == j) { 665 sw->qids_prioritized[qidx] = &sw->qids[i]; 666 qidx++; 667 } 668 } 669 } 670 671 if (sw_xstats_init(sw) < 0) 672 return -EINVAL; 673 674 rte_smp_wmb(); 675 sw->started = 1; 676 677 return 0; 678 } 679 680 static void 681 sw_stop(struct rte_eventdev *dev) 682 { 683 struct sw_evdev *sw = sw_pmd_priv(dev); 684 sw_xstats_uninit(sw); 685 sw->started = 0; 686 rte_smp_wmb(); 687 } 688 689 static int 690 sw_close(struct rte_eventdev *dev) 691 { 692 struct sw_evdev *sw = sw_pmd_priv(dev); 693 uint32_t i; 694 695 for (i = 0; i < sw->qid_count; i++) 696 sw_queue_release(dev, i); 697 sw->qid_count = 0; 698 699 for (i = 0; i < sw->port_count; i++) 700 sw_port_release(&sw->ports[i]); 701 sw->port_count = 0; 702 703 memset(&sw->stats, 0, sizeof(sw->stats)); 704 sw->sched_called = 0; 705 sw->sched_no_iq_enqueues = 0; 706 sw->sched_no_cq_enqueues = 0; 707 sw->sched_cq_qid_called = 0; 708 709 return 0; 710 } 711 712 static int 713 assign_numa_node(const char *key __rte_unused, const char *value, void *opaque) 714 { 715 int *socket_id = opaque; 716 *socket_id = atoi(value); 717 if (*socket_id >= RTE_MAX_NUMA_NODES) 718 return -1; 719 return 0; 720 } 721 722 static int 723 set_sched_quanta(const char *key __rte_unused, const char *value, void *opaque) 724 { 725 int *quanta = opaque; 726 *quanta = atoi(value); 727 if (*quanta < 0 || *quanta >= 4096) 728 return -1; 729 return 0; 730 } 731 732 static int 733 set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque) 734 { 735 int *credit = opaque; 736 *credit = atoi(value); 737 if (*credit < 0 || *credit >= 128) 738 return -1; 739 return 0; 740 } 741 742 743 static int32_t sw_sched_service_func(void *args) 744 { 745 struct rte_eventdev *dev = args; 746 sw_event_schedule(dev); 747 return 0; 748 } 749 750 static int 751 sw_probe(struct rte_vdev_device *vdev) 752 { 753 static const struct rte_eventdev_ops evdev_sw_ops = { 754 .dev_configure = sw_dev_configure, 755 .dev_infos_get = sw_info_get, 756 .dev_close = sw_close, 757 .dev_start = sw_start, 758 .dev_stop = sw_stop, 759 .dump = sw_dump, 760 761 .queue_def_conf = sw_queue_def_conf, 762 .queue_setup = sw_queue_setup, 763 .queue_release = sw_queue_release, 764 .port_def_conf = sw_port_def_conf, 765 .port_setup = sw_port_setup, 766 .port_release = sw_port_release, 767 .port_link = sw_port_link, 768 .port_unlink = sw_port_unlink, 769 770 .eth_rx_adapter_caps_get = sw_eth_rx_adapter_caps_get, 771 772 .xstats_get = sw_xstats_get, 773 .xstats_get_names = sw_xstats_get_names, 774 .xstats_get_by_name = sw_xstats_get_by_name, 775 .xstats_reset = sw_xstats_reset, 776 }; 777 778 static const char *const args[] = { 779 NUMA_NODE_ARG, 780 SCHED_QUANTA_ARG, 781 CREDIT_QUANTA_ARG, 782 NULL 783 }; 784 const char *name; 785 const char *params; 786 struct rte_eventdev *dev; 787 struct sw_evdev *sw; 788 int socket_id = rte_socket_id(); 789 int sched_quanta = SW_DEFAULT_SCHED_QUANTA; 790 int credit_quanta = SW_DEFAULT_CREDIT_QUANTA; 791 792 name = rte_vdev_device_name(vdev); 793 params = rte_vdev_device_args(vdev); 794 if (params != NULL && params[0] != '\0') { 795 struct rte_kvargs *kvlist = rte_kvargs_parse(params, args); 796 797 if (!kvlist) { 798 SW_LOG_INFO( 799 "Ignoring unsupported parameters when creating device '%s'\n", 800 name); 801 } else { 802 int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG, 803 assign_numa_node, &socket_id); 804 if (ret != 0) { 805 SW_LOG_ERR( 806 "%s: Error parsing numa node parameter", 807 name); 808 rte_kvargs_free(kvlist); 809 return ret; 810 } 811 812 ret = rte_kvargs_process(kvlist, SCHED_QUANTA_ARG, 813 set_sched_quanta, &sched_quanta); 814 if (ret != 0) { 815 SW_LOG_ERR( 816 "%s: Error parsing sched quanta parameter", 817 name); 818 rte_kvargs_free(kvlist); 819 return ret; 820 } 821 822 ret = rte_kvargs_process(kvlist, CREDIT_QUANTA_ARG, 823 set_credit_quanta, &credit_quanta); 824 if (ret != 0) { 825 SW_LOG_ERR( 826 "%s: Error parsing credit quanta parameter", 827 name); 828 rte_kvargs_free(kvlist); 829 return ret; 830 } 831 832 rte_kvargs_free(kvlist); 833 } 834 } 835 836 SW_LOG_INFO( 837 "Creating eventdev sw device %s, numa_node=%d, sched_quanta=%d, credit_quanta=%d\n", 838 name, socket_id, sched_quanta, credit_quanta); 839 840 dev = rte_event_pmd_vdev_init(name, 841 sizeof(struct sw_evdev), socket_id); 842 if (dev == NULL) { 843 SW_LOG_ERR("eventdev vdev init() failed"); 844 return -EFAULT; 845 } 846 dev->dev_ops = &evdev_sw_ops; 847 dev->enqueue = sw_event_enqueue; 848 dev->enqueue_burst = sw_event_enqueue_burst; 849 dev->enqueue_new_burst = sw_event_enqueue_burst; 850 dev->enqueue_forward_burst = sw_event_enqueue_burst; 851 dev->dequeue = sw_event_dequeue; 852 dev->dequeue_burst = sw_event_dequeue_burst; 853 854 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 855 return 0; 856 857 sw = dev->data->dev_private; 858 sw->data = dev->data; 859 860 /* copy values passed from vdev command line to instance */ 861 sw->credit_update_quanta = credit_quanta; 862 sw->sched_quanta = sched_quanta; 863 864 /* register service with EAL */ 865 struct rte_service_spec service; 866 memset(&service, 0, sizeof(struct rte_service_spec)); 867 snprintf(service.name, sizeof(service.name), "%s_service", name); 868 snprintf(sw->service_name, sizeof(sw->service_name), "%s_service", 869 name); 870 service.socket_id = socket_id; 871 service.callback = sw_sched_service_func; 872 service.callback_userdata = (void *)dev; 873 874 int32_t ret = rte_service_component_register(&service, &sw->service_id); 875 if (ret) { 876 SW_LOG_ERR("service register() failed"); 877 return -ENOEXEC; 878 } 879 880 dev->data->service_inited = 1; 881 dev->data->service_id = sw->service_id; 882 883 return 0; 884 } 885 886 static int 887 sw_remove(struct rte_vdev_device *vdev) 888 { 889 const char *name; 890 891 name = rte_vdev_device_name(vdev); 892 if (name == NULL) 893 return -EINVAL; 894 895 SW_LOG_INFO("Closing eventdev sw device %s\n", name); 896 897 return rte_event_pmd_vdev_uninit(name); 898 } 899 900 static struct rte_vdev_driver evdev_sw_pmd_drv = { 901 .probe = sw_probe, 902 .remove = sw_remove 903 }; 904 905 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_SW_PMD, evdev_sw_pmd_drv); 906 RTE_PMD_REGISTER_PARAM_STRING(event_sw, NUMA_NODE_ARG "=<int> " 907 SCHED_QUANTA_ARG "=<int>" CREDIT_QUANTA_ARG "=<int>"); 908