1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017,2019 NXP 3 */ 4 5 #include <assert.h> 6 #include <stdio.h> 7 #include <stdbool.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <sys/epoll.h> 12 13 #include <rte_atomic.h> 14 #include <rte_byteorder.h> 15 #include <rte_common.h> 16 #include <rte_debug.h> 17 #include <rte_dev.h> 18 #include <rte_eal.h> 19 #include <rte_fslmc.h> 20 #include <rte_lcore.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memcpy.h> 24 #include <rte_memory.h> 25 #include <rte_pci.h> 26 #include <rte_bus_vdev.h> 27 #include <rte_ethdev_driver.h> 28 #include <rte_cryptodev.h> 29 #include <rte_event_eth_rx_adapter.h> 30 #include <rte_event_eth_tx_adapter.h> 31 32 #include <fslmc_vfio.h> 33 #include <dpaa2_hw_pvt.h> 34 #include <dpaa2_hw_mempool.h> 35 #include <dpaa2_hw_dpio.h> 36 #include <dpaa2_ethdev.h> 37 #include <dpaa2_sec_event.h> 38 #include "dpaa2_eventdev.h" 39 #include "dpaa2_eventdev_logs.h" 40 #include <portal/dpaa2_hw_pvt.h> 41 #include <mc/fsl_dpci.h> 42 43 /* Clarifications 44 * Evendev = SoC Instance 45 * Eventport = DPIO Instance 46 * Eventqueue = DPCON Instance 47 * 1 Eventdev can have N Eventqueue 48 * Soft Event Flow is DPCI Instance 49 */ 50 51 #define DPAA2_EV_TX_RETRY_COUNT 10000 52 53 static uint16_t 54 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], 55 uint16_t nb_events) 56 { 57 58 struct dpaa2_port *dpaa2_portal = port; 59 struct dpaa2_dpio_dev *dpio_dev; 60 uint32_t queue_id = ev[0].queue_id; 61 struct dpaa2_eventq *evq_info; 62 uint32_t fqid, retry_count; 63 struct qbman_swp *swp; 64 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 65 uint32_t loop, frames_to_send; 66 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 67 uint16_t num_tx = 0; 68 int i, n, ret; 69 uint8_t channel_index; 70 71 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 72 /* Affine current thread context to a qman portal */ 73 ret = dpaa2_affine_qbman_swp(); 74 if (ret < 0) { 75 DPAA2_EVENTDEV_ERR( 76 "Failed to allocate IO portal, tid: %d\n", 77 rte_gettid()); 78 return 0; 79 } 80 } 81 /* todo - dpaa2_portal shall have dpio_dev - no per thread variable */ 82 dpio_dev = DPAA2_PER_LCORE_DPIO; 83 swp = DPAA2_PER_LCORE_PORTAL; 84 85 if (likely(dpaa2_portal->is_port_linked)) 86 goto skip_linking; 87 88 /* Create mapping between portal and channel to receive packets */ 89 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 90 evq_info = &dpaa2_portal->evq_info[i]; 91 if (!evq_info->event_port) 92 continue; 93 94 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 95 CMD_PRI_LOW, 96 dpio_dev->token, 97 evq_info->dpcon->dpcon_id, 98 &channel_index); 99 if (ret < 0) { 100 DPAA2_EVENTDEV_ERR( 101 "Static dequeue config failed: err(%d)", ret); 102 goto err; 103 } 104 105 qbman_swp_push_set(swp, channel_index, 1); 106 evq_info->dpcon->channel_index = channel_index; 107 } 108 dpaa2_portal->is_port_linked = true; 109 110 skip_linking: 111 evq_info = &dpaa2_portal->evq_info[queue_id]; 112 113 while (nb_events) { 114 frames_to_send = (nb_events > dpaa2_eqcr_size) ? 115 dpaa2_eqcr_size : nb_events; 116 117 for (loop = 0; loop < frames_to_send; loop++) { 118 const struct rte_event *event = &ev[num_tx + loop]; 119 120 if (event->sched_type != RTE_SCHED_TYPE_ATOMIC) 121 fqid = evq_info->dpci->rx_queue[ 122 DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid; 123 else 124 fqid = evq_info->dpci->rx_queue[ 125 DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid; 126 127 /* Prepare enqueue descriptor */ 128 qbman_eq_desc_clear(&eqdesc[loop]); 129 qbman_eq_desc_set_fq(&eqdesc[loop], fqid); 130 qbman_eq_desc_set_no_orp(&eqdesc[loop], 0); 131 qbman_eq_desc_set_response(&eqdesc[loop], 0, 0); 132 133 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC 134 && event->mbuf->seqn) { 135 uint8_t dqrr_index = event->mbuf->seqn - 1; 136 137 qbman_eq_desc_set_dca(&eqdesc[loop], 1, 138 dqrr_index, 0); 139 DPAA2_PER_LCORE_DQRR_SIZE--; 140 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 141 } 142 143 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 144 145 /* 146 * todo - need to align with hw context data 147 * to avoid copy 148 */ 149 struct rte_event *ev_temp = rte_malloc(NULL, 150 sizeof(struct rte_event), 0); 151 152 if (!ev_temp) { 153 if (!loop) 154 return num_tx; 155 frames_to_send = loop; 156 DPAA2_EVENTDEV_ERR( 157 "Unable to allocate event object"); 158 goto send_partial; 159 } 160 rte_memcpy(ev_temp, event, sizeof(struct rte_event)); 161 DPAA2_SET_FD_ADDR((&fd_arr[loop]), (size_t)ev_temp); 162 DPAA2_SET_FD_LEN((&fd_arr[loop]), 163 sizeof(struct rte_event)); 164 } 165 send_partial: 166 loop = 0; 167 retry_count = 0; 168 while (loop < frames_to_send) { 169 ret = qbman_swp_enqueue_multiple_desc(swp, 170 &eqdesc[loop], &fd_arr[loop], 171 frames_to_send - loop); 172 if (unlikely(ret < 0)) { 173 retry_count++; 174 if (retry_count > DPAA2_EV_TX_RETRY_COUNT) { 175 num_tx += loop; 176 nb_events -= loop; 177 return num_tx + loop; 178 } 179 } else { 180 loop += ret; 181 retry_count = 0; 182 } 183 } 184 num_tx += loop; 185 nb_events -= loop; 186 } 187 188 return num_tx; 189 err: 190 for (n = 0; n < i; n++) { 191 evq_info = &dpaa2_portal->evq_info[n]; 192 if (!evq_info->event_port) 193 continue; 194 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 195 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 196 dpio_dev->token, 197 evq_info->dpcon->dpcon_id); 198 } 199 return 0; 200 201 } 202 203 static uint16_t 204 dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev) 205 { 206 return dpaa2_eventdev_enqueue_burst(port, ev, 1); 207 } 208 209 static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks) 210 { 211 struct epoll_event epoll_ev; 212 213 qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL, 214 QBMAN_SWP_INTERRUPT_DQRI); 215 216 epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd, 217 &epoll_ev, 1, timeout_ticks); 218 } 219 220 static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp, 221 const struct qbman_fd *fd, 222 const struct qbman_result *dq, 223 struct dpaa2_queue *rxq, 224 struct rte_event *ev) 225 { 226 struct rte_event *ev_temp = 227 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 228 229 RTE_SET_USED(rxq); 230 231 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 232 rte_free(ev_temp); 233 234 qbman_swp_dqrr_consume(swp, dq); 235 } 236 237 static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp, 238 const struct qbman_fd *fd, 239 const struct qbman_result *dq, 240 struct dpaa2_queue *rxq, 241 struct rte_event *ev) 242 { 243 struct rte_event *ev_temp = 244 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 245 uint8_t dqrr_index = qbman_get_dqrr_idx(dq); 246 247 RTE_SET_USED(swp); 248 RTE_SET_USED(rxq); 249 250 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 251 rte_free(ev_temp); 252 ev->mbuf->seqn = dqrr_index + 1; 253 DPAA2_PER_LCORE_DQRR_SIZE++; 254 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 255 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 256 } 257 258 static uint16_t 259 dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[], 260 uint16_t nb_events, uint64_t timeout_ticks) 261 { 262 const struct qbman_result *dq; 263 struct dpaa2_dpio_dev *dpio_dev = NULL; 264 struct dpaa2_port *dpaa2_portal = port; 265 struct dpaa2_eventq *evq_info; 266 struct qbman_swp *swp; 267 const struct qbman_fd *fd; 268 struct dpaa2_queue *rxq; 269 int num_pkts = 0, ret, i = 0, n; 270 uint8_t channel_index; 271 272 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 273 /* Affine current thread context to a qman portal */ 274 ret = dpaa2_affine_qbman_swp(); 275 if (ret < 0) { 276 DPAA2_EVENTDEV_ERR( 277 "Failed to allocate IO portal, tid: %d\n", 278 rte_gettid()); 279 return 0; 280 } 281 } 282 283 dpio_dev = DPAA2_PER_LCORE_DPIO; 284 swp = DPAA2_PER_LCORE_PORTAL; 285 286 if (likely(dpaa2_portal->is_port_linked)) 287 goto skip_linking; 288 289 /* Create mapping between portal and channel to receive packets */ 290 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 291 evq_info = &dpaa2_portal->evq_info[i]; 292 if (!evq_info->event_port) 293 continue; 294 295 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 296 CMD_PRI_LOW, 297 dpio_dev->token, 298 evq_info->dpcon->dpcon_id, 299 &channel_index); 300 if (ret < 0) { 301 DPAA2_EVENTDEV_ERR( 302 "Static dequeue config failed: err(%d)", ret); 303 goto err; 304 } 305 306 qbman_swp_push_set(swp, channel_index, 1); 307 evq_info->dpcon->channel_index = channel_index; 308 } 309 dpaa2_portal->is_port_linked = true; 310 311 skip_linking: 312 /* Check if there are atomic contexts to be released */ 313 while (DPAA2_PER_LCORE_DQRR_SIZE) { 314 if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) { 315 qbman_swp_dqrr_idx_consume(swp, i); 316 DPAA2_PER_LCORE_DQRR_SIZE--; 317 DPAA2_PER_LCORE_DQRR_MBUF(i)->seqn = 318 DPAA2_INVALID_MBUF_SEQN; 319 } 320 i++; 321 } 322 DPAA2_PER_LCORE_DQRR_HELD = 0; 323 324 do { 325 dq = qbman_swp_dqrr_next(swp); 326 if (!dq) { 327 if (!num_pkts && timeout_ticks) { 328 dpaa2_eventdev_dequeue_wait(timeout_ticks); 329 timeout_ticks = 0; 330 continue; 331 } 332 return num_pkts; 333 } 334 qbman_swp_prefetch_dqrr_next(swp); 335 336 fd = qbman_result_DQ_fd(dq); 337 rxq = (struct dpaa2_queue *)(size_t)qbman_result_DQ_fqd_ctx(dq); 338 if (rxq) { 339 rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]); 340 } else { 341 qbman_swp_dqrr_consume(swp, dq); 342 DPAA2_EVENTDEV_ERR("Null Return VQ received"); 343 return 0; 344 } 345 346 num_pkts++; 347 } while (num_pkts < nb_events); 348 349 return num_pkts; 350 err: 351 for (n = 0; n < i; n++) { 352 evq_info = &dpaa2_portal->evq_info[n]; 353 if (!evq_info->event_port) 354 continue; 355 356 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 357 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 358 dpio_dev->token, 359 evq_info->dpcon->dpcon_id); 360 } 361 return 0; 362 } 363 364 static uint16_t 365 dpaa2_eventdev_dequeue(void *port, struct rte_event *ev, 366 uint64_t timeout_ticks) 367 { 368 return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks); 369 } 370 371 static void 372 dpaa2_eventdev_info_get(struct rte_eventdev *dev, 373 struct rte_event_dev_info *dev_info) 374 { 375 struct dpaa2_eventdev *priv = dev->data->dev_private; 376 377 EVENTDEV_INIT_FUNC_TRACE(); 378 379 RTE_SET_USED(dev); 380 381 memset(dev_info, 0, sizeof(struct rte_event_dev_info)); 382 dev_info->min_dequeue_timeout_ns = 383 DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; 384 dev_info->max_dequeue_timeout_ns = 385 DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT; 386 dev_info->dequeue_timeout_ns = 387 DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 388 dev_info->max_event_queues = priv->max_event_queues; 389 dev_info->max_event_queue_flows = 390 DPAA2_EVENT_MAX_QUEUE_FLOWS; 391 dev_info->max_event_queue_priority_levels = 392 DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS; 393 dev_info->max_event_priority_levels = 394 DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS; 395 dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO); 396 /* we only support dpio up to number of cores */ 397 if (dev_info->max_event_ports > rte_lcore_count()) 398 dev_info->max_event_ports = rte_lcore_count(); 399 dev_info->max_event_port_dequeue_depth = 400 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 401 dev_info->max_event_port_enqueue_depth = 402 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 403 dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS; 404 dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | 405 RTE_EVENT_DEV_CAP_BURST_MODE| 406 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK | 407 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT | 408 RTE_EVENT_DEV_CAP_NONSEQ_MODE; 409 410 } 411 412 static int 413 dpaa2_eventdev_configure(const struct rte_eventdev *dev) 414 { 415 struct dpaa2_eventdev *priv = dev->data->dev_private; 416 struct rte_event_dev_config *conf = &dev->data->dev_conf; 417 418 EVENTDEV_INIT_FUNC_TRACE(); 419 420 priv->nb_event_queues = conf->nb_event_queues; 421 priv->nb_event_ports = conf->nb_event_ports; 422 priv->nb_event_queue_flows = conf->nb_event_queue_flows; 423 priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth; 424 priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth; 425 priv->event_dev_cfg = conf->event_dev_cfg; 426 427 /* Check dequeue timeout method is per dequeue or global */ 428 if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { 429 /* 430 * Use timeout value as given in dequeue operation. 431 * So invalidating this timeout value. 432 */ 433 priv->dequeue_timeout_ns = 0; 434 435 } else if (conf->dequeue_timeout_ns == 0) { 436 priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 437 } else { 438 priv->dequeue_timeout_ns = conf->dequeue_timeout_ns; 439 } 440 441 DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d", 442 dev->data->dev_id); 443 return 0; 444 } 445 446 static int 447 dpaa2_eventdev_start(struct rte_eventdev *dev) 448 { 449 EVENTDEV_INIT_FUNC_TRACE(); 450 451 RTE_SET_USED(dev); 452 453 return 0; 454 } 455 456 static void 457 dpaa2_eventdev_stop(struct rte_eventdev *dev) 458 { 459 EVENTDEV_INIT_FUNC_TRACE(); 460 461 RTE_SET_USED(dev); 462 } 463 464 static int 465 dpaa2_eventdev_close(struct rte_eventdev *dev) 466 { 467 EVENTDEV_INIT_FUNC_TRACE(); 468 469 RTE_SET_USED(dev); 470 471 return 0; 472 } 473 474 static void 475 dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 476 struct rte_event_queue_conf *queue_conf) 477 { 478 EVENTDEV_INIT_FUNC_TRACE(); 479 480 RTE_SET_USED(dev); 481 RTE_SET_USED(queue_id); 482 483 queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS; 484 queue_conf->nb_atomic_order_sequences = 485 DPAA2_EVENT_QUEUE_ORDER_SEQUENCES; 486 queue_conf->schedule_type = RTE_SCHED_TYPE_PARALLEL; 487 queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 488 } 489 490 static int 491 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 492 const struct rte_event_queue_conf *queue_conf) 493 { 494 struct dpaa2_eventdev *priv = dev->data->dev_private; 495 struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id]; 496 497 EVENTDEV_INIT_FUNC_TRACE(); 498 499 switch (queue_conf->schedule_type) { 500 case RTE_SCHED_TYPE_PARALLEL: 501 case RTE_SCHED_TYPE_ATOMIC: 502 case RTE_SCHED_TYPE_ORDERED: 503 break; 504 default: 505 DPAA2_EVENTDEV_ERR("Schedule type is not supported."); 506 return -1; 507 } 508 evq_info->event_queue_cfg = queue_conf->event_queue_cfg; 509 evq_info->event_queue_id = queue_id; 510 511 return 0; 512 } 513 514 static void 515 dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) 516 { 517 EVENTDEV_INIT_FUNC_TRACE(); 518 519 RTE_SET_USED(dev); 520 RTE_SET_USED(queue_id); 521 } 522 523 static void 524 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 525 struct rte_event_port_conf *port_conf) 526 { 527 EVENTDEV_INIT_FUNC_TRACE(); 528 529 RTE_SET_USED(dev); 530 RTE_SET_USED(port_id); 531 532 port_conf->new_event_threshold = 533 DPAA2_EVENT_MAX_NUM_EVENTS; 534 port_conf->dequeue_depth = 535 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 536 port_conf->enqueue_depth = 537 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 538 port_conf->disable_implicit_release = 0; 539 } 540 541 static int 542 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, 543 const struct rte_event_port_conf *port_conf) 544 { 545 char event_port_name[32]; 546 struct dpaa2_port *portal; 547 548 EVENTDEV_INIT_FUNC_TRACE(); 549 550 RTE_SET_USED(port_conf); 551 552 sprintf(event_port_name, "event-port-%d", port_id); 553 portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0); 554 if (!portal) { 555 DPAA2_EVENTDEV_ERR("Memory allocation failure"); 556 return -ENOMEM; 557 } 558 559 memset(portal, 0, sizeof(struct dpaa2_port)); 560 dev->data->ports[port_id] = portal; 561 return 0; 562 } 563 564 static void 565 dpaa2_eventdev_port_release(void *port) 566 { 567 struct dpaa2_port *portal = port; 568 569 EVENTDEV_INIT_FUNC_TRACE(); 570 571 /* TODO: Cleanup is required when ports are in linked state. */ 572 if (portal->is_port_linked) 573 DPAA2_EVENTDEV_WARN("Event port must be unlinked before release"); 574 575 if (portal) 576 rte_free(portal); 577 578 portal = NULL; 579 } 580 581 static int 582 dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, 583 const uint8_t queues[], const uint8_t priorities[], 584 uint16_t nb_links) 585 { 586 struct dpaa2_eventdev *priv = dev->data->dev_private; 587 struct dpaa2_port *dpaa2_portal = port; 588 struct dpaa2_eventq *evq_info; 589 uint16_t i; 590 591 EVENTDEV_INIT_FUNC_TRACE(); 592 593 RTE_SET_USED(priorities); 594 595 for (i = 0; i < nb_links; i++) { 596 evq_info = &priv->evq_info[queues[i]]; 597 memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info, 598 sizeof(struct dpaa2_eventq)); 599 dpaa2_portal->evq_info[queues[i]].event_port = port; 600 dpaa2_portal->num_linked_evq++; 601 } 602 603 return (int)nb_links; 604 } 605 606 static int 607 dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, 608 uint8_t queues[], uint16_t nb_unlinks) 609 { 610 struct dpaa2_port *dpaa2_portal = port; 611 int i; 612 struct dpaa2_dpio_dev *dpio_dev = NULL; 613 struct dpaa2_eventq *evq_info; 614 struct qbman_swp *swp; 615 616 EVENTDEV_INIT_FUNC_TRACE(); 617 618 RTE_SET_USED(dev); 619 RTE_SET_USED(queues); 620 621 for (i = 0; i < nb_unlinks; i++) { 622 evq_info = &dpaa2_portal->evq_info[queues[i]]; 623 624 if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) { 625 /* todo dpaa2_portal shall have dpio_dev-no per lcore*/ 626 dpio_dev = DPAA2_PER_LCORE_DPIO; 627 swp = DPAA2_PER_LCORE_PORTAL; 628 629 qbman_swp_push_set(swp, 630 evq_info->dpcon->channel_index, 0); 631 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 632 dpio_dev->token, 633 evq_info->dpcon->dpcon_id); 634 } 635 memset(evq_info, 0, sizeof(struct dpaa2_eventq)); 636 if (dpaa2_portal->num_linked_evq) 637 dpaa2_portal->num_linked_evq--; 638 } 639 640 if (!dpaa2_portal->num_linked_evq) 641 dpaa2_portal->is_port_linked = false; 642 643 return (int)nb_unlinks; 644 } 645 646 647 static int 648 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, 649 uint64_t *timeout_ticks) 650 { 651 uint32_t scale = 1000*1000; 652 653 EVENTDEV_INIT_FUNC_TRACE(); 654 655 RTE_SET_USED(dev); 656 *timeout_ticks = ns / scale; 657 658 return 0; 659 } 660 661 static void 662 dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f) 663 { 664 EVENTDEV_INIT_FUNC_TRACE(); 665 666 RTE_SET_USED(dev); 667 RTE_SET_USED(f); 668 } 669 670 static int 671 dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev, 672 const struct rte_eth_dev *eth_dev, 673 uint32_t *caps) 674 { 675 const char *ethdev_driver = eth_dev->device->driver->name; 676 677 EVENTDEV_INIT_FUNC_TRACE(); 678 679 RTE_SET_USED(dev); 680 681 if (!strcmp(ethdev_driver, "net_dpaa2")) 682 *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP; 683 else 684 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 685 686 return 0; 687 } 688 689 static int 690 dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev, 691 const struct rte_eth_dev *eth_dev, 692 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 693 { 694 struct dpaa2_eventdev *priv = dev->data->dev_private; 695 uint8_t ev_qid = queue_conf->ev.queue_id; 696 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 697 int i, ret; 698 699 EVENTDEV_INIT_FUNC_TRACE(); 700 701 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 702 ret = dpaa2_eth_eventq_attach(eth_dev, i, 703 dpcon, queue_conf); 704 if (ret) { 705 DPAA2_EVENTDEV_ERR( 706 "Event queue attach failed: err(%d)", ret); 707 goto fail; 708 } 709 } 710 return 0; 711 fail: 712 for (i = (i - 1); i >= 0 ; i--) 713 dpaa2_eth_eventq_detach(eth_dev, i); 714 715 return ret; 716 } 717 718 static int 719 dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev, 720 const struct rte_eth_dev *eth_dev, 721 int32_t rx_queue_id, 722 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 723 { 724 struct dpaa2_eventdev *priv = dev->data->dev_private; 725 uint8_t ev_qid = queue_conf->ev.queue_id; 726 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 727 int ret; 728 729 EVENTDEV_INIT_FUNC_TRACE(); 730 731 if (rx_queue_id == -1) 732 return dpaa2_eventdev_eth_queue_add_all(dev, 733 eth_dev, queue_conf); 734 735 ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id, 736 dpcon, queue_conf); 737 if (ret) { 738 DPAA2_EVENTDEV_ERR( 739 "Event queue attach failed: err(%d)", ret); 740 return ret; 741 } 742 return 0; 743 } 744 745 static int 746 dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev, 747 const struct rte_eth_dev *eth_dev) 748 { 749 int i, ret; 750 751 EVENTDEV_INIT_FUNC_TRACE(); 752 753 RTE_SET_USED(dev); 754 755 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 756 ret = dpaa2_eth_eventq_detach(eth_dev, i); 757 if (ret) { 758 DPAA2_EVENTDEV_ERR( 759 "Event queue detach failed: err(%d)", ret); 760 return ret; 761 } 762 } 763 764 return 0; 765 } 766 767 static int 768 dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev, 769 const struct rte_eth_dev *eth_dev, 770 int32_t rx_queue_id) 771 { 772 int ret; 773 774 EVENTDEV_INIT_FUNC_TRACE(); 775 776 if (rx_queue_id == -1) 777 return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev); 778 779 ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id); 780 if (ret) { 781 DPAA2_EVENTDEV_ERR( 782 "Event queue detach failed: err(%d)", ret); 783 return ret; 784 } 785 786 return 0; 787 } 788 789 static int 790 dpaa2_eventdev_eth_start(const struct rte_eventdev *dev, 791 const struct rte_eth_dev *eth_dev) 792 { 793 EVENTDEV_INIT_FUNC_TRACE(); 794 795 RTE_SET_USED(dev); 796 RTE_SET_USED(eth_dev); 797 798 return 0; 799 } 800 801 static int 802 dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev, 803 const struct rte_eth_dev *eth_dev) 804 { 805 EVENTDEV_INIT_FUNC_TRACE(); 806 807 RTE_SET_USED(dev); 808 RTE_SET_USED(eth_dev); 809 810 return 0; 811 } 812 813 static int 814 dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev, 815 const struct rte_cryptodev *cdev, 816 uint32_t *caps) 817 { 818 const char *name = cdev->data->name; 819 820 EVENTDEV_INIT_FUNC_TRACE(); 821 822 RTE_SET_USED(dev); 823 824 if (!strncmp(name, "dpsec-", 6)) 825 *caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP; 826 else 827 return -1; 828 829 return 0; 830 } 831 832 static int 833 dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev, 834 const struct rte_cryptodev *cryptodev, 835 const struct rte_event *ev) 836 { 837 struct dpaa2_eventdev *priv = dev->data->dev_private; 838 uint8_t ev_qid = ev->queue_id; 839 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 840 int i, ret; 841 842 EVENTDEV_INIT_FUNC_TRACE(); 843 844 for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) { 845 ret = dpaa2_sec_eventq_attach(cryptodev, i, dpcon, ev); 846 if (ret) { 847 DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n", 848 ret); 849 goto fail; 850 } 851 } 852 return 0; 853 fail: 854 for (i = (i - 1); i >= 0 ; i--) 855 dpaa2_sec_eventq_detach(cryptodev, i); 856 857 return ret; 858 } 859 860 static int 861 dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev, 862 const struct rte_cryptodev *cryptodev, 863 int32_t rx_queue_id, 864 const struct rte_event *ev) 865 { 866 struct dpaa2_eventdev *priv = dev->data->dev_private; 867 uint8_t ev_qid = ev->queue_id; 868 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 869 int ret; 870 871 EVENTDEV_INIT_FUNC_TRACE(); 872 873 if (rx_queue_id == -1) 874 return dpaa2_eventdev_crypto_queue_add_all(dev, 875 cryptodev, ev); 876 877 ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id, 878 dpcon, ev); 879 if (ret) { 880 DPAA2_EVENTDEV_ERR( 881 "dpaa2_sec_eventq_attach failed: ret: %d\n", ret); 882 return ret; 883 } 884 return 0; 885 } 886 887 static int 888 dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev, 889 const struct rte_cryptodev *cdev) 890 { 891 int i, ret; 892 893 EVENTDEV_INIT_FUNC_TRACE(); 894 895 RTE_SET_USED(dev); 896 897 for (i = 0; i < cdev->data->nb_queue_pairs; i++) { 898 ret = dpaa2_sec_eventq_detach(cdev, i); 899 if (ret) { 900 DPAA2_EVENTDEV_ERR( 901 "dpaa2_sec_eventq_detach failed:ret %d\n", ret); 902 return ret; 903 } 904 } 905 906 return 0; 907 } 908 909 static int 910 dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev, 911 const struct rte_cryptodev *cryptodev, 912 int32_t rx_queue_id) 913 { 914 int ret; 915 916 EVENTDEV_INIT_FUNC_TRACE(); 917 918 if (rx_queue_id == -1) 919 return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev); 920 921 ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id); 922 if (ret) { 923 DPAA2_EVENTDEV_ERR( 924 "dpaa2_sec_eventq_detach failed: ret: %d\n", ret); 925 return ret; 926 } 927 928 return 0; 929 } 930 931 static int 932 dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev, 933 const struct rte_cryptodev *cryptodev) 934 { 935 EVENTDEV_INIT_FUNC_TRACE(); 936 937 RTE_SET_USED(dev); 938 RTE_SET_USED(cryptodev); 939 940 return 0; 941 } 942 943 static int 944 dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev, 945 const struct rte_cryptodev *cryptodev) 946 { 947 EVENTDEV_INIT_FUNC_TRACE(); 948 949 RTE_SET_USED(dev); 950 RTE_SET_USED(cryptodev); 951 952 return 0; 953 } 954 955 static int 956 dpaa2_eventdev_tx_adapter_create(uint8_t id, 957 const struct rte_eventdev *dev) 958 { 959 RTE_SET_USED(id); 960 RTE_SET_USED(dev); 961 962 /* Nothing to do. Simply return. */ 963 return 0; 964 } 965 966 static int 967 dpaa2_eventdev_tx_adapter_caps(const struct rte_eventdev *dev, 968 const struct rte_eth_dev *eth_dev, 969 uint32_t *caps) 970 { 971 RTE_SET_USED(dev); 972 RTE_SET_USED(eth_dev); 973 974 *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT; 975 return 0; 976 } 977 978 static uint16_t 979 dpaa2_eventdev_txa_enqueue_same_dest(void *port, 980 struct rte_event ev[], 981 uint16_t nb_events) 982 { 983 struct rte_mbuf *m[DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH], *m0; 984 uint8_t qid, i; 985 986 RTE_SET_USED(port); 987 988 m0 = (struct rte_mbuf *)ev[0].mbuf; 989 qid = rte_event_eth_tx_adapter_txq_get(m0); 990 991 for (i = 0; i < nb_events; i++) 992 m[i] = (struct rte_mbuf *)ev[i].mbuf; 993 994 return rte_eth_tx_burst(m0->port, qid, m, nb_events); 995 } 996 997 static uint16_t 998 dpaa2_eventdev_txa_enqueue(void *port, 999 struct rte_event ev[], 1000 uint16_t nb_events) 1001 { 1002 struct rte_mbuf *m = (struct rte_mbuf *)ev[0].mbuf; 1003 uint8_t qid, i; 1004 1005 RTE_SET_USED(port); 1006 1007 for (i = 0; i < nb_events; i++) { 1008 qid = rte_event_eth_tx_adapter_txq_get(m); 1009 rte_eth_tx_burst(m->port, qid, &m, 1); 1010 } 1011 1012 return nb_events; 1013 } 1014 1015 static struct rte_eventdev_ops dpaa2_eventdev_ops = { 1016 .dev_infos_get = dpaa2_eventdev_info_get, 1017 .dev_configure = dpaa2_eventdev_configure, 1018 .dev_start = dpaa2_eventdev_start, 1019 .dev_stop = dpaa2_eventdev_stop, 1020 .dev_close = dpaa2_eventdev_close, 1021 .queue_def_conf = dpaa2_eventdev_queue_def_conf, 1022 .queue_setup = dpaa2_eventdev_queue_setup, 1023 .queue_release = dpaa2_eventdev_queue_release, 1024 .port_def_conf = dpaa2_eventdev_port_def_conf, 1025 .port_setup = dpaa2_eventdev_port_setup, 1026 .port_release = dpaa2_eventdev_port_release, 1027 .port_link = dpaa2_eventdev_port_link, 1028 .port_unlink = dpaa2_eventdev_port_unlink, 1029 .timeout_ticks = dpaa2_eventdev_timeout_ticks, 1030 .dump = dpaa2_eventdev_dump, 1031 .dev_selftest = test_eventdev_dpaa2, 1032 .eth_rx_adapter_caps_get = dpaa2_eventdev_eth_caps_get, 1033 .eth_rx_adapter_queue_add = dpaa2_eventdev_eth_queue_add, 1034 .eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del, 1035 .eth_rx_adapter_start = dpaa2_eventdev_eth_start, 1036 .eth_rx_adapter_stop = dpaa2_eventdev_eth_stop, 1037 .eth_tx_adapter_caps_get = dpaa2_eventdev_tx_adapter_caps, 1038 .eth_tx_adapter_create = dpaa2_eventdev_tx_adapter_create, 1039 .crypto_adapter_caps_get = dpaa2_eventdev_crypto_caps_get, 1040 .crypto_adapter_queue_pair_add = dpaa2_eventdev_crypto_queue_add, 1041 .crypto_adapter_queue_pair_del = dpaa2_eventdev_crypto_queue_del, 1042 .crypto_adapter_start = dpaa2_eventdev_crypto_start, 1043 .crypto_adapter_stop = dpaa2_eventdev_crypto_stop, 1044 }; 1045 1046 static int 1047 dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, 1048 struct dpaa2_dpcon_dev *dpcon_dev) 1049 { 1050 struct dpci_rx_queue_cfg rx_queue_cfg; 1051 int ret, i; 1052 1053 /*Do settings to get the frame on a DPCON object*/ 1054 rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST | 1055 DPCI_QUEUE_OPT_USER_CTX; 1056 rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON; 1057 rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id; 1058 rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO; 1059 1060 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb = 1061 dpaa2_eventdev_process_parallel; 1062 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb = 1063 dpaa2_eventdev_process_atomic; 1064 1065 for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) { 1066 rx_queue_cfg.user_ctx = (size_t)(&dpci_dev->rx_queue[i]); 1067 ret = dpci_set_rx_queue(&dpci_dev->dpci, 1068 CMD_PRI_LOW, 1069 dpci_dev->token, i, 1070 &rx_queue_cfg); 1071 if (ret) { 1072 DPAA2_EVENTDEV_ERR( 1073 "DPCI Rx queue setup failed: err(%d)", 1074 ret); 1075 return ret; 1076 } 1077 } 1078 return 0; 1079 } 1080 1081 static int 1082 dpaa2_eventdev_create(const char *name) 1083 { 1084 struct rte_eventdev *eventdev; 1085 struct dpaa2_eventdev *priv; 1086 struct dpaa2_dpcon_dev *dpcon_dev = NULL; 1087 struct dpaa2_dpci_dev *dpci_dev = NULL; 1088 int ret; 1089 1090 eventdev = rte_event_pmd_vdev_init(name, 1091 sizeof(struct dpaa2_eventdev), 1092 rte_socket_id()); 1093 if (eventdev == NULL) { 1094 DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name); 1095 goto fail; 1096 } 1097 1098 eventdev->dev_ops = &dpaa2_eventdev_ops; 1099 eventdev->enqueue = dpaa2_eventdev_enqueue; 1100 eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst; 1101 eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst; 1102 eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst; 1103 eventdev->dequeue = dpaa2_eventdev_dequeue; 1104 eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst; 1105 eventdev->txa_enqueue = dpaa2_eventdev_txa_enqueue; 1106 eventdev->txa_enqueue_same_dest = dpaa2_eventdev_txa_enqueue_same_dest; 1107 1108 /* For secondary processes, the primary has done all the work */ 1109 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1110 return 0; 1111 1112 priv = eventdev->data->dev_private; 1113 priv->max_event_queues = 0; 1114 1115 do { 1116 dpcon_dev = rte_dpaa2_alloc_dpcon_dev(); 1117 if (!dpcon_dev) 1118 break; 1119 priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev; 1120 1121 dpci_dev = rte_dpaa2_alloc_dpci_dev(); 1122 if (!dpci_dev) { 1123 rte_dpaa2_free_dpcon_dev(dpcon_dev); 1124 break; 1125 } 1126 priv->evq_info[priv->max_event_queues].dpci = dpci_dev; 1127 1128 ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev); 1129 if (ret) { 1130 DPAA2_EVENTDEV_ERR( 1131 "DPCI setup failed: err(%d)", ret); 1132 return ret; 1133 } 1134 priv->max_event_queues++; 1135 } while (dpcon_dev && dpci_dev); 1136 1137 RTE_LOG(INFO, PMD, "%s eventdev created\n", name); 1138 1139 return 0; 1140 fail: 1141 return -EFAULT; 1142 } 1143 1144 static int 1145 dpaa2_eventdev_destroy(const char *name) 1146 { 1147 struct rte_eventdev *eventdev; 1148 struct dpaa2_eventdev *priv; 1149 int i; 1150 1151 eventdev = rte_event_pmd_get_named_dev(name); 1152 if (eventdev == NULL) { 1153 RTE_EDEV_LOG_ERR("eventdev with name %s not allocated", name); 1154 return -1; 1155 } 1156 1157 /* For secondary processes, the primary has done all the work */ 1158 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1159 return 0; 1160 1161 priv = eventdev->data->dev_private; 1162 for (i = 0; i < priv->max_event_queues; i++) { 1163 if (priv->evq_info[i].dpcon) 1164 rte_dpaa2_free_dpcon_dev(priv->evq_info[i].dpcon); 1165 1166 if (priv->evq_info[i].dpci) 1167 rte_dpaa2_free_dpci_dev(priv->evq_info[i].dpci); 1168 1169 } 1170 priv->max_event_queues = 0; 1171 1172 RTE_LOG(INFO, PMD, "%s eventdev cleaned\n", name); 1173 return 0; 1174 } 1175 1176 1177 static int 1178 dpaa2_eventdev_probe(struct rte_vdev_device *vdev) 1179 { 1180 const char *name; 1181 1182 name = rte_vdev_device_name(vdev); 1183 DPAA2_EVENTDEV_INFO("Initializing %s", name); 1184 return dpaa2_eventdev_create(name); 1185 } 1186 1187 static int 1188 dpaa2_eventdev_remove(struct rte_vdev_device *vdev) 1189 { 1190 const char *name; 1191 1192 name = rte_vdev_device_name(vdev); 1193 DPAA2_EVENTDEV_INFO("Closing %s", name); 1194 1195 dpaa2_eventdev_destroy(name); 1196 1197 return rte_event_pmd_vdev_uninit(name); 1198 } 1199 1200 static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = { 1201 .probe = dpaa2_eventdev_probe, 1202 .remove = dpaa2_eventdev_remove 1203 }; 1204 1205 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd); 1206 RTE_LOG_REGISTER(dpaa2_logtype_event, pmd.event.dpaa2, NOTICE); 1207