1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017,2019 NXP 3 */ 4 5 #include <assert.h> 6 #include <stdio.h> 7 #include <stdbool.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <sys/epoll.h> 12 13 #include <rte_atomic.h> 14 #include <rte_byteorder.h> 15 #include <rte_common.h> 16 #include <rte_debug.h> 17 #include <rte_dev.h> 18 #include <rte_eal.h> 19 #include <rte_fslmc.h> 20 #include <rte_lcore.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memcpy.h> 24 #include <rte_memory.h> 25 #include <rte_pci.h> 26 #include <rte_bus_vdev.h> 27 #include <rte_ethdev_driver.h> 28 #include <rte_cryptodev.h> 29 #include <rte_event_eth_rx_adapter.h> 30 31 #include <fslmc_vfio.h> 32 #include <dpaa2_hw_pvt.h> 33 #include <dpaa2_hw_mempool.h> 34 #include <dpaa2_hw_dpio.h> 35 #include <dpaa2_ethdev.h> 36 #include <dpaa2_sec_event.h> 37 #include "dpaa2_eventdev.h" 38 #include "dpaa2_eventdev_logs.h" 39 #include <portal/dpaa2_hw_pvt.h> 40 #include <mc/fsl_dpci.h> 41 42 /* Clarifications 43 * Evendev = SoC Instance 44 * Eventport = DPIO Instance 45 * Eventqueue = DPCON Instance 46 * 1 Eventdev can have N Eventqueue 47 * Soft Event Flow is DPCI Instance 48 */ 49 50 /* Dynamic logging identified for mempool */ 51 int dpaa2_logtype_event; 52 #define DPAA2_EV_TX_RETRY_COUNT 10000 53 54 static uint16_t 55 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], 56 uint16_t nb_events) 57 { 58 59 struct dpaa2_port *dpaa2_portal = port; 60 struct dpaa2_dpio_dev *dpio_dev; 61 uint32_t queue_id = ev[0].queue_id; 62 struct dpaa2_eventq *evq_info; 63 uint32_t fqid, retry_count; 64 struct qbman_swp *swp; 65 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 66 uint32_t loop, frames_to_send; 67 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 68 uint16_t num_tx = 0; 69 int i, n, ret; 70 uint8_t channel_index; 71 72 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 73 /* Affine current thread context to a qman portal */ 74 ret = dpaa2_affine_qbman_swp(); 75 if (ret < 0) { 76 DPAA2_EVENTDEV_ERR("Failure in affining portal"); 77 return 0; 78 } 79 } 80 /* todo - dpaa2_portal shall have dpio_dev - no per thread variable */ 81 dpio_dev = DPAA2_PER_LCORE_DPIO; 82 swp = DPAA2_PER_LCORE_PORTAL; 83 84 if (likely(dpaa2_portal->is_port_linked)) 85 goto skip_linking; 86 87 /* Create mapping between portal and channel to receive packets */ 88 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 89 evq_info = &dpaa2_portal->evq_info[i]; 90 if (!evq_info->event_port) 91 continue; 92 93 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 94 CMD_PRI_LOW, 95 dpio_dev->token, 96 evq_info->dpcon->dpcon_id, 97 &channel_index); 98 if (ret < 0) { 99 DPAA2_EVENTDEV_ERR( 100 "Static dequeue config failed: err(%d)", ret); 101 goto err; 102 } 103 104 qbman_swp_push_set(swp, channel_index, 1); 105 evq_info->dpcon->channel_index = channel_index; 106 } 107 dpaa2_portal->is_port_linked = true; 108 109 skip_linking: 110 evq_info = &dpaa2_portal->evq_info[queue_id]; 111 112 while (nb_events) { 113 frames_to_send = (nb_events > dpaa2_eqcr_size) ? 114 dpaa2_eqcr_size : nb_events; 115 116 for (loop = 0; loop < frames_to_send; loop++) { 117 const struct rte_event *event = &ev[num_tx + loop]; 118 119 if (event->sched_type != RTE_SCHED_TYPE_ATOMIC) 120 fqid = evq_info->dpci->rx_queue[ 121 DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid; 122 else 123 fqid = evq_info->dpci->rx_queue[ 124 DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid; 125 126 /* Prepare enqueue descriptor */ 127 qbman_eq_desc_clear(&eqdesc[loop]); 128 qbman_eq_desc_set_fq(&eqdesc[loop], fqid); 129 qbman_eq_desc_set_no_orp(&eqdesc[loop], 0); 130 qbman_eq_desc_set_response(&eqdesc[loop], 0, 0); 131 132 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC 133 && event->mbuf->seqn) { 134 uint8_t dqrr_index = event->mbuf->seqn - 1; 135 136 qbman_eq_desc_set_dca(&eqdesc[loop], 1, 137 dqrr_index, 0); 138 DPAA2_PER_LCORE_DQRR_SIZE--; 139 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 140 } 141 142 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 143 144 /* 145 * todo - need to align with hw context data 146 * to avoid copy 147 */ 148 struct rte_event *ev_temp = rte_malloc(NULL, 149 sizeof(struct rte_event), 0); 150 151 if (!ev_temp) { 152 if (!loop) 153 return num_tx; 154 frames_to_send = loop; 155 DPAA2_EVENTDEV_ERR( 156 "Unable to allocate event object"); 157 goto send_partial; 158 } 159 rte_memcpy(ev_temp, event, sizeof(struct rte_event)); 160 DPAA2_SET_FD_ADDR((&fd_arr[loop]), (size_t)ev_temp); 161 DPAA2_SET_FD_LEN((&fd_arr[loop]), 162 sizeof(struct rte_event)); 163 } 164 send_partial: 165 loop = 0; 166 retry_count = 0; 167 while (loop < frames_to_send) { 168 ret = qbman_swp_enqueue_multiple_desc(swp, 169 &eqdesc[loop], &fd_arr[loop], 170 frames_to_send - loop); 171 if (unlikely(ret < 0)) { 172 retry_count++; 173 if (retry_count > DPAA2_EV_TX_RETRY_COUNT) { 174 num_tx += loop; 175 nb_events -= loop; 176 return num_tx + loop; 177 } 178 } else { 179 loop += ret; 180 retry_count = 0; 181 } 182 } 183 num_tx += loop; 184 nb_events -= loop; 185 } 186 187 return num_tx; 188 err: 189 for (n = 0; n < i; n++) { 190 evq_info = &dpaa2_portal->evq_info[n]; 191 if (!evq_info->event_port) 192 continue; 193 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 194 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 195 dpio_dev->token, 196 evq_info->dpcon->dpcon_id); 197 } 198 return 0; 199 200 } 201 202 static uint16_t 203 dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev) 204 { 205 return dpaa2_eventdev_enqueue_burst(port, ev, 1); 206 } 207 208 static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks) 209 { 210 struct epoll_event epoll_ev; 211 212 qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL, 213 QBMAN_SWP_INTERRUPT_DQRI); 214 215 epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd, 216 &epoll_ev, 1, timeout_ticks); 217 } 218 219 static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp, 220 const struct qbman_fd *fd, 221 const struct qbman_result *dq, 222 struct dpaa2_queue *rxq, 223 struct rte_event *ev) 224 { 225 struct rte_event *ev_temp = 226 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 227 228 RTE_SET_USED(rxq); 229 230 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 231 rte_free(ev_temp); 232 233 qbman_swp_dqrr_consume(swp, dq); 234 } 235 236 static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp, 237 const struct qbman_fd *fd, 238 const struct qbman_result *dq, 239 struct dpaa2_queue *rxq, 240 struct rte_event *ev) 241 { 242 struct rte_event *ev_temp = 243 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 244 uint8_t dqrr_index = qbman_get_dqrr_idx(dq); 245 246 RTE_SET_USED(swp); 247 RTE_SET_USED(rxq); 248 249 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 250 rte_free(ev_temp); 251 ev->mbuf->seqn = dqrr_index + 1; 252 DPAA2_PER_LCORE_DQRR_SIZE++; 253 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 254 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 255 } 256 257 static uint16_t 258 dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[], 259 uint16_t nb_events, uint64_t timeout_ticks) 260 { 261 const struct qbman_result *dq; 262 struct dpaa2_dpio_dev *dpio_dev = NULL; 263 struct dpaa2_port *dpaa2_portal = port; 264 struct dpaa2_eventq *evq_info; 265 struct qbman_swp *swp; 266 const struct qbman_fd *fd; 267 struct dpaa2_queue *rxq; 268 int num_pkts = 0, ret, i = 0, n; 269 uint8_t channel_index; 270 271 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 272 /* Affine current thread context to a qman portal */ 273 ret = dpaa2_affine_qbman_swp(); 274 if (ret < 0) { 275 DPAA2_EVENTDEV_ERR("Failure in affining portal"); 276 return 0; 277 } 278 } 279 280 dpio_dev = DPAA2_PER_LCORE_DPIO; 281 swp = DPAA2_PER_LCORE_PORTAL; 282 283 if (likely(dpaa2_portal->is_port_linked)) 284 goto skip_linking; 285 286 /* Create mapping between portal and channel to receive packets */ 287 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 288 evq_info = &dpaa2_portal->evq_info[i]; 289 if (!evq_info->event_port) 290 continue; 291 292 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 293 CMD_PRI_LOW, 294 dpio_dev->token, 295 evq_info->dpcon->dpcon_id, 296 &channel_index); 297 if (ret < 0) { 298 DPAA2_EVENTDEV_ERR( 299 "Static dequeue config failed: err(%d)", ret); 300 goto err; 301 } 302 303 qbman_swp_push_set(swp, channel_index, 1); 304 evq_info->dpcon->channel_index = channel_index; 305 } 306 dpaa2_portal->is_port_linked = true; 307 308 skip_linking: 309 /* Check if there are atomic contexts to be released */ 310 while (DPAA2_PER_LCORE_DQRR_SIZE) { 311 if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) { 312 qbman_swp_dqrr_idx_consume(swp, i); 313 DPAA2_PER_LCORE_DQRR_SIZE--; 314 DPAA2_PER_LCORE_DQRR_MBUF(i)->seqn = 315 DPAA2_INVALID_MBUF_SEQN; 316 } 317 i++; 318 } 319 DPAA2_PER_LCORE_DQRR_HELD = 0; 320 321 do { 322 dq = qbman_swp_dqrr_next(swp); 323 if (!dq) { 324 if (!num_pkts && timeout_ticks) { 325 dpaa2_eventdev_dequeue_wait(timeout_ticks); 326 timeout_ticks = 0; 327 continue; 328 } 329 return num_pkts; 330 } 331 qbman_swp_prefetch_dqrr_next(swp); 332 333 fd = qbman_result_DQ_fd(dq); 334 rxq = (struct dpaa2_queue *)(size_t)qbman_result_DQ_fqd_ctx(dq); 335 if (rxq) { 336 rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]); 337 } else { 338 qbman_swp_dqrr_consume(swp, dq); 339 DPAA2_EVENTDEV_ERR("Null Return VQ received"); 340 return 0; 341 } 342 343 num_pkts++; 344 } while (num_pkts < nb_events); 345 346 return num_pkts; 347 err: 348 for (n = 0; n < i; n++) { 349 evq_info = &dpaa2_portal->evq_info[n]; 350 if (!evq_info->event_port) 351 continue; 352 353 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 354 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 355 dpio_dev->token, 356 evq_info->dpcon->dpcon_id); 357 } 358 return 0; 359 } 360 361 static uint16_t 362 dpaa2_eventdev_dequeue(void *port, struct rte_event *ev, 363 uint64_t timeout_ticks) 364 { 365 return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks); 366 } 367 368 static void 369 dpaa2_eventdev_info_get(struct rte_eventdev *dev, 370 struct rte_event_dev_info *dev_info) 371 { 372 struct dpaa2_eventdev *priv = dev->data->dev_private; 373 374 EVENTDEV_INIT_FUNC_TRACE(); 375 376 RTE_SET_USED(dev); 377 378 memset(dev_info, 0, sizeof(struct rte_event_dev_info)); 379 dev_info->min_dequeue_timeout_ns = 380 DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; 381 dev_info->max_dequeue_timeout_ns = 382 DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT; 383 dev_info->dequeue_timeout_ns = 384 DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 385 dev_info->max_event_queues = priv->max_event_queues; 386 dev_info->max_event_queue_flows = 387 DPAA2_EVENT_MAX_QUEUE_FLOWS; 388 dev_info->max_event_queue_priority_levels = 389 DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS; 390 dev_info->max_event_priority_levels = 391 DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS; 392 dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO); 393 /* we only support dpio upto number of cores*/ 394 if (dev_info->max_event_ports > rte_lcore_count()) 395 dev_info->max_event_ports = rte_lcore_count(); 396 dev_info->max_event_port_dequeue_depth = 397 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 398 dev_info->max_event_port_enqueue_depth = 399 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 400 dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS; 401 dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | 402 RTE_EVENT_DEV_CAP_BURST_MODE| 403 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK | 404 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT | 405 RTE_EVENT_DEV_CAP_NONSEQ_MODE; 406 407 } 408 409 static int 410 dpaa2_eventdev_configure(const struct rte_eventdev *dev) 411 { 412 struct dpaa2_eventdev *priv = dev->data->dev_private; 413 struct rte_event_dev_config *conf = &dev->data->dev_conf; 414 415 EVENTDEV_INIT_FUNC_TRACE(); 416 417 priv->nb_event_queues = conf->nb_event_queues; 418 priv->nb_event_ports = conf->nb_event_ports; 419 priv->nb_event_queue_flows = conf->nb_event_queue_flows; 420 priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth; 421 priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth; 422 priv->event_dev_cfg = conf->event_dev_cfg; 423 424 /* Check dequeue timeout method is per dequeue or global */ 425 if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { 426 /* 427 * Use timeout value as given in dequeue operation. 428 * So invalidating this timeout value. 429 */ 430 priv->dequeue_timeout_ns = 0; 431 432 } else if (conf->dequeue_timeout_ns == 0) { 433 priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 434 } else { 435 priv->dequeue_timeout_ns = conf->dequeue_timeout_ns; 436 } 437 438 DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d", 439 dev->data->dev_id); 440 return 0; 441 } 442 443 static int 444 dpaa2_eventdev_start(struct rte_eventdev *dev) 445 { 446 EVENTDEV_INIT_FUNC_TRACE(); 447 448 RTE_SET_USED(dev); 449 450 return 0; 451 } 452 453 static void 454 dpaa2_eventdev_stop(struct rte_eventdev *dev) 455 { 456 EVENTDEV_INIT_FUNC_TRACE(); 457 458 RTE_SET_USED(dev); 459 } 460 461 static int 462 dpaa2_eventdev_close(struct rte_eventdev *dev) 463 { 464 EVENTDEV_INIT_FUNC_TRACE(); 465 466 RTE_SET_USED(dev); 467 468 return 0; 469 } 470 471 static void 472 dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 473 struct rte_event_queue_conf *queue_conf) 474 { 475 EVENTDEV_INIT_FUNC_TRACE(); 476 477 RTE_SET_USED(dev); 478 RTE_SET_USED(queue_id); 479 RTE_SET_USED(queue_conf); 480 481 queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS; 482 queue_conf->schedule_type = RTE_SCHED_TYPE_PARALLEL; 483 queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 484 } 485 486 static int 487 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 488 const struct rte_event_queue_conf *queue_conf) 489 { 490 struct dpaa2_eventdev *priv = dev->data->dev_private; 491 struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id]; 492 493 EVENTDEV_INIT_FUNC_TRACE(); 494 495 switch (queue_conf->schedule_type) { 496 case RTE_SCHED_TYPE_PARALLEL: 497 case RTE_SCHED_TYPE_ATOMIC: 498 break; 499 case RTE_SCHED_TYPE_ORDERED: 500 DPAA2_EVENTDEV_ERR("Schedule type is not supported."); 501 return -1; 502 } 503 evq_info->event_queue_cfg = queue_conf->event_queue_cfg; 504 evq_info->event_queue_id = queue_id; 505 506 return 0; 507 } 508 509 static void 510 dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) 511 { 512 EVENTDEV_INIT_FUNC_TRACE(); 513 514 RTE_SET_USED(dev); 515 RTE_SET_USED(queue_id); 516 } 517 518 static void 519 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 520 struct rte_event_port_conf *port_conf) 521 { 522 EVENTDEV_INIT_FUNC_TRACE(); 523 524 RTE_SET_USED(dev); 525 RTE_SET_USED(port_id); 526 527 port_conf->new_event_threshold = 528 DPAA2_EVENT_MAX_NUM_EVENTS; 529 port_conf->dequeue_depth = 530 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 531 port_conf->enqueue_depth = 532 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 533 port_conf->disable_implicit_release = 0; 534 } 535 536 static int 537 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, 538 const struct rte_event_port_conf *port_conf) 539 { 540 char event_port_name[32]; 541 struct dpaa2_port *portal; 542 543 EVENTDEV_INIT_FUNC_TRACE(); 544 545 RTE_SET_USED(port_conf); 546 547 sprintf(event_port_name, "event-port-%d", port_id); 548 portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0); 549 if (!portal) { 550 DPAA2_EVENTDEV_ERR("Memory allocation failure"); 551 return -ENOMEM; 552 } 553 554 memset(portal, 0, sizeof(struct dpaa2_port)); 555 dev->data->ports[port_id] = portal; 556 return 0; 557 } 558 559 static void 560 dpaa2_eventdev_port_release(void *port) 561 { 562 struct dpaa2_port *portal = port; 563 564 EVENTDEV_INIT_FUNC_TRACE(); 565 566 /* TODO: Cleanup is required when ports are in linked state. */ 567 if (portal->is_port_linked) 568 DPAA2_EVENTDEV_WARN("Event port must be unlinked before release"); 569 570 if (portal) 571 rte_free(portal); 572 573 portal = NULL; 574 } 575 576 static int 577 dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, 578 const uint8_t queues[], const uint8_t priorities[], 579 uint16_t nb_links) 580 { 581 struct dpaa2_eventdev *priv = dev->data->dev_private; 582 struct dpaa2_port *dpaa2_portal = port; 583 struct dpaa2_eventq *evq_info; 584 uint16_t i; 585 586 EVENTDEV_INIT_FUNC_TRACE(); 587 588 RTE_SET_USED(priorities); 589 590 for (i = 0; i < nb_links; i++) { 591 evq_info = &priv->evq_info[queues[i]]; 592 memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info, 593 sizeof(struct dpaa2_eventq)); 594 dpaa2_portal->evq_info[queues[i]].event_port = port; 595 dpaa2_portal->num_linked_evq++; 596 } 597 598 return (int)nb_links; 599 } 600 601 static int 602 dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, 603 uint8_t queues[], uint16_t nb_unlinks) 604 { 605 struct dpaa2_port *dpaa2_portal = port; 606 int i; 607 struct dpaa2_dpio_dev *dpio_dev = NULL; 608 struct dpaa2_eventq *evq_info; 609 struct qbman_swp *swp; 610 611 EVENTDEV_INIT_FUNC_TRACE(); 612 613 RTE_SET_USED(dev); 614 RTE_SET_USED(queues); 615 616 for (i = 0; i < nb_unlinks; i++) { 617 evq_info = &dpaa2_portal->evq_info[queues[i]]; 618 619 if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) { 620 /* todo dpaa2_portal shall have dpio_dev-no per lcore*/ 621 dpio_dev = DPAA2_PER_LCORE_DPIO; 622 swp = DPAA2_PER_LCORE_PORTAL; 623 624 qbman_swp_push_set(swp, 625 evq_info->dpcon->channel_index, 0); 626 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 627 dpio_dev->token, 628 evq_info->dpcon->dpcon_id); 629 } 630 memset(evq_info, 0, sizeof(struct dpaa2_eventq)); 631 if (dpaa2_portal->num_linked_evq) 632 dpaa2_portal->num_linked_evq--; 633 } 634 635 if (!dpaa2_portal->num_linked_evq) 636 dpaa2_portal->is_port_linked = false; 637 638 return (int)nb_unlinks; 639 } 640 641 642 static int 643 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, 644 uint64_t *timeout_ticks) 645 { 646 uint32_t scale = 1000*1000; 647 648 EVENTDEV_INIT_FUNC_TRACE(); 649 650 RTE_SET_USED(dev); 651 *timeout_ticks = ns / scale; 652 653 return 0; 654 } 655 656 static void 657 dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f) 658 { 659 EVENTDEV_INIT_FUNC_TRACE(); 660 661 RTE_SET_USED(dev); 662 RTE_SET_USED(f); 663 } 664 665 static int 666 dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev, 667 const struct rte_eth_dev *eth_dev, 668 uint32_t *caps) 669 { 670 const char *ethdev_driver = eth_dev->device->driver->name; 671 672 EVENTDEV_INIT_FUNC_TRACE(); 673 674 RTE_SET_USED(dev); 675 676 if (!strcmp(ethdev_driver, "net_dpaa2")) 677 *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP; 678 else 679 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 680 681 return 0; 682 } 683 684 static int 685 dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev, 686 const struct rte_eth_dev *eth_dev, 687 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 688 { 689 struct dpaa2_eventdev *priv = dev->data->dev_private; 690 uint8_t ev_qid = queue_conf->ev.queue_id; 691 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; 692 int i, ret; 693 694 EVENTDEV_INIT_FUNC_TRACE(); 695 696 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 697 ret = dpaa2_eth_eventq_attach(eth_dev, i, 698 dpcon_id, queue_conf); 699 if (ret) { 700 DPAA2_EVENTDEV_ERR( 701 "Event queue attach failed: err(%d)", ret); 702 goto fail; 703 } 704 } 705 return 0; 706 fail: 707 for (i = (i - 1); i >= 0 ; i--) 708 dpaa2_eth_eventq_detach(eth_dev, i); 709 710 return ret; 711 } 712 713 static int 714 dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev, 715 const struct rte_eth_dev *eth_dev, 716 int32_t rx_queue_id, 717 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 718 { 719 struct dpaa2_eventdev *priv = dev->data->dev_private; 720 uint8_t ev_qid = queue_conf->ev.queue_id; 721 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; 722 int ret; 723 724 EVENTDEV_INIT_FUNC_TRACE(); 725 726 if (rx_queue_id == -1) 727 return dpaa2_eventdev_eth_queue_add_all(dev, 728 eth_dev, queue_conf); 729 730 ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id, 731 dpcon_id, queue_conf); 732 if (ret) { 733 DPAA2_EVENTDEV_ERR( 734 "Event queue attach failed: err(%d)", ret); 735 return ret; 736 } 737 return 0; 738 } 739 740 static int 741 dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev, 742 const struct rte_eth_dev *eth_dev) 743 { 744 int i, ret; 745 746 EVENTDEV_INIT_FUNC_TRACE(); 747 748 RTE_SET_USED(dev); 749 750 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 751 ret = dpaa2_eth_eventq_detach(eth_dev, i); 752 if (ret) { 753 DPAA2_EVENTDEV_ERR( 754 "Event queue detach failed: err(%d)", ret); 755 return ret; 756 } 757 } 758 759 return 0; 760 } 761 762 static int 763 dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev, 764 const struct rte_eth_dev *eth_dev, 765 int32_t rx_queue_id) 766 { 767 int ret; 768 769 EVENTDEV_INIT_FUNC_TRACE(); 770 771 if (rx_queue_id == -1) 772 return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev); 773 774 ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id); 775 if (ret) { 776 DPAA2_EVENTDEV_ERR( 777 "Event queue detach failed: err(%d)", ret); 778 return ret; 779 } 780 781 return 0; 782 } 783 784 static int 785 dpaa2_eventdev_eth_start(const struct rte_eventdev *dev, 786 const struct rte_eth_dev *eth_dev) 787 { 788 EVENTDEV_INIT_FUNC_TRACE(); 789 790 RTE_SET_USED(dev); 791 RTE_SET_USED(eth_dev); 792 793 return 0; 794 } 795 796 static int 797 dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev, 798 const struct rte_eth_dev *eth_dev) 799 { 800 EVENTDEV_INIT_FUNC_TRACE(); 801 802 RTE_SET_USED(dev); 803 RTE_SET_USED(eth_dev); 804 805 return 0; 806 } 807 808 static int 809 dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev, 810 const struct rte_cryptodev *cdev, 811 uint32_t *caps) 812 { 813 const char *name = cdev->data->name; 814 815 EVENTDEV_INIT_FUNC_TRACE(); 816 817 RTE_SET_USED(dev); 818 819 if (!strncmp(name, "dpsec-", 6)) 820 *caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP; 821 else 822 return -1; 823 824 return 0; 825 } 826 827 static int 828 dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev, 829 const struct rte_cryptodev *cryptodev, 830 const struct rte_event *ev) 831 { 832 struct dpaa2_eventdev *priv = dev->data->dev_private; 833 uint8_t ev_qid = ev->queue_id; 834 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; 835 int i, ret; 836 837 EVENTDEV_INIT_FUNC_TRACE(); 838 839 for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) { 840 ret = dpaa2_sec_eventq_attach(cryptodev, i, 841 dpcon_id, ev); 842 if (ret) { 843 DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n", 844 ret); 845 goto fail; 846 } 847 } 848 return 0; 849 fail: 850 for (i = (i - 1); i >= 0 ; i--) 851 dpaa2_sec_eventq_detach(cryptodev, i); 852 853 return ret; 854 } 855 856 static int 857 dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev, 858 const struct rte_cryptodev *cryptodev, 859 int32_t rx_queue_id, 860 const struct rte_event *ev) 861 { 862 struct dpaa2_eventdev *priv = dev->data->dev_private; 863 uint8_t ev_qid = ev->queue_id; 864 uint16_t dpcon_id = priv->evq_info[ev_qid].dpcon->dpcon_id; 865 int ret; 866 867 EVENTDEV_INIT_FUNC_TRACE(); 868 869 if (rx_queue_id == -1) 870 return dpaa2_eventdev_crypto_queue_add_all(dev, 871 cryptodev, ev); 872 873 ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id, 874 dpcon_id, ev); 875 if (ret) { 876 DPAA2_EVENTDEV_ERR( 877 "dpaa2_sec_eventq_attach failed: ret: %d\n", ret); 878 return ret; 879 } 880 return 0; 881 } 882 883 static int 884 dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev, 885 const struct rte_cryptodev *cdev) 886 { 887 int i, ret; 888 889 EVENTDEV_INIT_FUNC_TRACE(); 890 891 RTE_SET_USED(dev); 892 893 for (i = 0; i < cdev->data->nb_queue_pairs; i++) { 894 ret = dpaa2_sec_eventq_detach(cdev, i); 895 if (ret) { 896 DPAA2_EVENTDEV_ERR( 897 "dpaa2_sec_eventq_detach failed:ret %d\n", ret); 898 return ret; 899 } 900 } 901 902 return 0; 903 } 904 905 static int 906 dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev, 907 const struct rte_cryptodev *cryptodev, 908 int32_t rx_queue_id) 909 { 910 int ret; 911 912 EVENTDEV_INIT_FUNC_TRACE(); 913 914 if (rx_queue_id == -1) 915 return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev); 916 917 ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id); 918 if (ret) { 919 DPAA2_EVENTDEV_ERR( 920 "dpaa2_sec_eventq_detach failed: ret: %d\n", ret); 921 return ret; 922 } 923 924 return 0; 925 } 926 927 static int 928 dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev, 929 const struct rte_cryptodev *cryptodev) 930 { 931 EVENTDEV_INIT_FUNC_TRACE(); 932 933 RTE_SET_USED(dev); 934 RTE_SET_USED(cryptodev); 935 936 return 0; 937 } 938 939 static int 940 dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev, 941 const struct rte_cryptodev *cryptodev) 942 { 943 EVENTDEV_INIT_FUNC_TRACE(); 944 945 RTE_SET_USED(dev); 946 RTE_SET_USED(cryptodev); 947 948 return 0; 949 } 950 951 static struct rte_eventdev_ops dpaa2_eventdev_ops = { 952 .dev_infos_get = dpaa2_eventdev_info_get, 953 .dev_configure = dpaa2_eventdev_configure, 954 .dev_start = dpaa2_eventdev_start, 955 .dev_stop = dpaa2_eventdev_stop, 956 .dev_close = dpaa2_eventdev_close, 957 .queue_def_conf = dpaa2_eventdev_queue_def_conf, 958 .queue_setup = dpaa2_eventdev_queue_setup, 959 .queue_release = dpaa2_eventdev_queue_release, 960 .port_def_conf = dpaa2_eventdev_port_def_conf, 961 .port_setup = dpaa2_eventdev_port_setup, 962 .port_release = dpaa2_eventdev_port_release, 963 .port_link = dpaa2_eventdev_port_link, 964 .port_unlink = dpaa2_eventdev_port_unlink, 965 .timeout_ticks = dpaa2_eventdev_timeout_ticks, 966 .dump = dpaa2_eventdev_dump, 967 .dev_selftest = test_eventdev_dpaa2, 968 .eth_rx_adapter_caps_get = dpaa2_eventdev_eth_caps_get, 969 .eth_rx_adapter_queue_add = dpaa2_eventdev_eth_queue_add, 970 .eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del, 971 .eth_rx_adapter_start = dpaa2_eventdev_eth_start, 972 .eth_rx_adapter_stop = dpaa2_eventdev_eth_stop, 973 .crypto_adapter_caps_get = dpaa2_eventdev_crypto_caps_get, 974 .crypto_adapter_queue_pair_add = dpaa2_eventdev_crypto_queue_add, 975 .crypto_adapter_queue_pair_del = dpaa2_eventdev_crypto_queue_del, 976 .crypto_adapter_start = dpaa2_eventdev_crypto_start, 977 .crypto_adapter_stop = dpaa2_eventdev_crypto_stop, 978 }; 979 980 static int 981 dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, 982 struct dpaa2_dpcon_dev *dpcon_dev) 983 { 984 struct dpci_rx_queue_cfg rx_queue_cfg; 985 int ret, i; 986 987 /*Do settings to get the frame on a DPCON object*/ 988 rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST | 989 DPCI_QUEUE_OPT_USER_CTX; 990 rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON; 991 rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id; 992 rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO; 993 994 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb = 995 dpaa2_eventdev_process_parallel; 996 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb = 997 dpaa2_eventdev_process_atomic; 998 999 for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) { 1000 rx_queue_cfg.user_ctx = (size_t)(&dpci_dev->rx_queue[i]); 1001 ret = dpci_set_rx_queue(&dpci_dev->dpci, 1002 CMD_PRI_LOW, 1003 dpci_dev->token, i, 1004 &rx_queue_cfg); 1005 if (ret) { 1006 DPAA2_EVENTDEV_ERR( 1007 "DPCI Rx queue setup failed: err(%d)", 1008 ret); 1009 return ret; 1010 } 1011 } 1012 return 0; 1013 } 1014 1015 static int 1016 dpaa2_eventdev_create(const char *name) 1017 { 1018 struct rte_eventdev *eventdev; 1019 struct dpaa2_eventdev *priv; 1020 struct dpaa2_dpcon_dev *dpcon_dev = NULL; 1021 struct dpaa2_dpci_dev *dpci_dev = NULL; 1022 int ret; 1023 1024 eventdev = rte_event_pmd_vdev_init(name, 1025 sizeof(struct dpaa2_eventdev), 1026 rte_socket_id()); 1027 if (eventdev == NULL) { 1028 DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name); 1029 goto fail; 1030 } 1031 1032 eventdev->dev_ops = &dpaa2_eventdev_ops; 1033 eventdev->enqueue = dpaa2_eventdev_enqueue; 1034 eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst; 1035 eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst; 1036 eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst; 1037 eventdev->dequeue = dpaa2_eventdev_dequeue; 1038 eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst; 1039 1040 /* For secondary processes, the primary has done all the work */ 1041 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1042 return 0; 1043 1044 priv = eventdev->data->dev_private; 1045 priv->max_event_queues = 0; 1046 1047 do { 1048 dpcon_dev = rte_dpaa2_alloc_dpcon_dev(); 1049 if (!dpcon_dev) 1050 break; 1051 priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev; 1052 1053 dpci_dev = rte_dpaa2_alloc_dpci_dev(); 1054 if (!dpci_dev) { 1055 rte_dpaa2_free_dpcon_dev(dpcon_dev); 1056 break; 1057 } 1058 priv->evq_info[priv->max_event_queues].dpci = dpci_dev; 1059 1060 ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev); 1061 if (ret) { 1062 DPAA2_EVENTDEV_ERR( 1063 "DPCI setup failed: err(%d)", ret); 1064 return ret; 1065 } 1066 priv->max_event_queues++; 1067 } while (dpcon_dev && dpci_dev); 1068 1069 RTE_LOG(INFO, PMD, "%s eventdev created\n", name); 1070 1071 return 0; 1072 fail: 1073 return -EFAULT; 1074 } 1075 1076 static int 1077 dpaa2_eventdev_destroy(const char *name) 1078 { 1079 struct rte_eventdev *eventdev; 1080 struct dpaa2_eventdev *priv; 1081 int i; 1082 1083 eventdev = rte_event_pmd_get_named_dev(name); 1084 if (eventdev == NULL) { 1085 RTE_EDEV_LOG_ERR("eventdev with name %s not allocated", name); 1086 return -1; 1087 } 1088 1089 /* For secondary processes, the primary has done all the work */ 1090 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1091 return 0; 1092 1093 priv = eventdev->data->dev_private; 1094 for (i = 0; i < priv->max_event_queues; i++) { 1095 if (priv->evq_info[i].dpcon) 1096 rte_dpaa2_free_dpcon_dev(priv->evq_info[i].dpcon); 1097 1098 if (priv->evq_info[i].dpci) 1099 rte_dpaa2_free_dpci_dev(priv->evq_info[i].dpci); 1100 1101 } 1102 priv->max_event_queues = 0; 1103 1104 RTE_LOG(INFO, PMD, "%s eventdev cleaned\n", name); 1105 return 0; 1106 } 1107 1108 1109 static int 1110 dpaa2_eventdev_probe(struct rte_vdev_device *vdev) 1111 { 1112 const char *name; 1113 1114 name = rte_vdev_device_name(vdev); 1115 DPAA2_EVENTDEV_INFO("Initializing %s", name); 1116 return dpaa2_eventdev_create(name); 1117 } 1118 1119 static int 1120 dpaa2_eventdev_remove(struct rte_vdev_device *vdev) 1121 { 1122 const char *name; 1123 1124 name = rte_vdev_device_name(vdev); 1125 DPAA2_EVENTDEV_INFO("Closing %s", name); 1126 1127 dpaa2_eventdev_destroy(name); 1128 1129 return rte_event_pmd_vdev_uninit(name); 1130 } 1131 1132 static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = { 1133 .probe = dpaa2_eventdev_probe, 1134 .remove = dpaa2_eventdev_remove 1135 }; 1136 1137 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd); 1138 1139 RTE_INIT(dpaa2_eventdev_init_log) 1140 { 1141 dpaa2_logtype_event = rte_log_register("pmd.event.dpaa2"); 1142 if (dpaa2_logtype_event >= 0) 1143 rte_log_set_level(dpaa2_logtype_event, RTE_LOG_NOTICE); 1144 } 1145