1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017,2019 NXP 3 */ 4 5 #include <assert.h> 6 #include <stdio.h> 7 #include <stdbool.h> 8 #include <errno.h> 9 #include <stdint.h> 10 #include <string.h> 11 #include <sys/epoll.h> 12 13 #include <rte_atomic.h> 14 #include <rte_byteorder.h> 15 #include <rte_common.h> 16 #include <rte_debug.h> 17 #include <rte_dev.h> 18 #include <rte_eal.h> 19 #include <rte_fslmc.h> 20 #include <rte_lcore.h> 21 #include <rte_log.h> 22 #include <rte_malloc.h> 23 #include <rte_memcpy.h> 24 #include <rte_memory.h> 25 #include <rte_pci.h> 26 #include <rte_bus_vdev.h> 27 #include <rte_ethdev_driver.h> 28 #include <rte_cryptodev.h> 29 #include <rte_event_eth_rx_adapter.h> 30 #include <rte_event_eth_tx_adapter.h> 31 32 #include <fslmc_vfio.h> 33 #include <dpaa2_hw_pvt.h> 34 #include <dpaa2_hw_mempool.h> 35 #include <dpaa2_hw_dpio.h> 36 #include <dpaa2_ethdev.h> 37 #include <dpaa2_sec_event.h> 38 #include "dpaa2_eventdev.h" 39 #include "dpaa2_eventdev_logs.h" 40 #include <portal/dpaa2_hw_pvt.h> 41 #include <mc/fsl_dpci.h> 42 43 /* Clarifications 44 * Evendev = SoC Instance 45 * Eventport = DPIO Instance 46 * Eventqueue = DPCON Instance 47 * 1 Eventdev can have N Eventqueue 48 * Soft Event Flow is DPCI Instance 49 */ 50 51 #define DPAA2_EV_TX_RETRY_COUNT 10000 52 53 static uint16_t 54 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[], 55 uint16_t nb_events) 56 { 57 58 struct dpaa2_port *dpaa2_portal = port; 59 struct dpaa2_dpio_dev *dpio_dev; 60 uint32_t queue_id = ev[0].queue_id; 61 struct dpaa2_eventq *evq_info; 62 uint32_t fqid, retry_count; 63 struct qbman_swp *swp; 64 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 65 uint32_t loop, frames_to_send; 66 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 67 uint16_t num_tx = 0; 68 int i, n, ret; 69 uint8_t channel_index; 70 71 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 72 /* Affine current thread context to a qman portal */ 73 ret = dpaa2_affine_qbman_swp(); 74 if (ret < 0) { 75 DPAA2_EVENTDEV_ERR( 76 "Failed to allocate IO portal, tid: %d\n", 77 rte_gettid()); 78 return 0; 79 } 80 } 81 /* todo - dpaa2_portal shall have dpio_dev - no per thread variable */ 82 dpio_dev = DPAA2_PER_LCORE_DPIO; 83 swp = DPAA2_PER_LCORE_PORTAL; 84 85 if (likely(dpaa2_portal->is_port_linked)) 86 goto skip_linking; 87 88 /* Create mapping between portal and channel to receive packets */ 89 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 90 evq_info = &dpaa2_portal->evq_info[i]; 91 if (!evq_info->event_port) 92 continue; 93 94 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 95 CMD_PRI_LOW, 96 dpio_dev->token, 97 evq_info->dpcon->dpcon_id, 98 &channel_index); 99 if (ret < 0) { 100 DPAA2_EVENTDEV_ERR( 101 "Static dequeue config failed: err(%d)", ret); 102 goto err; 103 } 104 105 qbman_swp_push_set(swp, channel_index, 1); 106 evq_info->dpcon->channel_index = channel_index; 107 } 108 dpaa2_portal->is_port_linked = true; 109 110 skip_linking: 111 evq_info = &dpaa2_portal->evq_info[queue_id]; 112 113 while (nb_events) { 114 frames_to_send = (nb_events > dpaa2_eqcr_size) ? 115 dpaa2_eqcr_size : nb_events; 116 117 for (loop = 0; loop < frames_to_send; loop++) { 118 const struct rte_event *event = &ev[num_tx + loop]; 119 120 if (event->sched_type != RTE_SCHED_TYPE_ATOMIC) 121 fqid = evq_info->dpci->rx_queue[ 122 DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid; 123 else 124 fqid = evq_info->dpci->rx_queue[ 125 DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid; 126 127 /* Prepare enqueue descriptor */ 128 qbman_eq_desc_clear(&eqdesc[loop]); 129 qbman_eq_desc_set_fq(&eqdesc[loop], fqid); 130 qbman_eq_desc_set_no_orp(&eqdesc[loop], 0); 131 qbman_eq_desc_set_response(&eqdesc[loop], 0, 0); 132 133 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC 134 && event->mbuf->seqn) { 135 uint8_t dqrr_index = event->mbuf->seqn - 1; 136 137 qbman_eq_desc_set_dca(&eqdesc[loop], 1, 138 dqrr_index, 0); 139 DPAA2_PER_LCORE_DQRR_SIZE--; 140 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index); 141 } 142 143 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 144 145 /* 146 * todo - need to align with hw context data 147 * to avoid copy 148 */ 149 struct rte_event *ev_temp = rte_malloc(NULL, 150 sizeof(struct rte_event), 0); 151 152 if (!ev_temp) { 153 if (!loop) 154 return num_tx; 155 frames_to_send = loop; 156 DPAA2_EVENTDEV_ERR( 157 "Unable to allocate event object"); 158 goto send_partial; 159 } 160 rte_memcpy(ev_temp, event, sizeof(struct rte_event)); 161 DPAA2_SET_FD_ADDR((&fd_arr[loop]), (size_t)ev_temp); 162 DPAA2_SET_FD_LEN((&fd_arr[loop]), 163 sizeof(struct rte_event)); 164 } 165 send_partial: 166 loop = 0; 167 retry_count = 0; 168 while (loop < frames_to_send) { 169 ret = qbman_swp_enqueue_multiple_desc(swp, 170 &eqdesc[loop], &fd_arr[loop], 171 frames_to_send - loop); 172 if (unlikely(ret < 0)) { 173 retry_count++; 174 if (retry_count > DPAA2_EV_TX_RETRY_COUNT) { 175 num_tx += loop; 176 nb_events -= loop; 177 return num_tx + loop; 178 } 179 } else { 180 loop += ret; 181 retry_count = 0; 182 } 183 } 184 num_tx += loop; 185 nb_events -= loop; 186 } 187 188 return num_tx; 189 err: 190 for (n = 0; n < i; n++) { 191 evq_info = &dpaa2_portal->evq_info[n]; 192 if (!evq_info->event_port) 193 continue; 194 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 195 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 196 dpio_dev->token, 197 evq_info->dpcon->dpcon_id); 198 } 199 return 0; 200 201 } 202 203 static uint16_t 204 dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev) 205 { 206 return dpaa2_eventdev_enqueue_burst(port, ev, 1); 207 } 208 209 static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks) 210 { 211 struct epoll_event epoll_ev; 212 213 qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL, 214 QBMAN_SWP_INTERRUPT_DQRI); 215 216 epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd, 217 &epoll_ev, 1, timeout_ticks); 218 } 219 220 static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp, 221 const struct qbman_fd *fd, 222 const struct qbman_result *dq, 223 struct dpaa2_queue *rxq, 224 struct rte_event *ev) 225 { 226 struct rte_event *ev_temp = 227 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 228 229 RTE_SET_USED(rxq); 230 231 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 232 rte_free(ev_temp); 233 234 qbman_swp_dqrr_consume(swp, dq); 235 } 236 237 static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp, 238 const struct qbman_fd *fd, 239 const struct qbman_result *dq, 240 struct dpaa2_queue *rxq, 241 struct rte_event *ev) 242 { 243 struct rte_event *ev_temp = 244 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd); 245 uint8_t dqrr_index = qbman_get_dqrr_idx(dq); 246 247 RTE_SET_USED(swp); 248 RTE_SET_USED(rxq); 249 250 rte_memcpy(ev, ev_temp, sizeof(struct rte_event)); 251 rte_free(ev_temp); 252 ev->mbuf->seqn = dqrr_index + 1; 253 DPAA2_PER_LCORE_DQRR_SIZE++; 254 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 255 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; 256 } 257 258 static uint16_t 259 dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[], 260 uint16_t nb_events, uint64_t timeout_ticks) 261 { 262 const struct qbman_result *dq; 263 struct dpaa2_dpio_dev *dpio_dev = NULL; 264 struct dpaa2_port *dpaa2_portal = port; 265 struct dpaa2_eventq *evq_info; 266 struct qbman_swp *swp; 267 const struct qbman_fd *fd; 268 struct dpaa2_queue *rxq; 269 int num_pkts = 0, ret, i = 0, n; 270 uint8_t channel_index; 271 272 if (unlikely(!DPAA2_PER_LCORE_DPIO)) { 273 /* Affine current thread context to a qman portal */ 274 ret = dpaa2_affine_qbman_swp(); 275 if (ret < 0) { 276 DPAA2_EVENTDEV_ERR( 277 "Failed to allocate IO portal, tid: %d\n", 278 rte_gettid()); 279 return 0; 280 } 281 } 282 283 dpio_dev = DPAA2_PER_LCORE_DPIO; 284 swp = DPAA2_PER_LCORE_PORTAL; 285 286 if (likely(dpaa2_portal->is_port_linked)) 287 goto skip_linking; 288 289 /* Create mapping between portal and channel to receive packets */ 290 for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) { 291 evq_info = &dpaa2_portal->evq_info[i]; 292 if (!evq_info->event_port) 293 continue; 294 295 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio, 296 CMD_PRI_LOW, 297 dpio_dev->token, 298 evq_info->dpcon->dpcon_id, 299 &channel_index); 300 if (ret < 0) { 301 DPAA2_EVENTDEV_ERR( 302 "Static dequeue config failed: err(%d)", ret); 303 goto err; 304 } 305 306 qbman_swp_push_set(swp, channel_index, 1); 307 evq_info->dpcon->channel_index = channel_index; 308 } 309 dpaa2_portal->is_port_linked = true; 310 311 skip_linking: 312 /* Check if there are atomic contexts to be released */ 313 while (DPAA2_PER_LCORE_DQRR_SIZE) { 314 if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) { 315 qbman_swp_dqrr_idx_consume(swp, i); 316 DPAA2_PER_LCORE_DQRR_SIZE--; 317 DPAA2_PER_LCORE_DQRR_MBUF(i)->seqn = 318 DPAA2_INVALID_MBUF_SEQN; 319 } 320 i++; 321 } 322 DPAA2_PER_LCORE_DQRR_HELD = 0; 323 324 do { 325 dq = qbman_swp_dqrr_next(swp); 326 if (!dq) { 327 if (!num_pkts && timeout_ticks) { 328 dpaa2_eventdev_dequeue_wait(timeout_ticks); 329 timeout_ticks = 0; 330 continue; 331 } 332 return num_pkts; 333 } 334 qbman_swp_prefetch_dqrr_next(swp); 335 336 fd = qbman_result_DQ_fd(dq); 337 rxq = (struct dpaa2_queue *)(size_t)qbman_result_DQ_fqd_ctx(dq); 338 if (rxq) { 339 rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]); 340 } else { 341 qbman_swp_dqrr_consume(swp, dq); 342 DPAA2_EVENTDEV_ERR("Null Return VQ received"); 343 return 0; 344 } 345 346 num_pkts++; 347 } while (num_pkts < nb_events); 348 349 return num_pkts; 350 err: 351 for (n = 0; n < i; n++) { 352 evq_info = &dpaa2_portal->evq_info[n]; 353 if (!evq_info->event_port) 354 continue; 355 356 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0); 357 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 358 dpio_dev->token, 359 evq_info->dpcon->dpcon_id); 360 } 361 return 0; 362 } 363 364 static uint16_t 365 dpaa2_eventdev_dequeue(void *port, struct rte_event *ev, 366 uint64_t timeout_ticks) 367 { 368 return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks); 369 } 370 371 static void 372 dpaa2_eventdev_info_get(struct rte_eventdev *dev, 373 struct rte_event_dev_info *dev_info) 374 { 375 struct dpaa2_eventdev *priv = dev->data->dev_private; 376 377 EVENTDEV_INIT_FUNC_TRACE(); 378 379 RTE_SET_USED(dev); 380 381 memset(dev_info, 0, sizeof(struct rte_event_dev_info)); 382 dev_info->min_dequeue_timeout_ns = 383 DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT; 384 dev_info->max_dequeue_timeout_ns = 385 DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT; 386 dev_info->dequeue_timeout_ns = 387 DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 388 dev_info->max_event_queues = priv->max_event_queues; 389 dev_info->max_event_queue_flows = 390 DPAA2_EVENT_MAX_QUEUE_FLOWS; 391 dev_info->max_event_queue_priority_levels = 392 DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS; 393 dev_info->max_event_priority_levels = 394 DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS; 395 dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO); 396 /* we only support dpio up to number of cores */ 397 if (dev_info->max_event_ports > rte_lcore_count()) 398 dev_info->max_event_ports = rte_lcore_count(); 399 dev_info->max_event_port_dequeue_depth = 400 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 401 dev_info->max_event_port_enqueue_depth = 402 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 403 dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS; 404 dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED | 405 RTE_EVENT_DEV_CAP_BURST_MODE| 406 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK | 407 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT | 408 RTE_EVENT_DEV_CAP_NONSEQ_MODE | 409 RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES | 410 RTE_EVENT_DEV_CAP_CARRY_FLOW_ID; 411 412 } 413 414 static int 415 dpaa2_eventdev_configure(const struct rte_eventdev *dev) 416 { 417 struct dpaa2_eventdev *priv = dev->data->dev_private; 418 struct rte_event_dev_config *conf = &dev->data->dev_conf; 419 420 EVENTDEV_INIT_FUNC_TRACE(); 421 422 priv->nb_event_queues = conf->nb_event_queues; 423 priv->nb_event_ports = conf->nb_event_ports; 424 priv->nb_event_queue_flows = conf->nb_event_queue_flows; 425 priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth; 426 priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth; 427 priv->event_dev_cfg = conf->event_dev_cfg; 428 429 /* Check dequeue timeout method is per dequeue or global */ 430 if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) { 431 /* 432 * Use timeout value as given in dequeue operation. 433 * So invalidating this timeout value. 434 */ 435 priv->dequeue_timeout_ns = 0; 436 437 } else if (conf->dequeue_timeout_ns == 0) { 438 priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS; 439 } else { 440 priv->dequeue_timeout_ns = conf->dequeue_timeout_ns; 441 } 442 443 DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d", 444 dev->data->dev_id); 445 return 0; 446 } 447 448 static int 449 dpaa2_eventdev_start(struct rte_eventdev *dev) 450 { 451 EVENTDEV_INIT_FUNC_TRACE(); 452 453 RTE_SET_USED(dev); 454 455 return 0; 456 } 457 458 static void 459 dpaa2_eventdev_stop(struct rte_eventdev *dev) 460 { 461 EVENTDEV_INIT_FUNC_TRACE(); 462 463 RTE_SET_USED(dev); 464 } 465 466 static int 467 dpaa2_eventdev_close(struct rte_eventdev *dev) 468 { 469 EVENTDEV_INIT_FUNC_TRACE(); 470 471 RTE_SET_USED(dev); 472 473 return 0; 474 } 475 476 static void 477 dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id, 478 struct rte_event_queue_conf *queue_conf) 479 { 480 EVENTDEV_INIT_FUNC_TRACE(); 481 482 RTE_SET_USED(dev); 483 RTE_SET_USED(queue_id); 484 485 queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS; 486 queue_conf->nb_atomic_order_sequences = 487 DPAA2_EVENT_QUEUE_ORDER_SEQUENCES; 488 queue_conf->schedule_type = RTE_SCHED_TYPE_PARALLEL; 489 queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL; 490 } 491 492 static int 493 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id, 494 const struct rte_event_queue_conf *queue_conf) 495 { 496 struct dpaa2_eventdev *priv = dev->data->dev_private; 497 struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id]; 498 499 EVENTDEV_INIT_FUNC_TRACE(); 500 501 switch (queue_conf->schedule_type) { 502 case RTE_SCHED_TYPE_PARALLEL: 503 case RTE_SCHED_TYPE_ATOMIC: 504 case RTE_SCHED_TYPE_ORDERED: 505 break; 506 default: 507 DPAA2_EVENTDEV_ERR("Schedule type is not supported."); 508 return -1; 509 } 510 evq_info->event_queue_cfg = queue_conf->event_queue_cfg; 511 evq_info->event_queue_id = queue_id; 512 513 return 0; 514 } 515 516 static void 517 dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id) 518 { 519 EVENTDEV_INIT_FUNC_TRACE(); 520 521 RTE_SET_USED(dev); 522 RTE_SET_USED(queue_id); 523 } 524 525 static void 526 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id, 527 struct rte_event_port_conf *port_conf) 528 { 529 EVENTDEV_INIT_FUNC_TRACE(); 530 531 RTE_SET_USED(dev); 532 RTE_SET_USED(port_id); 533 534 port_conf->new_event_threshold = 535 DPAA2_EVENT_MAX_NUM_EVENTS; 536 port_conf->dequeue_depth = 537 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH; 538 port_conf->enqueue_depth = 539 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH; 540 port_conf->event_port_cfg = 0; 541 } 542 543 static int 544 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id, 545 const struct rte_event_port_conf *port_conf) 546 { 547 char event_port_name[32]; 548 struct dpaa2_port *portal; 549 550 EVENTDEV_INIT_FUNC_TRACE(); 551 552 RTE_SET_USED(port_conf); 553 554 sprintf(event_port_name, "event-port-%d", port_id); 555 portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0); 556 if (!portal) { 557 DPAA2_EVENTDEV_ERR("Memory allocation failure"); 558 return -ENOMEM; 559 } 560 561 memset(portal, 0, sizeof(struct dpaa2_port)); 562 dev->data->ports[port_id] = portal; 563 return 0; 564 } 565 566 static void 567 dpaa2_eventdev_port_release(void *port) 568 { 569 struct dpaa2_port *portal = port; 570 571 EVENTDEV_INIT_FUNC_TRACE(); 572 573 if (portal == NULL) 574 return; 575 576 /* TODO: Cleanup is required when ports are in linked state. */ 577 if (portal->is_port_linked) 578 DPAA2_EVENTDEV_WARN("Event port must be unlinked before release"); 579 580 rte_free(portal); 581 } 582 583 static int 584 dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, 585 const uint8_t queues[], const uint8_t priorities[], 586 uint16_t nb_links) 587 { 588 struct dpaa2_eventdev *priv = dev->data->dev_private; 589 struct dpaa2_port *dpaa2_portal = port; 590 struct dpaa2_eventq *evq_info; 591 uint16_t i; 592 593 EVENTDEV_INIT_FUNC_TRACE(); 594 595 RTE_SET_USED(priorities); 596 597 for (i = 0; i < nb_links; i++) { 598 evq_info = &priv->evq_info[queues[i]]; 599 memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info, 600 sizeof(struct dpaa2_eventq)); 601 dpaa2_portal->evq_info[queues[i]].event_port = port; 602 dpaa2_portal->num_linked_evq++; 603 } 604 605 return (int)nb_links; 606 } 607 608 static int 609 dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, 610 uint8_t queues[], uint16_t nb_unlinks) 611 { 612 struct dpaa2_port *dpaa2_portal = port; 613 int i; 614 struct dpaa2_dpio_dev *dpio_dev = NULL; 615 struct dpaa2_eventq *evq_info; 616 struct qbman_swp *swp; 617 618 EVENTDEV_INIT_FUNC_TRACE(); 619 620 RTE_SET_USED(dev); 621 RTE_SET_USED(queues); 622 623 for (i = 0; i < nb_unlinks; i++) { 624 evq_info = &dpaa2_portal->evq_info[queues[i]]; 625 626 if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) { 627 /* todo dpaa2_portal shall have dpio_dev-no per lcore*/ 628 dpio_dev = DPAA2_PER_LCORE_DPIO; 629 swp = DPAA2_PER_LCORE_PORTAL; 630 631 qbman_swp_push_set(swp, 632 evq_info->dpcon->channel_index, 0); 633 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0, 634 dpio_dev->token, 635 evq_info->dpcon->dpcon_id); 636 } 637 memset(evq_info, 0, sizeof(struct dpaa2_eventq)); 638 if (dpaa2_portal->num_linked_evq) 639 dpaa2_portal->num_linked_evq--; 640 } 641 642 if (!dpaa2_portal->num_linked_evq) 643 dpaa2_portal->is_port_linked = false; 644 645 return (int)nb_unlinks; 646 } 647 648 649 static int 650 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns, 651 uint64_t *timeout_ticks) 652 { 653 uint32_t scale = 1000*1000; 654 655 EVENTDEV_INIT_FUNC_TRACE(); 656 657 RTE_SET_USED(dev); 658 *timeout_ticks = ns / scale; 659 660 return 0; 661 } 662 663 static void 664 dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f) 665 { 666 EVENTDEV_INIT_FUNC_TRACE(); 667 668 RTE_SET_USED(dev); 669 RTE_SET_USED(f); 670 } 671 672 static int 673 dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev, 674 const struct rte_eth_dev *eth_dev, 675 uint32_t *caps) 676 { 677 const char *ethdev_driver = eth_dev->device->driver->name; 678 679 EVENTDEV_INIT_FUNC_TRACE(); 680 681 RTE_SET_USED(dev); 682 683 if (!strcmp(ethdev_driver, "net_dpaa2")) 684 *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP; 685 else 686 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP; 687 688 return 0; 689 } 690 691 static int 692 dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev, 693 const struct rte_eth_dev *eth_dev, 694 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 695 { 696 struct dpaa2_eventdev *priv = dev->data->dev_private; 697 uint8_t ev_qid = queue_conf->ev.queue_id; 698 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 699 int i, ret; 700 701 EVENTDEV_INIT_FUNC_TRACE(); 702 703 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 704 ret = dpaa2_eth_eventq_attach(eth_dev, i, 705 dpcon, queue_conf); 706 if (ret) { 707 DPAA2_EVENTDEV_ERR( 708 "Event queue attach failed: err(%d)", ret); 709 goto fail; 710 } 711 } 712 return 0; 713 fail: 714 for (i = (i - 1); i >= 0 ; i--) 715 dpaa2_eth_eventq_detach(eth_dev, i); 716 717 return ret; 718 } 719 720 static int 721 dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev, 722 const struct rte_eth_dev *eth_dev, 723 int32_t rx_queue_id, 724 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf) 725 { 726 struct dpaa2_eventdev *priv = dev->data->dev_private; 727 uint8_t ev_qid = queue_conf->ev.queue_id; 728 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 729 int ret; 730 731 EVENTDEV_INIT_FUNC_TRACE(); 732 733 if (rx_queue_id == -1) 734 return dpaa2_eventdev_eth_queue_add_all(dev, 735 eth_dev, queue_conf); 736 737 ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id, 738 dpcon, queue_conf); 739 if (ret) { 740 DPAA2_EVENTDEV_ERR( 741 "Event queue attach failed: err(%d)", ret); 742 return ret; 743 } 744 return 0; 745 } 746 747 static int 748 dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev, 749 const struct rte_eth_dev *eth_dev) 750 { 751 int i, ret; 752 753 EVENTDEV_INIT_FUNC_TRACE(); 754 755 RTE_SET_USED(dev); 756 757 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 758 ret = dpaa2_eth_eventq_detach(eth_dev, i); 759 if (ret) { 760 DPAA2_EVENTDEV_ERR( 761 "Event queue detach failed: err(%d)", ret); 762 return ret; 763 } 764 } 765 766 return 0; 767 } 768 769 static int 770 dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev, 771 const struct rte_eth_dev *eth_dev, 772 int32_t rx_queue_id) 773 { 774 int ret; 775 776 EVENTDEV_INIT_FUNC_TRACE(); 777 778 if (rx_queue_id == -1) 779 return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev); 780 781 ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id); 782 if (ret) { 783 DPAA2_EVENTDEV_ERR( 784 "Event queue detach failed: err(%d)", ret); 785 return ret; 786 } 787 788 return 0; 789 } 790 791 static int 792 dpaa2_eventdev_eth_start(const struct rte_eventdev *dev, 793 const struct rte_eth_dev *eth_dev) 794 { 795 EVENTDEV_INIT_FUNC_TRACE(); 796 797 RTE_SET_USED(dev); 798 RTE_SET_USED(eth_dev); 799 800 return 0; 801 } 802 803 static int 804 dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev, 805 const struct rte_eth_dev *eth_dev) 806 { 807 EVENTDEV_INIT_FUNC_TRACE(); 808 809 RTE_SET_USED(dev); 810 RTE_SET_USED(eth_dev); 811 812 return 0; 813 } 814 815 static int 816 dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev, 817 const struct rte_cryptodev *cdev, 818 uint32_t *caps) 819 { 820 const char *name = cdev->data->name; 821 822 EVENTDEV_INIT_FUNC_TRACE(); 823 824 RTE_SET_USED(dev); 825 826 if (!strncmp(name, "dpsec-", 6)) 827 *caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP; 828 else 829 return -1; 830 831 return 0; 832 } 833 834 static int 835 dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev, 836 const struct rte_cryptodev *cryptodev, 837 const struct rte_event *ev) 838 { 839 struct dpaa2_eventdev *priv = dev->data->dev_private; 840 uint8_t ev_qid = ev->queue_id; 841 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 842 int i, ret; 843 844 EVENTDEV_INIT_FUNC_TRACE(); 845 846 for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) { 847 ret = dpaa2_sec_eventq_attach(cryptodev, i, dpcon, ev); 848 if (ret) { 849 DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n", 850 ret); 851 goto fail; 852 } 853 } 854 return 0; 855 fail: 856 for (i = (i - 1); i >= 0 ; i--) 857 dpaa2_sec_eventq_detach(cryptodev, i); 858 859 return ret; 860 } 861 862 static int 863 dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev, 864 const struct rte_cryptodev *cryptodev, 865 int32_t rx_queue_id, 866 const struct rte_event *ev) 867 { 868 struct dpaa2_eventdev *priv = dev->data->dev_private; 869 uint8_t ev_qid = ev->queue_id; 870 struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon; 871 int ret; 872 873 EVENTDEV_INIT_FUNC_TRACE(); 874 875 if (rx_queue_id == -1) 876 return dpaa2_eventdev_crypto_queue_add_all(dev, 877 cryptodev, ev); 878 879 ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id, 880 dpcon, ev); 881 if (ret) { 882 DPAA2_EVENTDEV_ERR( 883 "dpaa2_sec_eventq_attach failed: ret: %d\n", ret); 884 return ret; 885 } 886 return 0; 887 } 888 889 static int 890 dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev, 891 const struct rte_cryptodev *cdev) 892 { 893 int i, ret; 894 895 EVENTDEV_INIT_FUNC_TRACE(); 896 897 RTE_SET_USED(dev); 898 899 for (i = 0; i < cdev->data->nb_queue_pairs; i++) { 900 ret = dpaa2_sec_eventq_detach(cdev, i); 901 if (ret) { 902 DPAA2_EVENTDEV_ERR( 903 "dpaa2_sec_eventq_detach failed:ret %d\n", ret); 904 return ret; 905 } 906 } 907 908 return 0; 909 } 910 911 static int 912 dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev, 913 const struct rte_cryptodev *cryptodev, 914 int32_t rx_queue_id) 915 { 916 int ret; 917 918 EVENTDEV_INIT_FUNC_TRACE(); 919 920 if (rx_queue_id == -1) 921 return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev); 922 923 ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id); 924 if (ret) { 925 DPAA2_EVENTDEV_ERR( 926 "dpaa2_sec_eventq_detach failed: ret: %d\n", ret); 927 return ret; 928 } 929 930 return 0; 931 } 932 933 static int 934 dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev, 935 const struct rte_cryptodev *cryptodev) 936 { 937 EVENTDEV_INIT_FUNC_TRACE(); 938 939 RTE_SET_USED(dev); 940 RTE_SET_USED(cryptodev); 941 942 return 0; 943 } 944 945 static int 946 dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev, 947 const struct rte_cryptodev *cryptodev) 948 { 949 EVENTDEV_INIT_FUNC_TRACE(); 950 951 RTE_SET_USED(dev); 952 RTE_SET_USED(cryptodev); 953 954 return 0; 955 } 956 957 static int 958 dpaa2_eventdev_tx_adapter_create(uint8_t id, 959 const struct rte_eventdev *dev) 960 { 961 RTE_SET_USED(id); 962 RTE_SET_USED(dev); 963 964 /* Nothing to do. Simply return. */ 965 return 0; 966 } 967 968 static int 969 dpaa2_eventdev_tx_adapter_caps(const struct rte_eventdev *dev, 970 const struct rte_eth_dev *eth_dev, 971 uint32_t *caps) 972 { 973 RTE_SET_USED(dev); 974 RTE_SET_USED(eth_dev); 975 976 *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT; 977 return 0; 978 } 979 980 static uint16_t 981 dpaa2_eventdev_txa_enqueue_same_dest(void *port, 982 struct rte_event ev[], 983 uint16_t nb_events) 984 { 985 struct rte_mbuf *m[DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH], *m0; 986 uint8_t qid, i; 987 988 RTE_SET_USED(port); 989 990 m0 = (struct rte_mbuf *)ev[0].mbuf; 991 qid = rte_event_eth_tx_adapter_txq_get(m0); 992 993 for (i = 0; i < nb_events; i++) 994 m[i] = (struct rte_mbuf *)ev[i].mbuf; 995 996 return rte_eth_tx_burst(m0->port, qid, m, nb_events); 997 } 998 999 static uint16_t 1000 dpaa2_eventdev_txa_enqueue(void *port, 1001 struct rte_event ev[], 1002 uint16_t nb_events) 1003 { 1004 struct rte_mbuf *m = (struct rte_mbuf *)ev[0].mbuf; 1005 uint8_t qid, i; 1006 1007 RTE_SET_USED(port); 1008 1009 for (i = 0; i < nb_events; i++) { 1010 qid = rte_event_eth_tx_adapter_txq_get(m); 1011 rte_eth_tx_burst(m->port, qid, &m, 1); 1012 } 1013 1014 return nb_events; 1015 } 1016 1017 static struct rte_eventdev_ops dpaa2_eventdev_ops = { 1018 .dev_infos_get = dpaa2_eventdev_info_get, 1019 .dev_configure = dpaa2_eventdev_configure, 1020 .dev_start = dpaa2_eventdev_start, 1021 .dev_stop = dpaa2_eventdev_stop, 1022 .dev_close = dpaa2_eventdev_close, 1023 .queue_def_conf = dpaa2_eventdev_queue_def_conf, 1024 .queue_setup = dpaa2_eventdev_queue_setup, 1025 .queue_release = dpaa2_eventdev_queue_release, 1026 .port_def_conf = dpaa2_eventdev_port_def_conf, 1027 .port_setup = dpaa2_eventdev_port_setup, 1028 .port_release = dpaa2_eventdev_port_release, 1029 .port_link = dpaa2_eventdev_port_link, 1030 .port_unlink = dpaa2_eventdev_port_unlink, 1031 .timeout_ticks = dpaa2_eventdev_timeout_ticks, 1032 .dump = dpaa2_eventdev_dump, 1033 .dev_selftest = test_eventdev_dpaa2, 1034 .eth_rx_adapter_caps_get = dpaa2_eventdev_eth_caps_get, 1035 .eth_rx_adapter_queue_add = dpaa2_eventdev_eth_queue_add, 1036 .eth_rx_adapter_queue_del = dpaa2_eventdev_eth_queue_del, 1037 .eth_rx_adapter_start = dpaa2_eventdev_eth_start, 1038 .eth_rx_adapter_stop = dpaa2_eventdev_eth_stop, 1039 .eth_tx_adapter_caps_get = dpaa2_eventdev_tx_adapter_caps, 1040 .eth_tx_adapter_create = dpaa2_eventdev_tx_adapter_create, 1041 .crypto_adapter_caps_get = dpaa2_eventdev_crypto_caps_get, 1042 .crypto_adapter_queue_pair_add = dpaa2_eventdev_crypto_queue_add, 1043 .crypto_adapter_queue_pair_del = dpaa2_eventdev_crypto_queue_del, 1044 .crypto_adapter_start = dpaa2_eventdev_crypto_start, 1045 .crypto_adapter_stop = dpaa2_eventdev_crypto_stop, 1046 }; 1047 1048 static int 1049 dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev, 1050 struct dpaa2_dpcon_dev *dpcon_dev) 1051 { 1052 struct dpci_rx_queue_cfg rx_queue_cfg; 1053 int ret, i; 1054 1055 /*Do settings to get the frame on a DPCON object*/ 1056 rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST | 1057 DPCI_QUEUE_OPT_USER_CTX; 1058 rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON; 1059 rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id; 1060 rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO; 1061 1062 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb = 1063 dpaa2_eventdev_process_parallel; 1064 dpci_dev->rx_queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb = 1065 dpaa2_eventdev_process_atomic; 1066 1067 for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) { 1068 rx_queue_cfg.user_ctx = (size_t)(&dpci_dev->rx_queue[i]); 1069 ret = dpci_set_rx_queue(&dpci_dev->dpci, 1070 CMD_PRI_LOW, 1071 dpci_dev->token, i, 1072 &rx_queue_cfg); 1073 if (ret) { 1074 DPAA2_EVENTDEV_ERR( 1075 "DPCI Rx queue setup failed: err(%d)", 1076 ret); 1077 return ret; 1078 } 1079 } 1080 return 0; 1081 } 1082 1083 static int 1084 dpaa2_eventdev_create(const char *name) 1085 { 1086 struct rte_eventdev *eventdev; 1087 struct dpaa2_eventdev *priv; 1088 struct dpaa2_dpcon_dev *dpcon_dev = NULL; 1089 struct dpaa2_dpci_dev *dpci_dev = NULL; 1090 int ret; 1091 1092 eventdev = rte_event_pmd_vdev_init(name, 1093 sizeof(struct dpaa2_eventdev), 1094 rte_socket_id()); 1095 if (eventdev == NULL) { 1096 DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name); 1097 goto fail; 1098 } 1099 1100 eventdev->dev_ops = &dpaa2_eventdev_ops; 1101 eventdev->enqueue = dpaa2_eventdev_enqueue; 1102 eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst; 1103 eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst; 1104 eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst; 1105 eventdev->dequeue = dpaa2_eventdev_dequeue; 1106 eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst; 1107 eventdev->txa_enqueue = dpaa2_eventdev_txa_enqueue; 1108 eventdev->txa_enqueue_same_dest = dpaa2_eventdev_txa_enqueue_same_dest; 1109 1110 /* For secondary processes, the primary has done all the work */ 1111 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1112 return 0; 1113 1114 priv = eventdev->data->dev_private; 1115 priv->max_event_queues = 0; 1116 1117 do { 1118 dpcon_dev = rte_dpaa2_alloc_dpcon_dev(); 1119 if (!dpcon_dev) 1120 break; 1121 priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev; 1122 1123 dpci_dev = rte_dpaa2_alloc_dpci_dev(); 1124 if (!dpci_dev) { 1125 rte_dpaa2_free_dpcon_dev(dpcon_dev); 1126 break; 1127 } 1128 priv->evq_info[priv->max_event_queues].dpci = dpci_dev; 1129 1130 ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev); 1131 if (ret) { 1132 DPAA2_EVENTDEV_ERR( 1133 "DPCI setup failed: err(%d)", ret); 1134 return ret; 1135 } 1136 priv->max_event_queues++; 1137 } while (dpcon_dev && dpci_dev); 1138 1139 RTE_LOG(INFO, PMD, "%s eventdev created\n", name); 1140 1141 return 0; 1142 fail: 1143 return -EFAULT; 1144 } 1145 1146 static int 1147 dpaa2_eventdev_destroy(const char *name) 1148 { 1149 struct rte_eventdev *eventdev; 1150 struct dpaa2_eventdev *priv; 1151 int i; 1152 1153 eventdev = rte_event_pmd_get_named_dev(name); 1154 if (eventdev == NULL) { 1155 RTE_EDEV_LOG_ERR("eventdev with name %s not allocated", name); 1156 return -1; 1157 } 1158 1159 /* For secondary processes, the primary has done all the work */ 1160 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1161 return 0; 1162 1163 priv = eventdev->data->dev_private; 1164 for (i = 0; i < priv->max_event_queues; i++) { 1165 if (priv->evq_info[i].dpcon) 1166 rte_dpaa2_free_dpcon_dev(priv->evq_info[i].dpcon); 1167 1168 if (priv->evq_info[i].dpci) 1169 rte_dpaa2_free_dpci_dev(priv->evq_info[i].dpci); 1170 1171 } 1172 priv->max_event_queues = 0; 1173 1174 RTE_LOG(INFO, PMD, "%s eventdev cleaned\n", name); 1175 return 0; 1176 } 1177 1178 1179 static int 1180 dpaa2_eventdev_probe(struct rte_vdev_device *vdev) 1181 { 1182 const char *name; 1183 1184 name = rte_vdev_device_name(vdev); 1185 DPAA2_EVENTDEV_INFO("Initializing %s", name); 1186 return dpaa2_eventdev_create(name); 1187 } 1188 1189 static int 1190 dpaa2_eventdev_remove(struct rte_vdev_device *vdev) 1191 { 1192 const char *name; 1193 1194 name = rte_vdev_device_name(vdev); 1195 DPAA2_EVENTDEV_INFO("Closing %s", name); 1196 1197 dpaa2_eventdev_destroy(name); 1198 1199 return rte_event_pmd_vdev_uninit(name); 1200 } 1201 1202 static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = { 1203 .probe = dpaa2_eventdev_probe, 1204 .remove = dpaa2_eventdev_remove 1205 }; 1206 1207 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd); 1208 RTE_LOG_REGISTER(dpaa2_logtype_event, pmd.event.dpaa2, NOTICE); 1209