1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2020 Intel Corporation 3 */ 4 5 #ifndef _DLB2_PRIV_H_ 6 #define _DLB2_PRIV_H_ 7 8 #include <emmintrin.h> 9 #include <stdbool.h> 10 11 #include <rte_eventdev.h> 12 #include <rte_config.h> 13 #include "dlb2_user.h" 14 #include "dlb2_log.h" 15 #include "rte_pmd_dlb2.h" 16 17 #ifndef RTE_LIBRTE_PMD_DLB2_QUELL_STATS 18 #define DLB2_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val) 19 #else 20 #define DLB2_INC_STAT(_stat, _incr_val) 21 #endif 22 23 #define EVDEV_DLB2_NAME_PMD dlb2_event 24 25 /* Default values for command line devargs */ 26 #define DLB2_POLL_INTERVAL_DEFAULT 1000 27 #define DLB2_SW_CREDIT_QUANTA_DEFAULT 32 /* Default = Worker */ 28 #define DLB2_SW_CREDIT_P_QUANTA_DEFAULT 256 /* Producer */ 29 #define DLB2_SW_CREDIT_C_QUANTA_DEFAULT 256 /* Consumer */ 30 #define DLB2_DEPTH_THRESH_DEFAULT 256 31 32 /* command line arg strings */ 33 #define NUMA_NODE_ARG "numa_node" 34 #define DLB2_MAX_NUM_EVENTS "max_num_events" 35 #define DLB2_NUM_DIR_CREDITS "num_dir_credits" 36 #define DEV_ID_ARG "dev_id" 37 #define DLB2_QID_DEPTH_THRESH_ARG "qid_depth_thresh" 38 #define DLB2_COS_ARG "cos" 39 #define DLB2_POLL_INTERVAL_ARG "poll_interval" 40 #define DLB2_SW_CREDIT_QUANTA_ARG "sw_credit_quanta" 41 #define DLB2_HW_CREDIT_QUANTA_ARG "hw_credit_quanta" 42 #define DLB2_DEPTH_THRESH_ARG "default_depth_thresh" 43 #define DLB2_VECTOR_OPTS_ENAB_ARG "vector_opts_enable" 44 45 /* Begin HW related defines and structs */ 46 47 #define DLB2_HW_V2 0 48 #define DLB2_HW_V2_5 1 49 #define DLB2_MAX_NUM_DOMAINS 32 50 #define DLB2_MAX_NUM_VFS 16 51 #define DLB2_MAX_NUM_LDB_QUEUES 32 52 #define DLB2_MAX_NUM_LDB_PORTS 64 53 #define DLB2_MAX_NUM_DIR_PORTS_V2 DLB2_MAX_NUM_DIR_QUEUES_V2 54 #define DLB2_MAX_NUM_DIR_PORTS_V2_5 DLB2_MAX_NUM_DIR_QUEUES_V2_5 55 #define DLB2_MAX_NUM_DIR_PORTS(ver) (ver == DLB2_HW_V2 ? \ 56 DLB2_MAX_NUM_DIR_PORTS_V2 : \ 57 DLB2_MAX_NUM_DIR_PORTS_V2_5) 58 #define DLB2_MAX_NUM_DIR_QUEUES_V2 64 /* DIR == directed */ 59 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5 96 60 /* When needed for array sizing, the DLB 2.5 macro is used */ 61 #define DLB2_MAX_NUM_DIR_QUEUES(ver) (ver == DLB2_HW_V2 ? \ 62 DLB2_MAX_NUM_DIR_QUEUES_V2 : \ 63 DLB2_MAX_NUM_DIR_QUEUES_V2_5) 64 #define DLB2_MAX_NUM_FLOWS (64 * 1024) 65 #define DLB2_MAX_NUM_LDB_CREDITS (8 * 1024) 66 #define DLB2_MAX_NUM_DIR_CREDITS(ver) (ver == DLB2_HW_V2 ? 4096 : 0) 67 #define DLB2_MAX_NUM_CREDITS(ver) (ver == DLB2_HW_V2 ? \ 68 0 : DLB2_MAX_NUM_LDB_CREDITS) 69 #define DLB2_MAX_NUM_LDB_CREDIT_POOLS 64 70 #define DLB2_MAX_NUM_DIR_CREDIT_POOLS 64 71 #define DLB2_MAX_NUM_HIST_LIST_ENTRIES 2048 72 #define DLB2_MAX_NUM_QIDS_PER_LDB_CQ 8 73 #define DLB2_QID_PRIORITIES 8 74 #define DLB2_MAX_DEVICE_PATH 32 75 #define DLB2_MIN_DEQUEUE_TIMEOUT_NS 1 76 /* Note: "- 1" here to support the timeout range check in eventdev_autotest */ 77 #define DLB2_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1) 78 #define DLB2_SW_CREDIT_BATCH_SZ 32 /* Default - Worker */ 79 #define DLB2_SW_CREDIT_P_BATCH_SZ 256 /* Producer */ 80 #define DLB2_SW_CREDIT_C_BATCH_SZ 256 /* Consumer */ 81 #define DLB2_NUM_SN_GROUPS 2 82 #define DLB2_MAX_LDB_SN_ALLOC 1024 83 #define DLB2_MAX_QUEUE_DEPTH_THRESHOLD 8191 84 85 /* 2048 total hist list entries and 64 total ldb ports, which 86 * makes for 2048/64 == 32 hist list entries per port. However, CQ 87 * depth must be a power of 2 and must also be >= HIST LIST entries. 88 * As a result we just limit the maximum dequeue depth to 32. 89 */ 90 #define DLB2_MIN_CQ_DEPTH 1 91 #define DLB2_MAX_CQ_DEPTH 32 92 #define DLB2_MIN_HARDWARE_CQ_DEPTH 8 93 #define DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \ 94 DLB2_MAX_CQ_DEPTH 95 96 #define DLB2_HW_DEVICE_FROM_PCI_ID(_pdev) \ 97 (((_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_PF) || \ 98 (_pdev->id.device_id == PCI_DEVICE_ID_INTEL_DLB2_5_VF)) ? \ 99 DLB2_HW_V2_5 : DLB2_HW_V2) 100 101 /* 102 * Static per queue/port provisioning values 103 */ 104 #define DLB2_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 64 105 106 #define CQ_BASE(is_dir) ((is_dir) ? DLB2_DIR_CQ_BASE : DLB2_LDB_CQ_BASE) 107 #define CQ_SIZE(is_dir) ((is_dir) ? DLB2_DIR_CQ_MAX_SIZE : \ 108 DLB2_LDB_CQ_MAX_SIZE) 109 #define PP_BASE(is_dir) ((is_dir) ? DLB2_DIR_PP_BASE : DLB2_LDB_PP_BASE) 110 111 #define DLB2_NUM_QES_PER_CACHE_LINE 4 112 113 #define DLB2_MAX_ENQUEUE_DEPTH 64 114 #define DLB2_MIN_ENQUEUE_DEPTH 4 115 116 #define DLB2_NAME_SIZE 64 117 118 #define DLB2_1K 1024 119 #define DLB2_2K (2 * DLB2_1K) 120 #define DLB2_4K (4 * DLB2_1K) 121 #define DLB2_16K (16 * DLB2_1K) 122 #define DLB2_32K (32 * DLB2_1K) 123 #define DLB2_1MB (DLB2_1K * DLB2_1K) 124 #define DLB2_16MB (16 * DLB2_1MB) 125 126 /* Use the upper 3 bits of the event priority to select the DLB2 priority */ 127 #define EV_TO_DLB2_PRIO(x) ((x) >> 5) 128 #define DLB2_TO_EV_PRIO(x) ((x) << 5) 129 130 enum dlb2_hw_ver { 131 DLB2_HW_VER_2, 132 DLB2_HW_VER_2_5, 133 }; 134 135 enum dlb2_hw_port_types { 136 DLB2_LDB_PORT, 137 DLB2_DIR_PORT, 138 DLB2_NUM_PORT_TYPES /* Must be last */ 139 }; 140 141 enum dlb2_hw_queue_types { 142 DLB2_LDB_QUEUE, 143 DLB2_DIR_QUEUE, 144 DLB2_NUM_QUEUE_TYPES /* Must be last */ 145 }; 146 147 #define DLB2_COMBINED_POOL DLB2_LDB_QUEUE 148 149 #define PORT_TYPE(p) ((p)->is_directed ? DLB2_DIR_PORT : DLB2_LDB_PORT) 150 151 /* Do not change - must match hardware! */ 152 enum dlb2_hw_sched_type { 153 DLB2_SCHED_ATOMIC = 0, 154 DLB2_SCHED_UNORDERED, 155 DLB2_SCHED_ORDERED, 156 DLB2_SCHED_DIRECTED, 157 /* DLB2_NUM_HW_SCHED_TYPES must be last */ 158 DLB2_NUM_HW_SCHED_TYPES 159 }; 160 161 struct dlb2_hw_rsrcs { 162 int32_t nb_events_limit; 163 uint32_t num_queues; /* Total queues (lb + dir) */ 164 uint32_t num_ldb_queues; /* Number of available ldb queues */ 165 uint32_t num_ldb_ports; /* Number of load balanced ports */ 166 uint32_t num_dir_ports; /* Number of directed ports */ 167 union { 168 struct { 169 uint32_t num_ldb_credits; /* Number of ldb credits */ 170 uint32_t num_dir_credits; /* Number of dir credits */ 171 }; 172 struct { 173 uint32_t num_credits; /* Number of combined credits */ 174 }; 175 }; 176 uint32_t reorder_window_size; /* Size of reorder window */ 177 }; 178 179 struct dlb2_hw_resource_info { 180 /**> Max resources that can be provided */ 181 struct dlb2_hw_rsrcs hw_rsrc_max; 182 int num_sched_domains; 183 uint32_t socket_id; 184 }; 185 186 enum dlb2_enqueue_type { 187 /**> 188 * New : Used to inject a new packet into the QM. 189 */ 190 DLB2_ENQ_NEW, 191 /**> 192 * Forward : Enqueues a packet, and 193 * - if atomic: release any lock it holds in the QM 194 * - if ordered: release the packet for egress re-ordering 195 */ 196 DLB2_ENQ_FWD, 197 /**> 198 * Enqueue Drop : Release an inflight packet. Must be called with 199 * event == NULL. Used to drop a packet. 200 * 201 * Note that all packets dequeued from a load-balanced port must be 202 * released, either with DLB2_ENQ_DROP or DLB2_ENQ_FWD. 203 */ 204 DLB2_ENQ_DROP, 205 206 /* marker for array sizing etc. */ 207 _DLB2_NB_ENQ_TYPES 208 }; 209 210 /* hw-specific format - do not change */ 211 212 struct dlb2_event_type { 213 uint16_t major:4; 214 uint16_t unused:4; 215 uint16_t sub:8; 216 }; 217 218 union dlb2_opaque_data { 219 uint16_t opaque_data; 220 struct dlb2_event_type event_type; 221 }; 222 223 struct dlb2_msg_info { 224 uint8_t qid; 225 uint8_t sched_type:2; 226 uint8_t priority:3; 227 uint8_t msg_type:3; 228 }; 229 230 #define DLB2_NEW_CMD_BYTE 0x08 231 #define DLB2_FWD_CMD_BYTE 0x0A 232 #define DLB2_COMP_CMD_BYTE 0x02 233 #define DLB2_POP_CMD_BYTE 0x01 234 #define DLB2_NOOP_CMD_BYTE 0x00 235 236 /* hw-specific format - do not change */ 237 struct dlb2_enqueue_qe { 238 uint64_t data; 239 /* Word 3 */ 240 union dlb2_opaque_data u; 241 uint8_t qid; 242 uint8_t sched_type:2; 243 uint8_t priority:3; 244 uint8_t msg_type:3; 245 /* Word 4 */ 246 uint16_t lock_id; 247 uint8_t meas_lat:1; 248 uint8_t rsvd1:2; 249 uint8_t no_dec:1; 250 uint8_t cmp_id:4; 251 union { 252 uint8_t cmd_byte; 253 struct { 254 uint8_t cq_token:1; 255 uint8_t qe_comp:1; 256 uint8_t qe_frag:1; 257 uint8_t qe_valid:1; 258 uint8_t rsvd3:1; 259 uint8_t error:1; 260 uint8_t rsvd:2; 261 }; 262 }; 263 }; 264 265 /* hw-specific format - do not change */ 266 struct dlb2_cq_pop_qe { 267 uint64_t data; 268 union dlb2_opaque_data u; 269 uint8_t qid; 270 uint8_t sched_type:2; 271 uint8_t priority:3; 272 uint8_t msg_type:3; 273 uint16_t tokens:10; 274 uint16_t rsvd2:6; 275 uint8_t meas_lat:1; 276 uint8_t rsvd1:2; 277 uint8_t no_dec:1; 278 uint8_t cmp_id:4; 279 union { 280 uint8_t cmd_byte; 281 struct { 282 uint8_t cq_token:1; 283 uint8_t qe_comp:1; 284 uint8_t qe_frag:1; 285 uint8_t qe_valid:1; 286 uint8_t rsvd3:1; 287 uint8_t error:1; 288 uint8_t rsvd:2; 289 }; 290 }; 291 }; 292 293 /* hw-specific format - do not change */ 294 struct dlb2_dequeue_qe { 295 uint64_t data; 296 union dlb2_opaque_data u; 297 uint8_t qid; 298 uint8_t sched_type:2; 299 uint8_t priority:3; 300 uint8_t msg_type:3; 301 uint16_t flow_id:16; /* was pp_id in v1 */ 302 uint8_t debug; 303 uint8_t cq_gen:1; 304 uint8_t qid_depth:2; /* 2 bits in v2 */ 305 uint8_t rsvd1:2; 306 uint8_t error:1; 307 uint8_t rsvd2:2; 308 }; 309 310 union dlb2_port_config { 311 struct dlb2_create_ldb_port_args ldb; 312 struct dlb2_create_dir_port_args dir; 313 }; 314 315 enum dlb2_port_state { 316 PORT_CLOSED, 317 PORT_STARTED, 318 PORT_STOPPED 319 }; 320 321 enum dlb2_configuration_state { 322 /* The resource has not been configured */ 323 DLB2_NOT_CONFIGURED, 324 /* The resource was configured, but the device was stopped */ 325 DLB2_PREV_CONFIGURED, 326 /* The resource is currently configured */ 327 DLB2_CONFIGURED 328 }; 329 330 struct dlb2_port { 331 uint32_t id; 332 bool is_directed; 333 bool gen_bit; 334 uint16_t dir_credits; 335 uint32_t dequeue_depth; 336 enum dlb2_token_pop_mode token_pop_mode; 337 union dlb2_port_config cfg; 338 uint32_t *credit_pool[DLB2_NUM_QUEUE_TYPES]; /* use __atomic builtins */ 339 union { 340 struct { 341 uint16_t cached_ldb_credits; 342 uint16_t ldb_credits; 343 uint16_t cached_dir_credits; 344 }; 345 struct { 346 uint16_t cached_credits; 347 uint16_t credits; 348 }; 349 }; 350 bool int_armed; 351 uint16_t owed_tokens; 352 int16_t issued_releases; 353 int16_t token_pop_thresh; 354 int cq_depth; 355 uint16_t cq_idx; 356 uint16_t cq_idx_unmasked; 357 uint16_t cq_depth_mask; 358 uint16_t gen_bit_shift; 359 uint64_t cq_rolling_mask; /* 360 * rotate to always have right expected 361 * gen bits 362 */ 363 uint64_t cq_rolling_mask_2; 364 void *cq_addr_cached; /* avoid multiple refs */ 365 enum dlb2_port_state state; 366 enum dlb2_configuration_state config_state; 367 int num_mapped_qids; 368 uint8_t *qid_mappings; 369 struct dlb2_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */ 370 struct dlb2_enqueue_qe *int_arm_qe; 371 struct dlb2_cq_pop_qe *consume_qe; 372 struct dlb2_eventdev *dlb2; /* back ptr */ 373 struct dlb2_eventdev_port *ev_port; /* back ptr */ 374 bool use_scalar; /* force usage of scalar code */ 375 uint16_t hw_credit_quanta; 376 }; 377 378 /* Per-process per-port mmio and memory pointers */ 379 struct process_local_port_data { 380 uint64_t *pp_addr; 381 struct dlb2_dequeue_qe *cq_base; 382 const struct rte_memzone *mz; 383 bool mmaped; 384 }; 385 386 struct dlb2_eventdev; 387 388 struct dlb2_port_low_level_io_functions { 389 void (*pp_enqueue_four)(void *qe4, void *pp_addr); 390 }; 391 392 struct dlb2_config { 393 int configured; 394 int reserved; 395 union { 396 struct { 397 uint32_t num_ldb_credits; 398 uint32_t num_dir_credits; 399 }; 400 struct { 401 uint32_t num_credits; 402 }; 403 }; 404 struct dlb2_create_sched_domain_args resources; 405 }; 406 407 enum dlb2_cos { 408 DLB2_COS_DEFAULT = -1, 409 DLB2_COS_0 = 0, 410 DLB2_COS_1, 411 DLB2_COS_2, 412 DLB2_COS_3 413 }; 414 415 struct dlb2_hw_dev { 416 struct dlb2_config cfg; 417 struct dlb2_hw_resource_info info; 418 void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb2_dev) */ 419 uint32_t domain_id; 420 enum dlb2_cos cos_id; 421 rte_spinlock_t resource_lock; /* for MP support */ 422 } __rte_cache_aligned; 423 424 /* End HW related defines and structs */ 425 426 /* Begin DLB2 PMD Eventdev related defines and structs */ 427 428 #define DLB2_MAX_NUM_QUEUES(ver) \ 429 (DLB2_MAX_NUM_DIR_QUEUES(ver) + DLB2_MAX_NUM_LDB_QUEUES) 430 431 #define DLB2_MAX_NUM_PORTS(ver) \ 432 (DLB2_MAX_NUM_DIR_PORTS(ver) + DLB2_MAX_NUM_LDB_PORTS) 433 434 #define DLB2_MAX_NUM_DIR_QUEUES_V2_5 96 435 #define DLB2_MAX_NUM_DIR_PORTS_V2_5 DLB2_MAX_NUM_DIR_QUEUES_V2_5 436 #define DLB2_MAX_NUM_QUEUES_ALL \ 437 (DLB2_MAX_NUM_DIR_QUEUES_V2_5 + DLB2_MAX_NUM_LDB_QUEUES) 438 #define DLB2_MAX_NUM_PORTS_ALL \ 439 (DLB2_MAX_NUM_DIR_PORTS_V2_5 + DLB2_MAX_NUM_LDB_PORTS) 440 #define DLB2_MAX_INPUT_QUEUE_DEPTH 256 441 442 /** Structure to hold the queue to port link establishment attributes */ 443 444 struct dlb2_event_queue_link { 445 uint8_t queue_id; 446 uint8_t priority; 447 bool mapped; 448 bool valid; 449 }; 450 451 struct dlb2_traffic_stats { 452 uint64_t rx_ok; 453 uint64_t rx_drop; 454 uint64_t rx_interrupt_wait; 455 uint64_t rx_umonitor_umwait; 456 uint64_t tx_ok; 457 uint64_t total_polls; 458 uint64_t zero_polls; 459 union { 460 struct { 461 uint64_t tx_nospc_ldb_hw_credits; 462 uint64_t tx_nospc_dir_hw_credits; 463 }; 464 struct { 465 uint64_t tx_nospc_hw_credits; 466 }; 467 }; 468 uint64_t tx_nospc_inflight_max; 469 uint64_t tx_nospc_new_event_limit; 470 uint64_t tx_nospc_inflight_credits; 471 }; 472 473 /* DLB2 HW sets the 2bit qid_depth in rx QEs based on the programmable depth 474 * threshold. The global default value in config/common_base (or rte_config.h) 475 * can be overridden on a per-qid basis using a vdev command line parameter. 476 * 3: depth > threshold 477 * 2: threshold >= depth > 3/4 threshold 478 * 1: 3/4 threshold >= depth > 1/2 threshold 479 * 0: depth <= 1/2 threshold. 480 */ 481 #define DLB2_QID_DEPTH_LE50 0 482 #define DLB2_QID_DEPTH_GT50_LE75 1 483 #define DLB2_QID_DEPTH_GT75_LE100 2 484 #define DLB2_QID_DEPTH_GT100 3 485 #define DLB2_NUM_QID_DEPTH_STAT_VALS 4 /* 2 bits */ 486 487 struct dlb2_queue_stats { 488 uint64_t enq_ok; 489 uint64_t qid_depth[DLB2_NUM_QID_DEPTH_STAT_VALS]; 490 }; 491 492 struct dlb2_port_stats { 493 struct dlb2_traffic_stats traffic; 494 uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */ 495 uint64_t tx_implicit_rel; 496 uint64_t tx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES]; 497 uint64_t tx_invalid; 498 uint64_t rx_sched_cnt[DLB2_NUM_HW_SCHED_TYPES]; 499 uint64_t rx_sched_invalid; 500 struct dlb2_queue_stats queue[DLB2_MAX_NUM_QUEUES_ALL]; 501 }; 502 503 struct dlb2_eventdev_port { 504 struct dlb2_port qm_port; /* hw specific data structure */ 505 struct rte_event_port_conf conf; /* user-supplied configuration */ 506 uint16_t inflight_credits; /* num credits this port has right now */ 507 uint16_t credit_update_quanta; 508 struct dlb2_eventdev *dlb2; /* backlink optimization */ 509 struct dlb2_port_stats stats __rte_cache_aligned; 510 struct dlb2_event_queue_link link[DLB2_MAX_NUM_QIDS_PER_LDB_CQ]; 511 int num_links; 512 uint32_t id; /* port id */ 513 /* num releases yet to be completed on this port. 514 * Only applies to load-balanced ports. 515 */ 516 uint16_t outstanding_releases; 517 uint16_t inflight_max; /* app requested max inflights for this port */ 518 /* setup_done is set when the event port is setup */ 519 bool setup_done; 520 /* enq_configured is set when the qm port is created */ 521 bool enq_configured; 522 uint8_t implicit_release; /* release events before dequeuing */ 523 } __rte_cache_aligned; 524 525 struct dlb2_queue { 526 uint32_t num_qid_inflights; /* User config */ 527 uint32_t num_atm_inflights; /* User config */ 528 enum dlb2_configuration_state config_state; 529 int sched_type; /* LB queue only */ 530 uint8_t id; 531 bool is_directed; 532 }; 533 534 struct dlb2_eventdev_queue { 535 struct dlb2_queue qm_queue; 536 struct rte_event_queue_conf conf; /* User config */ 537 int depth_threshold; /* use default if 0 */ 538 uint32_t id; 539 bool setup_done; 540 uint8_t num_links; 541 }; 542 543 enum dlb2_run_state { 544 DLB2_RUN_STATE_STOPPED = 0, 545 DLB2_RUN_STATE_STOPPING, 546 DLB2_RUN_STATE_STARTING, 547 DLB2_RUN_STATE_STARTED 548 }; 549 550 struct dlb2_eventdev { 551 struct dlb2_eventdev_port ev_ports[DLB2_MAX_NUM_PORTS_ALL]; 552 struct dlb2_eventdev_queue ev_queues[DLB2_MAX_NUM_QUEUES_ALL]; 553 uint8_t qm_ldb_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL]; 554 uint8_t qm_dir_to_ev_queue_id[DLB2_MAX_NUM_QUEUES_ALL]; 555 /* store num stats and offset of the stats for each queue */ 556 uint16_t xstats_count_per_qid[DLB2_MAX_NUM_QUEUES_ALL]; 557 uint16_t xstats_offset_for_qid[DLB2_MAX_NUM_QUEUES_ALL]; 558 /* store num stats and offset of the stats for each port */ 559 uint16_t xstats_count_per_port[DLB2_MAX_NUM_PORTS_ALL]; 560 uint16_t xstats_offset_for_port[DLB2_MAX_NUM_PORTS_ALL]; 561 struct dlb2_get_num_resources_args hw_rsrc_query_results; 562 uint32_t xstats_count_mode_queue; 563 struct dlb2_hw_dev qm_instance; /* strictly hw related */ 564 uint64_t global_dequeue_wait_ticks; 565 struct dlb2_xstats_entry *xstats; 566 struct rte_eventdev *event_dev; /* backlink to dev */ 567 uint32_t xstats_count_mode_dev; 568 uint32_t xstats_count_mode_port; 569 uint32_t xstats_count; 570 uint32_t inflights; /* use __atomic builtins */ 571 uint32_t new_event_limit; 572 int max_num_events_override; 573 int num_dir_credits_override; 574 bool vector_opts_enabled; 575 volatile enum dlb2_run_state run_state; 576 uint16_t num_dir_queues; /* total num of evdev dir queues requested */ 577 union { 578 struct { 579 uint16_t num_dir_credits; 580 uint16_t num_ldb_credits; 581 }; 582 struct { 583 uint16_t num_credits; 584 }; 585 }; 586 uint16_t num_queues; /* total queues */ 587 uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */ 588 uint16_t num_ports; /* total num of evdev ports requested */ 589 uint16_t num_ldb_ports; /* total num of ldb ports requested */ 590 uint16_t num_dir_ports; /* total num of dir ports requested */ 591 bool umwait_allowed; 592 bool global_dequeue_wait; /* Not using per dequeue wait if true */ 593 enum dlb2_cq_poll_modes poll_mode; 594 int poll_interval; 595 int sw_credit_quanta; 596 int hw_credit_quanta; 597 int default_depth_thresh; 598 uint8_t revision; 599 uint8_t version; 600 bool configured; 601 union { 602 struct { 603 uint16_t max_ldb_credits; 604 uint16_t max_dir_credits; 605 /* use __atomic builtins */ /* shared hw cred */ 606 uint32_t ldb_credit_pool __rte_cache_aligned; 607 /* use __atomic builtins */ /* shared hw cred */ 608 uint32_t dir_credit_pool __rte_cache_aligned; 609 }; 610 struct { 611 uint16_t max_credits; 612 /* use __atomic builtins */ /* shared hw cred */ 613 uint32_t credit_pool __rte_cache_aligned; 614 }; 615 }; 616 }; 617 618 /* used for collecting and passing around the dev args */ 619 struct dlb2_qid_depth_thresholds { 620 int val[DLB2_MAX_NUM_QUEUES_ALL]; 621 }; 622 623 struct dlb2_devargs { 624 int socket_id; 625 int max_num_events; 626 int num_dir_credits_override; 627 int dev_id; 628 struct dlb2_qid_depth_thresholds qid_depth_thresholds; 629 enum dlb2_cos cos_id; 630 int poll_interval; 631 int sw_credit_quanta; 632 int hw_credit_quanta; 633 int default_depth_thresh; 634 bool vector_opts_enabled; 635 }; 636 637 /* End Eventdev related defines and structs */ 638 639 /* Forwards for non-inlined functions */ 640 641 void dlb2_eventdev_dump(struct rte_eventdev *dev, FILE *f); 642 643 int dlb2_xstats_init(struct dlb2_eventdev *dlb2); 644 645 void dlb2_xstats_uninit(struct dlb2_eventdev *dlb2); 646 647 int dlb2_eventdev_xstats_get(const struct rte_eventdev *dev, 648 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, 649 const unsigned int ids[], uint64_t values[], unsigned int n); 650 651 int dlb2_eventdev_xstats_get_names(const struct rte_eventdev *dev, 652 enum rte_event_dev_xstats_mode mode, uint8_t queue_port_id, 653 struct rte_event_dev_xstats_name *xstat_names, 654 unsigned int *ids, unsigned int size); 655 656 uint64_t dlb2_eventdev_xstats_get_by_name(const struct rte_eventdev *dev, 657 const char *name, unsigned int *id); 658 659 int dlb2_eventdev_xstats_reset(struct rte_eventdev *dev, 660 enum rte_event_dev_xstats_mode mode, 661 int16_t queue_port_id, 662 const uint32_t ids[], 663 uint32_t nb_ids); 664 665 int test_dlb2_eventdev(void); 666 667 int dlb2_primary_eventdev_probe(struct rte_eventdev *dev, 668 const char *name, 669 struct dlb2_devargs *dlb2_args); 670 671 int dlb2_secondary_eventdev_probe(struct rte_eventdev *dev, 672 const char *name); 673 674 uint32_t dlb2_get_queue_depth(struct dlb2_eventdev *dlb2, 675 struct dlb2_eventdev_queue *queue); 676 677 int dlb2_parse_params(const char *params, 678 const char *name, 679 struct dlb2_devargs *dlb2_args, 680 uint8_t version); 681 682 /* Extern globals */ 683 extern struct process_local_port_data dlb2_port[][DLB2_NUM_PORT_TYPES]; 684 685 #endif /* _DLB2_PRIV_H_ */ 686