1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 20 #include "main.h" 21 #include "test_bbdev_vector.h" 22 23 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 24 25 #define MAX_QUEUES RTE_MAX_LCORE 26 27 #define OPS_CACHE_SIZE 256U 28 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 29 30 #define SYNC_WAIT 0 31 #define SYNC_START 1 32 33 #define INVALID_QUEUE_ID -1 34 35 static struct test_bbdev_vector test_vector; 36 37 /* Switch between PMD and Interrupt for throughput TC */ 38 static bool intr_enabled; 39 40 /* Represents tested active devices */ 41 static struct active_device { 42 const char *driver_name; 43 uint8_t dev_id; 44 uint16_t supported_ops; 45 uint16_t queue_ids[MAX_QUEUES]; 46 uint16_t nb_queues; 47 struct rte_mempool *ops_mempool; 48 struct rte_mempool *in_mbuf_pool; 49 struct rte_mempool *hard_out_mbuf_pool; 50 struct rte_mempool *soft_out_mbuf_pool; 51 } active_devs[RTE_BBDEV_MAX_DEVS]; 52 53 static uint8_t nb_active_devs; 54 55 /* Data buffers used by BBDEV ops */ 56 struct test_buffers { 57 struct rte_bbdev_op_data *inputs; 58 struct rte_bbdev_op_data *hard_outputs; 59 struct rte_bbdev_op_data *soft_outputs; 60 }; 61 62 /* Operation parameters specific for given test case */ 63 struct test_op_params { 64 struct rte_mempool *mp; 65 struct rte_bbdev_dec_op *ref_dec_op; 66 struct rte_bbdev_enc_op *ref_enc_op; 67 uint16_t burst_sz; 68 uint16_t num_to_process; 69 uint16_t num_lcores; 70 int vector_mask; 71 rte_atomic16_t sync; 72 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 73 }; 74 75 /* Contains per lcore params */ 76 struct thread_params { 77 uint8_t dev_id; 78 uint16_t queue_id; 79 uint64_t start_time; 80 double mops; 81 double mbps; 82 rte_atomic16_t nb_dequeued; 83 rte_atomic16_t processing_status; 84 struct test_op_params *op_params; 85 }; 86 87 typedef int (test_case_function)(struct active_device *ad, 88 struct test_op_params *op_params); 89 90 static inline void 91 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 92 { 93 ad->supported_ops |= (1 << op_type); 94 } 95 96 static inline bool 97 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 98 { 99 return ad->supported_ops & (1 << op_type); 100 } 101 102 static inline bool 103 flags_match(uint32_t flags_req, uint32_t flags_present) 104 { 105 return (flags_req & flags_present) == flags_req; 106 } 107 108 static void 109 clear_soft_out_cap(uint32_t *op_flags) 110 { 111 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 112 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 113 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 114 } 115 116 static int 117 check_dev_cap(const struct rte_bbdev_info *dev_info) 118 { 119 unsigned int i; 120 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs; 121 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 122 123 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 124 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 125 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 126 127 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 128 if (op_cap->type != test_vector.op_type) 129 continue; 130 131 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 132 const struct rte_bbdev_op_cap_turbo_dec *cap = 133 &op_cap->cap.turbo_dec; 134 /* Ignore lack of soft output capability, just skip 135 * checking if soft output is valid. 136 */ 137 if ((test_vector.turbo_dec.op_flags & 138 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 139 !(cap->capability_flags & 140 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 141 printf( 142 "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 143 dev_info->dev_name); 144 clear_soft_out_cap( 145 &test_vector.turbo_dec.op_flags); 146 } 147 148 if (!flags_match(test_vector.turbo_dec.op_flags, 149 cap->capability_flags)) 150 return TEST_FAILED; 151 if (nb_inputs > cap->num_buffers_src) { 152 printf("Too many inputs defined: %u, max: %u\n", 153 nb_inputs, cap->num_buffers_src); 154 return TEST_FAILED; 155 } 156 if (nb_soft_outputs > cap->num_buffers_soft_out && 157 (test_vector.turbo_dec.op_flags & 158 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 159 printf( 160 "Too many soft outputs defined: %u, max: %u\n", 161 nb_soft_outputs, 162 cap->num_buffers_soft_out); 163 return TEST_FAILED; 164 } 165 if (nb_hard_outputs > cap->num_buffers_hard_out) { 166 printf( 167 "Too many hard outputs defined: %u, max: %u\n", 168 nb_hard_outputs, 169 cap->num_buffers_hard_out); 170 return TEST_FAILED; 171 } 172 if (intr_enabled && !(cap->capability_flags & 173 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 174 printf( 175 "Dequeue interrupts are not supported!\n"); 176 return TEST_FAILED; 177 } 178 179 return TEST_SUCCESS; 180 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 181 const struct rte_bbdev_op_cap_turbo_enc *cap = 182 &op_cap->cap.turbo_enc; 183 184 if (!flags_match(test_vector.turbo_enc.op_flags, 185 cap->capability_flags)) 186 return TEST_FAILED; 187 if (nb_inputs > cap->num_buffers_src) { 188 printf("Too many inputs defined: %u, max: %u\n", 189 nb_inputs, cap->num_buffers_src); 190 return TEST_FAILED; 191 } 192 if (nb_hard_outputs > cap->num_buffers_dst) { 193 printf( 194 "Too many hard outputs defined: %u, max: %u\n", 195 nb_hard_outputs, cap->num_buffers_src); 196 return TEST_FAILED; 197 } 198 if (intr_enabled && !(cap->capability_flags & 199 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 200 printf( 201 "Dequeue interrupts are not supported!\n"); 202 return TEST_FAILED; 203 } 204 205 return TEST_SUCCESS; 206 } 207 } 208 209 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 210 return TEST_SUCCESS; /* Special case for NULL device */ 211 212 return TEST_FAILED; 213 } 214 215 /* calculates optimal mempool size not smaller than the val */ 216 static unsigned int 217 optimal_mempool_size(unsigned int val) 218 { 219 return rte_align32pow2(val + 1) - 1; 220 } 221 222 /* allocates mbuf mempool for inputs and outputs */ 223 static struct rte_mempool * 224 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 225 int socket_id, unsigned int mbuf_pool_size, 226 const char *op_type_str) 227 { 228 unsigned int i; 229 uint32_t max_seg_sz = 0; 230 char pool_name[RTE_MEMPOOL_NAMESIZE]; 231 232 /* find max input segment size */ 233 for (i = 0; i < entries->nb_segments; ++i) 234 if (entries->segments[i].length > max_seg_sz) 235 max_seg_sz = entries->segments[i].length; 236 237 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 238 dev_id); 239 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 240 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 241 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 242 } 243 244 static int 245 create_mempools(struct active_device *ad, int socket_id, 246 enum rte_bbdev_op_type op_type, uint16_t num_ops) 247 { 248 struct rte_mempool *mp; 249 unsigned int ops_pool_size, mbuf_pool_size = 0; 250 char pool_name[RTE_MEMPOOL_NAMESIZE]; 251 const char *op_type_str; 252 253 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 254 struct op_data_entries *hard_out = 255 &test_vector.entries[DATA_HARD_OUTPUT]; 256 struct op_data_entries *soft_out = 257 &test_vector.entries[DATA_SOFT_OUTPUT]; 258 259 /* allocate ops mempool */ 260 ops_pool_size = optimal_mempool_size(RTE_MAX( 261 /* Ops used plus 1 reference op */ 262 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 263 /* Minimal cache size plus 1 reference op */ 264 (unsigned int)(1.5 * rte_lcore_count() * 265 OPS_CACHE_SIZE + 1)), 266 OPS_POOL_SIZE_MIN)); 267 268 op_type_str = rte_bbdev_op_type_str(op_type); 269 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 270 271 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 272 ad->dev_id); 273 mp = rte_bbdev_op_pool_create(pool_name, op_type, 274 ops_pool_size, OPS_CACHE_SIZE, socket_id); 275 TEST_ASSERT_NOT_NULL(mp, 276 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 277 ops_pool_size, 278 ad->dev_id, 279 socket_id); 280 ad->ops_mempool = mp; 281 282 /* Inputs */ 283 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 284 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 285 TEST_ASSERT_NOT_NULL(mp, 286 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 287 mbuf_pool_size, 288 ad->dev_id, 289 socket_id); 290 ad->in_mbuf_pool = mp; 291 292 /* Hard outputs */ 293 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 294 hard_out->nb_segments); 295 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 296 "hard_out"); 297 TEST_ASSERT_NOT_NULL(mp, 298 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 299 mbuf_pool_size, 300 ad->dev_id, 301 socket_id); 302 ad->hard_out_mbuf_pool = mp; 303 304 if (soft_out->nb_segments == 0) 305 return TEST_SUCCESS; 306 307 /* Soft outputs */ 308 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 309 soft_out->nb_segments); 310 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size, 311 "soft_out"); 312 TEST_ASSERT_NOT_NULL(mp, 313 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 314 mbuf_pool_size, 315 ad->dev_id, 316 socket_id); 317 ad->soft_out_mbuf_pool = mp; 318 319 return 0; 320 } 321 322 static int 323 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 324 struct test_bbdev_vector *vector) 325 { 326 int ret; 327 unsigned int queue_id; 328 struct rte_bbdev_queue_conf qconf; 329 struct active_device *ad = &active_devs[nb_active_devs]; 330 unsigned int nb_queues; 331 enum rte_bbdev_op_type op_type = vector->op_type; 332 333 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 334 /* setup device */ 335 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 336 if (ret < 0) { 337 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 338 dev_id, nb_queues, info->socket_id, ret); 339 return TEST_FAILED; 340 } 341 342 /* configure interrupts if needed */ 343 if (intr_enabled) { 344 ret = rte_bbdev_intr_enable(dev_id); 345 if (ret < 0) { 346 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 347 ret); 348 return TEST_FAILED; 349 } 350 } 351 352 /* setup device queues */ 353 qconf.socket = info->socket_id; 354 qconf.queue_size = info->drv.default_queue_conf.queue_size; 355 qconf.priority = 0; 356 qconf.deferred_start = 0; 357 qconf.op_type = op_type; 358 359 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 360 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 361 if (ret != 0) { 362 printf( 363 "Allocated all queues (id=%u) at prio%u on dev%u\n", 364 queue_id, qconf.priority, dev_id); 365 qconf.priority++; 366 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 367 &qconf); 368 } 369 if (ret != 0) { 370 printf("All queues on dev %u allocated: %u\n", 371 dev_id, queue_id); 372 break; 373 } 374 ad->queue_ids[queue_id] = queue_id; 375 } 376 TEST_ASSERT(queue_id != 0, 377 "ERROR Failed to configure any queues on dev %u", 378 dev_id); 379 ad->nb_queues = queue_id; 380 381 set_avail_op(ad, op_type); 382 383 return TEST_SUCCESS; 384 } 385 386 static int 387 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 388 struct test_bbdev_vector *vector) 389 { 390 int ret; 391 392 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 393 active_devs[nb_active_devs].dev_id = dev_id; 394 395 ret = add_bbdev_dev(dev_id, info, vector); 396 if (ret == TEST_SUCCESS) 397 ++nb_active_devs; 398 return ret; 399 } 400 401 static uint8_t 402 populate_active_devices(void) 403 { 404 int ret; 405 uint8_t dev_id; 406 uint8_t nb_devs_added = 0; 407 struct rte_bbdev_info info; 408 409 RTE_BBDEV_FOREACH(dev_id) { 410 rte_bbdev_info_get(dev_id, &info); 411 412 if (check_dev_cap(&info)) { 413 printf( 414 "Device %d (%s) does not support specified capabilities\n", 415 dev_id, info.dev_name); 416 continue; 417 } 418 419 ret = add_active_device(dev_id, &info, &test_vector); 420 if (ret != 0) { 421 printf("Adding active bbdev %s skipped\n", 422 info.dev_name); 423 continue; 424 } 425 nb_devs_added++; 426 } 427 428 return nb_devs_added; 429 } 430 431 static int 432 read_test_vector(void) 433 { 434 int ret; 435 436 memset(&test_vector, 0, sizeof(test_vector)); 437 printf("Test vector file = %s\n", get_vector_filename()); 438 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 439 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 440 get_vector_filename()); 441 442 return TEST_SUCCESS; 443 } 444 445 static int 446 testsuite_setup(void) 447 { 448 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 449 450 if (populate_active_devices() == 0) { 451 printf("No suitable devices found!\n"); 452 return TEST_SKIPPED; 453 } 454 455 return TEST_SUCCESS; 456 } 457 458 static int 459 interrupt_testsuite_setup(void) 460 { 461 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 462 463 /* Enable interrupts */ 464 intr_enabled = true; 465 466 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 467 if (populate_active_devices() == 0 || 468 test_vector.op_type == RTE_BBDEV_OP_NONE) { 469 intr_enabled = false; 470 printf("No suitable devices found!\n"); 471 return TEST_SKIPPED; 472 } 473 474 return TEST_SUCCESS; 475 } 476 477 static void 478 testsuite_teardown(void) 479 { 480 uint8_t dev_id; 481 482 /* Unconfigure devices */ 483 RTE_BBDEV_FOREACH(dev_id) 484 rte_bbdev_close(dev_id); 485 486 /* Clear active devices structs. */ 487 memset(active_devs, 0, sizeof(active_devs)); 488 nb_active_devs = 0; 489 } 490 491 static int 492 ut_setup(void) 493 { 494 uint8_t i, dev_id; 495 496 for (i = 0; i < nb_active_devs; i++) { 497 dev_id = active_devs[i].dev_id; 498 /* reset bbdev stats */ 499 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 500 "Failed to reset stats of bbdev %u", dev_id); 501 /* start the device */ 502 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 503 "Failed to start bbdev %u", dev_id); 504 } 505 506 return TEST_SUCCESS; 507 } 508 509 static void 510 ut_teardown(void) 511 { 512 uint8_t i, dev_id; 513 struct rte_bbdev_stats stats; 514 515 for (i = 0; i < nb_active_devs; i++) { 516 dev_id = active_devs[i].dev_id; 517 /* read stats and print */ 518 rte_bbdev_stats_get(dev_id, &stats); 519 /* Stop the device */ 520 rte_bbdev_stop(dev_id); 521 } 522 } 523 524 static int 525 init_op_data_objs(struct rte_bbdev_op_data *bufs, 526 struct op_data_entries *ref_entries, 527 struct rte_mempool *mbuf_pool, const uint16_t n, 528 enum op_data_type op_type, uint16_t min_alignment) 529 { 530 int ret; 531 unsigned int i, j; 532 533 for (i = 0; i < n; ++i) { 534 char *data; 535 struct op_data_buf *seg = &ref_entries->segments[0]; 536 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 537 TEST_ASSERT_NOT_NULL(m_head, 538 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 539 op_type, n * ref_entries->nb_segments, 540 mbuf_pool->size); 541 542 bufs[i].data = m_head; 543 bufs[i].offset = 0; 544 bufs[i].length = 0; 545 546 if (op_type == DATA_INPUT) { 547 data = rte_pktmbuf_append(m_head, seg->length); 548 TEST_ASSERT_NOT_NULL(data, 549 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 550 seg->length, op_type); 551 552 TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), 553 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 554 data, min_alignment); 555 rte_memcpy(data, seg->addr, seg->length); 556 bufs[i].length += seg->length; 557 558 559 for (j = 1; j < ref_entries->nb_segments; ++j) { 560 struct rte_mbuf *m_tail = 561 rte_pktmbuf_alloc(mbuf_pool); 562 TEST_ASSERT_NOT_NULL(m_tail, 563 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 564 op_type, 565 n * ref_entries->nb_segments, 566 mbuf_pool->size); 567 seg += 1; 568 569 data = rte_pktmbuf_append(m_tail, seg->length); 570 TEST_ASSERT_NOT_NULL(data, 571 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 572 seg->length, op_type); 573 574 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 575 min_alignment), 576 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 577 data, min_alignment); 578 rte_memcpy(data, seg->addr, seg->length); 579 bufs[i].length += seg->length; 580 581 ret = rte_pktmbuf_chain(m_head, m_tail); 582 TEST_ASSERT_SUCCESS(ret, 583 "Couldn't chain mbufs from %d data type mbuf pool", 584 op_type); 585 } 586 } 587 } 588 589 return 0; 590 } 591 592 static int 593 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 594 const int socket) 595 { 596 int i; 597 598 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 599 if (*buffers == NULL) { 600 printf("WARNING: Failed to allocate op_data on socket %d\n", 601 socket); 602 /* try to allocate memory on other detected sockets */ 603 for (i = 0; i < socket; i++) { 604 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 605 if (*buffers != NULL) 606 break; 607 } 608 } 609 610 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 611 } 612 613 static void 614 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 615 uint16_t n, int8_t max_llr_modulus) 616 { 617 uint16_t i, byte_idx; 618 619 for (i = 0; i < n; ++i) { 620 struct rte_mbuf *m = input_ops[i].data; 621 while (m != NULL) { 622 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 623 input_ops[i].offset); 624 for (byte_idx = 0; byte_idx < input_ops[i].length; 625 ++byte_idx) 626 llr[byte_idx] = round((double)max_llr_modulus * 627 llr[byte_idx] / INT8_MAX); 628 629 m = m->next; 630 } 631 } 632 } 633 634 static int 635 fill_queue_buffers(struct test_op_params *op_params, 636 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 637 struct rte_mempool *soft_out_mp, uint16_t queue_id, 638 const struct rte_bbdev_op_cap *capabilities, 639 uint16_t min_alignment, const int socket_id) 640 { 641 int ret; 642 enum op_data_type type; 643 const uint16_t n = op_params->num_to_process; 644 645 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 646 in_mp, 647 soft_out_mp, 648 hard_out_mp, 649 }; 650 651 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 652 &op_params->q_bufs[socket_id][queue_id].inputs, 653 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 654 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 655 }; 656 657 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 658 struct op_data_entries *ref_entries = 659 &test_vector.entries[type]; 660 if (ref_entries->nb_segments == 0) 661 continue; 662 663 ret = allocate_buffers_on_socket(queue_ops[type], 664 n * sizeof(struct rte_bbdev_op_data), 665 socket_id); 666 TEST_ASSERT_SUCCESS(ret, 667 "Couldn't allocate memory for rte_bbdev_op_data structs"); 668 669 ret = init_op_data_objs(*queue_ops[type], ref_entries, 670 mbuf_pools[type], n, type, min_alignment); 671 TEST_ASSERT_SUCCESS(ret, 672 "Couldn't init rte_bbdev_op_data structs"); 673 } 674 675 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 676 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 677 capabilities->cap.turbo_dec.max_llr_modulus); 678 679 return 0; 680 } 681 682 static void 683 free_buffers(struct active_device *ad, struct test_op_params *op_params) 684 { 685 unsigned int i, j; 686 687 rte_mempool_free(ad->ops_mempool); 688 rte_mempool_free(ad->in_mbuf_pool); 689 rte_mempool_free(ad->hard_out_mbuf_pool); 690 rte_mempool_free(ad->soft_out_mbuf_pool); 691 692 for (i = 0; i < rte_lcore_count(); ++i) { 693 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 694 rte_free(op_params->q_bufs[j][i].inputs); 695 rte_free(op_params->q_bufs[j][i].hard_outputs); 696 rte_free(op_params->q_bufs[j][i].soft_outputs); 697 } 698 } 699 } 700 701 static void 702 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 703 unsigned int start_idx, 704 struct rte_bbdev_op_data *inputs, 705 struct rte_bbdev_op_data *hard_outputs, 706 struct rte_bbdev_op_data *soft_outputs, 707 struct rte_bbdev_dec_op *ref_op) 708 { 709 unsigned int i; 710 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 711 712 for (i = 0; i < n; ++i) { 713 if (turbo_dec->code_block_mode == 0) { 714 ops[i]->turbo_dec.tb_params.ea = 715 turbo_dec->tb_params.ea; 716 ops[i]->turbo_dec.tb_params.eb = 717 turbo_dec->tb_params.eb; 718 ops[i]->turbo_dec.tb_params.k_pos = 719 turbo_dec->tb_params.k_pos; 720 ops[i]->turbo_dec.tb_params.k_neg = 721 turbo_dec->tb_params.k_neg; 722 ops[i]->turbo_dec.tb_params.c = 723 turbo_dec->tb_params.c; 724 ops[i]->turbo_dec.tb_params.c_neg = 725 turbo_dec->tb_params.c_neg; 726 ops[i]->turbo_dec.tb_params.cab = 727 turbo_dec->tb_params.cab; 728 } else { 729 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 730 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 731 } 732 733 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 734 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 735 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 736 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 737 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 738 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 739 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 740 741 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 742 ops[i]->turbo_dec.input = inputs[start_idx + i]; 743 if (soft_outputs != NULL) 744 ops[i]->turbo_dec.soft_output = 745 soft_outputs[start_idx + i]; 746 } 747 } 748 749 static void 750 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 751 unsigned int start_idx, 752 struct rte_bbdev_op_data *inputs, 753 struct rte_bbdev_op_data *outputs, 754 struct rte_bbdev_enc_op *ref_op) 755 { 756 unsigned int i; 757 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 758 for (i = 0; i < n; ++i) { 759 if (turbo_enc->code_block_mode == 0) { 760 ops[i]->turbo_enc.tb_params.ea = 761 turbo_enc->tb_params.ea; 762 ops[i]->turbo_enc.tb_params.eb = 763 turbo_enc->tb_params.eb; 764 ops[i]->turbo_enc.tb_params.k_pos = 765 turbo_enc->tb_params.k_pos; 766 ops[i]->turbo_enc.tb_params.k_neg = 767 turbo_enc->tb_params.k_neg; 768 ops[i]->turbo_enc.tb_params.c = 769 turbo_enc->tb_params.c; 770 ops[i]->turbo_enc.tb_params.c_neg = 771 turbo_enc->tb_params.c_neg; 772 ops[i]->turbo_enc.tb_params.cab = 773 turbo_enc->tb_params.cab; 774 ops[i]->turbo_enc.tb_params.ncb_pos = 775 turbo_enc->tb_params.ncb_pos; 776 ops[i]->turbo_enc.tb_params.ncb_neg = 777 turbo_enc->tb_params.ncb_neg; 778 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 779 } else { 780 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 781 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 782 ops[i]->turbo_enc.cb_params.ncb = 783 turbo_enc->cb_params.ncb; 784 } 785 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 786 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 787 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 788 789 ops[i]->turbo_enc.output = outputs[start_idx + i]; 790 ops[i]->turbo_enc.input = inputs[start_idx + i]; 791 } 792 } 793 794 static int 795 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 796 unsigned int order_idx, const int expected_status) 797 { 798 TEST_ASSERT(op->status == expected_status, 799 "op_status (%d) != expected_status (%d)", 800 op->status, expected_status); 801 802 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 803 "Ordering error, expected %p, got %p", 804 (void *)(uintptr_t)order_idx, op->opaque_data); 805 806 return TEST_SUCCESS; 807 } 808 809 static int 810 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 811 unsigned int order_idx, const int expected_status) 812 { 813 TEST_ASSERT(op->status == expected_status, 814 "op_status (%d) != expected_status (%d)", 815 op->status, expected_status); 816 817 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 818 "Ordering error, expected %p, got %p", 819 (void *)(uintptr_t)order_idx, op->opaque_data); 820 821 return TEST_SUCCESS; 822 } 823 824 static inline int 825 validate_op_chain(struct rte_bbdev_op_data *op, 826 struct op_data_entries *orig_op) 827 { 828 uint8_t i; 829 struct rte_mbuf *m = op->data; 830 uint8_t nb_dst_segments = orig_op->nb_segments; 831 832 TEST_ASSERT(nb_dst_segments == m->nb_segs, 833 "Number of segments differ in original (%u) and filled (%u) op", 834 nb_dst_segments, m->nb_segs); 835 836 for (i = 0; i < nb_dst_segments; ++i) { 837 /* Apply offset to the first mbuf segment */ 838 uint16_t offset = (i == 0) ? op->offset : 0; 839 uint16_t data_len = m->data_len - offset; 840 841 TEST_ASSERT(orig_op->segments[i].length == data_len, 842 "Length of segment differ in original (%u) and filled (%u) op", 843 orig_op->segments[i].length, data_len); 844 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 845 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 846 data_len, 847 "Output buffers (CB=%u) are not equal", i); 848 m = m->next; 849 } 850 851 return TEST_SUCCESS; 852 } 853 854 static int 855 validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs, 856 const uint16_t num_to_process) 857 { 858 int i; 859 860 struct op_data_entries *hard_data_orig = 861 &test_vector.entries[DATA_HARD_OUTPUT]; 862 struct op_data_entries *soft_data_orig = 863 &test_vector.entries[DATA_SOFT_OUTPUT]; 864 865 for (i = 0; i < num_to_process; i++) { 866 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 867 hard_data_orig), 868 "Hard output buffers are not equal"); 869 if (ref_op->turbo_dec.op_flags & 870 RTE_BBDEV_TURBO_SOFT_OUTPUT) 871 TEST_ASSERT_SUCCESS(validate_op_chain( 872 &bufs->soft_outputs[i], 873 soft_data_orig), 874 "Soft output buffers are not equal"); 875 } 876 877 return TEST_SUCCESS; 878 } 879 880 static int 881 validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process) 882 { 883 int i; 884 885 struct op_data_entries *hard_data_orig = 886 &test_vector.entries[DATA_HARD_OUTPUT]; 887 888 for (i = 0; i < num_to_process; i++) 889 TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], 890 hard_data_orig), ""); 891 892 return TEST_SUCCESS; 893 } 894 895 static int 896 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 897 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 898 { 899 unsigned int i; 900 int ret; 901 struct op_data_entries *hard_data_orig = 902 &test_vector.entries[DATA_HARD_OUTPUT]; 903 struct op_data_entries *soft_data_orig = 904 &test_vector.entries[DATA_SOFT_OUTPUT]; 905 struct rte_bbdev_op_turbo_dec *ops_td; 906 struct rte_bbdev_op_data *hard_output; 907 struct rte_bbdev_op_data *soft_output; 908 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 909 910 for (i = 0; i < n; ++i) { 911 ops_td = &ops[i]->turbo_dec; 912 hard_output = &ops_td->hard_output; 913 soft_output = &ops_td->soft_output; 914 915 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 916 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 917 "Returned iter_count (%d) > expected iter_count (%d)", 918 ops_td->iter_count, ref_td->iter_count); 919 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 920 TEST_ASSERT_SUCCESS(ret, 921 "Checking status and ordering for decoder failed"); 922 923 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 924 hard_data_orig), 925 "Hard output buffers (CB=%u) are not equal", 926 i); 927 928 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 929 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 930 soft_data_orig), 931 "Soft output buffers (CB=%u) are not equal", 932 i); 933 } 934 935 return TEST_SUCCESS; 936 } 937 938 static int 939 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 940 struct rte_bbdev_enc_op *ref_op) 941 { 942 unsigned int i; 943 int ret; 944 struct op_data_entries *hard_data_orig = 945 &test_vector.entries[DATA_HARD_OUTPUT]; 946 947 for (i = 0; i < n; ++i) { 948 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 949 TEST_ASSERT_SUCCESS(ret, 950 "Checking status and ordering for encoder failed"); 951 TEST_ASSERT_SUCCESS(validate_op_chain( 952 &ops[i]->turbo_enc.output, 953 hard_data_orig), 954 "Output buffers (CB=%u) are not equal", 955 i); 956 } 957 958 return TEST_SUCCESS; 959 } 960 961 static void 962 create_reference_dec_op(struct rte_bbdev_dec_op *op) 963 { 964 unsigned int i; 965 struct op_data_entries *entry; 966 967 op->turbo_dec = test_vector.turbo_dec; 968 entry = &test_vector.entries[DATA_INPUT]; 969 for (i = 0; i < entry->nb_segments; ++i) 970 op->turbo_dec.input.length += 971 entry->segments[i].length; 972 } 973 974 static void 975 create_reference_enc_op(struct rte_bbdev_enc_op *op) 976 { 977 unsigned int i; 978 struct op_data_entries *entry; 979 980 op->turbo_enc = test_vector.turbo_enc; 981 entry = &test_vector.entries[DATA_INPUT]; 982 for (i = 0; i < entry->nb_segments; ++i) 983 op->turbo_enc.input.length += 984 entry->segments[i].length; 985 } 986 987 static int 988 init_test_op_params(struct test_op_params *op_params, 989 enum rte_bbdev_op_type op_type, const int expected_status, 990 const int vector_mask, struct rte_mempool *ops_mp, 991 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 992 { 993 int ret = 0; 994 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 995 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 996 &op_params->ref_dec_op, 1); 997 else 998 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 999 &op_params->ref_enc_op, 1); 1000 1001 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 1002 1003 op_params->mp = ops_mp; 1004 op_params->burst_sz = burst_sz; 1005 op_params->num_to_process = num_to_process; 1006 op_params->num_lcores = num_lcores; 1007 op_params->vector_mask = vector_mask; 1008 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1009 op_params->ref_dec_op->status = expected_status; 1010 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 1011 op_params->ref_enc_op->status = expected_status; 1012 1013 return 0; 1014 } 1015 1016 static int 1017 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 1018 struct test_op_params *op_params) 1019 { 1020 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 1021 unsigned int i; 1022 struct active_device *ad; 1023 unsigned int burst_sz = get_burst_sz(); 1024 enum rte_bbdev_op_type op_type = test_vector.op_type; 1025 const struct rte_bbdev_op_cap *capabilities = NULL; 1026 1027 ad = &active_devs[dev_id]; 1028 1029 /* Check if device supports op_type */ 1030 if (!is_avail_op(ad, test_vector.op_type)) 1031 return TEST_SUCCESS; 1032 1033 struct rte_bbdev_info info; 1034 rte_bbdev_info_get(ad->dev_id, &info); 1035 socket_id = GET_SOCKET(info.socket_id); 1036 1037 if (op_type == RTE_BBDEV_OP_NONE) 1038 op_type = RTE_BBDEV_OP_TURBO_ENC; 1039 f_ret = create_mempools(ad, socket_id, op_type, 1040 get_num_ops()); 1041 if (f_ret != TEST_SUCCESS) { 1042 printf("Couldn't create mempools"); 1043 goto fail; 1044 } 1045 1046 f_ret = init_test_op_params(op_params, test_vector.op_type, 1047 test_vector.expected_status, 1048 test_vector.mask, 1049 ad->ops_mempool, 1050 burst_sz, 1051 get_num_ops(), 1052 get_num_lcores()); 1053 if (f_ret != TEST_SUCCESS) { 1054 printf("Couldn't init test op params"); 1055 goto fail; 1056 } 1057 1058 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1059 /* Find Decoder capabilities */ 1060 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 1061 while (cap->type != RTE_BBDEV_OP_NONE) { 1062 if (cap->type == RTE_BBDEV_OP_TURBO_DEC) { 1063 capabilities = cap; 1064 break; 1065 } 1066 } 1067 TEST_ASSERT_NOT_NULL(capabilities, 1068 "Couldn't find Decoder capabilities"); 1069 1070 create_reference_dec_op(op_params->ref_dec_op); 1071 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1072 create_reference_enc_op(op_params->ref_enc_op); 1073 1074 for (i = 0; i < ad->nb_queues; ++i) { 1075 f_ret = fill_queue_buffers(op_params, 1076 ad->in_mbuf_pool, 1077 ad->hard_out_mbuf_pool, 1078 ad->soft_out_mbuf_pool, 1079 ad->queue_ids[i], 1080 capabilities, 1081 info.drv.min_alignment, 1082 socket_id); 1083 if (f_ret != TEST_SUCCESS) { 1084 printf("Couldn't init queue buffers"); 1085 goto fail; 1086 } 1087 } 1088 1089 /* Run test case function */ 1090 t_ret = test_case_func(ad, op_params); 1091 1092 /* Free active device resources and return */ 1093 free_buffers(ad, op_params); 1094 return t_ret; 1095 1096 fail: 1097 free_buffers(ad, op_params); 1098 return TEST_FAILED; 1099 } 1100 1101 /* Run given test function per active device per supported op type 1102 * per burst size. 1103 */ 1104 static int 1105 run_test_case(test_case_function *test_case_func) 1106 { 1107 int ret = 0; 1108 uint8_t dev; 1109 1110 /* Alloc op_params */ 1111 struct test_op_params *op_params = rte_zmalloc(NULL, 1112 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1113 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1114 RTE_ALIGN(sizeof(struct test_op_params), 1115 RTE_CACHE_LINE_SIZE)); 1116 1117 /* For each device run test case function */ 1118 for (dev = 0; dev < nb_active_devs; ++dev) 1119 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1120 1121 rte_free(op_params); 1122 1123 return ret; 1124 } 1125 1126 static void 1127 dequeue_event_callback(uint16_t dev_id, 1128 enum rte_bbdev_event_type event, void *cb_arg, 1129 void *ret_param) 1130 { 1131 int ret; 1132 uint16_t i; 1133 uint64_t total_time; 1134 uint16_t deq, burst_sz, num_to_process; 1135 uint16_t queue_id = INVALID_QUEUE_ID; 1136 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 1137 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 1138 struct test_buffers *bufs; 1139 struct rte_bbdev_info info; 1140 1141 /* Input length in bytes, million operations per second, 1142 * million bits per second. 1143 */ 1144 double in_len; 1145 1146 struct thread_params *tp = cb_arg; 1147 1148 RTE_SET_USED(ret_param); 1149 queue_id = tp->queue_id; 1150 1151 /* Find matching thread params using queue_id */ 1152 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1153 if (tp->queue_id == queue_id) 1154 break; 1155 1156 if (i == MAX_QUEUES) { 1157 printf("%s: Queue_id from interrupt details was not found!\n", 1158 __func__); 1159 return; 1160 } 1161 1162 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1163 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1164 printf( 1165 "Dequeue interrupt handler called for incorrect event!\n"); 1166 return; 1167 } 1168 1169 burst_sz = tp->op_params->burst_sz; 1170 num_to_process = tp->op_params->num_to_process; 1171 1172 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1173 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops, 1174 burst_sz); 1175 else 1176 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops, 1177 burst_sz); 1178 1179 if (deq < burst_sz) { 1180 printf( 1181 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1182 burst_sz, deq); 1183 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1184 return; 1185 } 1186 1187 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) { 1188 rte_atomic16_add(&tp->nb_dequeued, deq); 1189 return; 1190 } 1191 1192 total_time = rte_rdtsc_precise() - tp->start_time; 1193 1194 rte_bbdev_info_get(dev_id, &info); 1195 1196 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1197 1198 ret = TEST_SUCCESS; 1199 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1200 ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs, 1201 num_to_process); 1202 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1203 ret = validate_enc_buffers(bufs, num_to_process); 1204 1205 if (ret) { 1206 printf("Buffers validation failed\n"); 1207 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1208 } 1209 1210 switch (test_vector.op_type) { 1211 case RTE_BBDEV_OP_TURBO_DEC: 1212 in_len = tp->op_params->ref_dec_op->turbo_dec.input.length; 1213 break; 1214 case RTE_BBDEV_OP_TURBO_ENC: 1215 in_len = tp->op_params->ref_enc_op->turbo_enc.input.length; 1216 break; 1217 case RTE_BBDEV_OP_NONE: 1218 in_len = 0.0; 1219 break; 1220 default: 1221 printf("Unknown op type: %d\n", test_vector.op_type); 1222 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1223 return; 1224 } 1225 1226 tp->mops = ((double)num_to_process / 1000000.0) / 1227 ((double)total_time / (double)rte_get_tsc_hz()); 1228 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1229 ((double)total_time / (double)rte_get_tsc_hz()); 1230 1231 rte_atomic16_add(&tp->nb_dequeued, deq); 1232 } 1233 1234 static int 1235 throughput_intr_lcore_dec(void *arg) 1236 { 1237 struct thread_params *tp = arg; 1238 unsigned int enqueued; 1239 struct rte_bbdev_dec_op *ops[MAX_BURST]; 1240 const uint16_t queue_id = tp->queue_id; 1241 const uint16_t burst_sz = tp->op_params->burst_sz; 1242 const uint16_t num_to_process = tp->op_params->num_to_process; 1243 struct test_buffers *bufs = NULL; 1244 unsigned int allocs_failed = 0; 1245 struct rte_bbdev_info info; 1246 int ret; 1247 1248 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1249 "BURST_SIZE should be <= %u", MAX_BURST); 1250 1251 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1252 "Failed to enable interrupts for dev: %u, queue_id: %u", 1253 tp->dev_id, queue_id); 1254 1255 rte_bbdev_info_get(tp->dev_id, &info); 1256 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1257 1258 rte_atomic16_clear(&tp->processing_status); 1259 rte_atomic16_clear(&tp->nb_dequeued); 1260 1261 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1262 rte_pause(); 1263 1264 tp->start_time = rte_rdtsc_precise(); 1265 for (enqueued = 0; enqueued < num_to_process;) { 1266 1267 uint16_t num_to_enq = burst_sz; 1268 1269 if (unlikely(num_to_process - enqueued < num_to_enq)) 1270 num_to_enq = num_to_process - enqueued; 1271 1272 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1273 num_to_enq); 1274 if (ret != 0) { 1275 allocs_failed++; 1276 continue; 1277 } 1278 1279 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1280 copy_reference_dec_op(ops, num_to_enq, enqueued, 1281 bufs->inputs, 1282 bufs->hard_outputs, 1283 bufs->soft_outputs, 1284 tp->op_params->ref_dec_op); 1285 1286 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops, 1287 num_to_enq); 1288 1289 rte_bbdev_dec_op_free_bulk(ops, num_to_enq); 1290 } 1291 1292 if (allocs_failed > 0) 1293 printf("WARNING: op allocations failed: %u times\n", 1294 allocs_failed); 1295 1296 return TEST_SUCCESS; 1297 } 1298 1299 static int 1300 throughput_intr_lcore_enc(void *arg) 1301 { 1302 struct thread_params *tp = arg; 1303 unsigned int enqueued; 1304 struct rte_bbdev_enc_op *ops[MAX_BURST]; 1305 const uint16_t queue_id = tp->queue_id; 1306 const uint16_t burst_sz = tp->op_params->burst_sz; 1307 const uint16_t num_to_process = tp->op_params->num_to_process; 1308 struct test_buffers *bufs = NULL; 1309 unsigned int allocs_failed = 0; 1310 struct rte_bbdev_info info; 1311 int ret; 1312 1313 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1314 "BURST_SIZE should be <= %u", MAX_BURST); 1315 1316 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1317 "Failed to enable interrupts for dev: %u, queue_id: %u", 1318 tp->dev_id, queue_id); 1319 1320 rte_bbdev_info_get(tp->dev_id, &info); 1321 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1322 1323 rte_atomic16_clear(&tp->processing_status); 1324 rte_atomic16_clear(&tp->nb_dequeued); 1325 1326 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1327 rte_pause(); 1328 1329 tp->start_time = rte_rdtsc_precise(); 1330 for (enqueued = 0; enqueued < num_to_process;) { 1331 1332 uint16_t num_to_enq = burst_sz; 1333 1334 if (unlikely(num_to_process - enqueued < num_to_enq)) 1335 num_to_enq = num_to_process - enqueued; 1336 1337 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1338 num_to_enq); 1339 if (ret != 0) { 1340 allocs_failed++; 1341 continue; 1342 } 1343 1344 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1345 copy_reference_enc_op(ops, num_to_enq, enqueued, 1346 bufs->inputs, 1347 bufs->hard_outputs, 1348 tp->op_params->ref_enc_op); 1349 1350 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops, 1351 num_to_enq); 1352 1353 rte_bbdev_enc_op_free_bulk(ops, num_to_enq); 1354 } 1355 1356 if (allocs_failed > 0) 1357 printf("WARNING: op allocations failed: %u times\n", 1358 allocs_failed); 1359 1360 return TEST_SUCCESS; 1361 } 1362 1363 static int 1364 throughput_pmd_lcore_dec(void *arg) 1365 { 1366 struct thread_params *tp = arg; 1367 unsigned int enqueued, dequeued; 1368 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1369 uint64_t total_time, start_time; 1370 const uint16_t queue_id = tp->queue_id; 1371 const uint16_t burst_sz = tp->op_params->burst_sz; 1372 const uint16_t num_to_process = tp->op_params->num_to_process; 1373 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1374 struct test_buffers *bufs = NULL; 1375 unsigned int allocs_failed = 0; 1376 int ret; 1377 struct rte_bbdev_info info; 1378 1379 /* Input length in bytes, million operations per second, million bits 1380 * per second. 1381 */ 1382 double in_len; 1383 1384 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1385 "BURST_SIZE should be <= %u", MAX_BURST); 1386 1387 rte_bbdev_info_get(tp->dev_id, &info); 1388 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1389 1390 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1391 rte_pause(); 1392 1393 start_time = rte_rdtsc_precise(); 1394 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1395 uint16_t deq; 1396 1397 if (likely(enqueued < num_to_process)) { 1398 1399 uint16_t num_to_enq = burst_sz; 1400 1401 if (unlikely(num_to_process - enqueued < num_to_enq)) 1402 num_to_enq = num_to_process - enqueued; 1403 1404 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, 1405 ops_enq, num_to_enq); 1406 if (ret != 0) { 1407 allocs_failed++; 1408 goto do_dequeue; 1409 } 1410 1411 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1412 copy_reference_dec_op(ops_enq, num_to_enq, 1413 enqueued, 1414 bufs->inputs, 1415 bufs->hard_outputs, 1416 bufs->soft_outputs, 1417 ref_op); 1418 1419 enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1420 queue_id, ops_enq, num_to_enq); 1421 } 1422 do_dequeue: 1423 deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq, 1424 burst_sz); 1425 dequeued += deq; 1426 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1427 } 1428 total_time = rte_rdtsc_precise() - start_time; 1429 1430 if (allocs_failed > 0) 1431 printf("WARNING: op allocations failed: %u times\n", 1432 allocs_failed); 1433 1434 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1435 enqueued, dequeued); 1436 1437 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1438 ret = validate_dec_buffers(ref_op, bufs, num_to_process); 1439 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1440 } 1441 1442 in_len = ref_op->turbo_dec.input.length; 1443 tp->mops = ((double)num_to_process / 1000000.0) / 1444 ((double)total_time / (double)rte_get_tsc_hz()); 1445 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1446 ((double)total_time / (double)rte_get_tsc_hz()); 1447 1448 return TEST_SUCCESS; 1449 } 1450 1451 static int 1452 throughput_pmd_lcore_enc(void *arg) 1453 { 1454 struct thread_params *tp = arg; 1455 unsigned int enqueued, dequeued; 1456 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1457 uint64_t total_time, start_time; 1458 const uint16_t queue_id = tp->queue_id; 1459 const uint16_t burst_sz = tp->op_params->burst_sz; 1460 const uint16_t num_to_process = tp->op_params->num_to_process; 1461 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1462 struct test_buffers *bufs = NULL; 1463 unsigned int allocs_failed = 0; 1464 int ret; 1465 struct rte_bbdev_info info; 1466 1467 /* Input length in bytes, million operations per second, million bits 1468 * per second. 1469 */ 1470 double in_len; 1471 1472 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1473 "BURST_SIZE should be <= %u", MAX_BURST); 1474 1475 rte_bbdev_info_get(tp->dev_id, &info); 1476 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1477 1478 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1479 rte_pause(); 1480 1481 start_time = rte_rdtsc_precise(); 1482 for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { 1483 uint16_t deq; 1484 1485 if (likely(enqueued < num_to_process)) { 1486 1487 uint16_t num_to_enq = burst_sz; 1488 1489 if (unlikely(num_to_process - enqueued < num_to_enq)) 1490 num_to_enq = num_to_process - enqueued; 1491 1492 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, 1493 ops_enq, num_to_enq); 1494 if (ret != 0) { 1495 allocs_failed++; 1496 goto do_dequeue; 1497 } 1498 1499 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1500 copy_reference_enc_op(ops_enq, num_to_enq, 1501 enqueued, 1502 bufs->inputs, 1503 bufs->hard_outputs, 1504 ref_op); 1505 1506 enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1507 queue_id, ops_enq, num_to_enq); 1508 } 1509 do_dequeue: 1510 deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq, 1511 burst_sz); 1512 dequeued += deq; 1513 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1514 } 1515 total_time = rte_rdtsc_precise() - start_time; 1516 1517 if (allocs_failed > 0) 1518 printf("WARNING: op allocations failed: %u times\n", 1519 allocs_failed); 1520 1521 TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", 1522 enqueued, dequeued); 1523 1524 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1525 ret = validate_enc_buffers(bufs, num_to_process); 1526 TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); 1527 } 1528 1529 in_len = ref_op->turbo_enc.input.length; 1530 1531 tp->mops = ((double)num_to_process / 1000000.0) / 1532 ((double)total_time / (double)rte_get_tsc_hz()); 1533 tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / 1534 ((double)total_time / (double)rte_get_tsc_hz()); 1535 1536 return TEST_SUCCESS; 1537 } 1538 static void 1539 print_throughput(struct thread_params *t_params, unsigned int used_cores) 1540 { 1541 unsigned int lcore_id, iter = 0; 1542 double total_mops = 0, total_mbps = 0; 1543 1544 RTE_LCORE_FOREACH(lcore_id) { 1545 if (iter++ >= used_cores) 1546 break; 1547 printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n", 1548 lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps); 1549 total_mops += t_params[lcore_id].mops; 1550 total_mbps += t_params[lcore_id].mbps; 1551 } 1552 printf( 1553 "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n", 1554 used_cores, total_mops, total_mbps); 1555 } 1556 1557 /* 1558 * Test function that determines how long an enqueue + dequeue of a burst 1559 * takes on available lcores. 1560 */ 1561 static int 1562 throughput_test(struct active_device *ad, 1563 struct test_op_params *op_params) 1564 { 1565 int ret; 1566 unsigned int lcore_id, used_cores = 0; 1567 struct thread_params t_params[MAX_QUEUES]; 1568 struct rte_bbdev_info info; 1569 lcore_function_t *throughput_function; 1570 struct thread_params *tp; 1571 uint16_t num_lcores; 1572 const char *op_type_str; 1573 1574 rte_bbdev_info_get(ad->dev_id, &info); 1575 1576 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 1577 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 1578 test_vector.op_type); 1579 1580 printf( 1581 "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n", 1582 info.dev_name, ad->nb_queues, op_params->burst_sz, 1583 op_params->num_to_process, op_params->num_lcores, 1584 op_type_str, 1585 intr_enabled ? "Interrupt mode" : "PMD mode", 1586 (double)rte_get_tsc_hz() / 1000000000.0); 1587 1588 /* Set number of lcores */ 1589 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 1590 ? ad->nb_queues 1591 : op_params->num_lcores; 1592 1593 if (intr_enabled) { 1594 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1595 throughput_function = throughput_intr_lcore_dec; 1596 else 1597 throughput_function = throughput_intr_lcore_enc; 1598 1599 /* Dequeue interrupt callback registration */ 1600 ret = rte_bbdev_callback_register(ad->dev_id, 1601 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 1602 &t_params); 1603 if (ret < 0) 1604 return ret; 1605 } else { 1606 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1607 throughput_function = throughput_pmd_lcore_dec; 1608 else 1609 throughput_function = throughput_pmd_lcore_enc; 1610 } 1611 1612 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 1613 1614 t_params[rte_lcore_id()].dev_id = ad->dev_id; 1615 t_params[rte_lcore_id()].op_params = op_params; 1616 t_params[rte_lcore_id()].queue_id = 1617 ad->queue_ids[used_cores++]; 1618 1619 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1620 if (used_cores >= num_lcores) 1621 break; 1622 1623 t_params[lcore_id].dev_id = ad->dev_id; 1624 t_params[lcore_id].op_params = op_params; 1625 t_params[lcore_id].queue_id = ad->queue_ids[used_cores++]; 1626 1627 rte_eal_remote_launch(throughput_function, &t_params[lcore_id], 1628 lcore_id); 1629 } 1630 1631 rte_atomic16_set(&op_params->sync, SYNC_START); 1632 ret = throughput_function(&t_params[rte_lcore_id()]); 1633 1634 /* Master core is always used */ 1635 used_cores = 1; 1636 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1637 if (used_cores++ >= num_lcores) 1638 break; 1639 1640 ret |= rte_eal_wait_lcore(lcore_id); 1641 } 1642 1643 /* Return if test failed */ 1644 if (ret) 1645 return ret; 1646 1647 /* Print throughput if interrupts are disabled and test passed */ 1648 if (!intr_enabled) { 1649 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1650 print_throughput(t_params, num_lcores); 1651 return ret; 1652 } 1653 1654 /* In interrupt TC we need to wait for the interrupt callback to deqeue 1655 * all pending operations. Skip waiting for queues which reported an 1656 * error using processing_status variable. 1657 * Wait for master lcore operations. 1658 */ 1659 tp = &t_params[rte_lcore_id()]; 1660 while ((rte_atomic16_read(&tp->nb_dequeued) < 1661 op_params->num_to_process) && 1662 (rte_atomic16_read(&tp->processing_status) != 1663 TEST_FAILED)) 1664 rte_pause(); 1665 1666 ret |= rte_atomic16_read(&tp->processing_status); 1667 1668 /* Wait for slave lcores operations */ 1669 used_cores = 1; 1670 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 1671 tp = &t_params[lcore_id]; 1672 if (used_cores++ >= num_lcores) 1673 break; 1674 1675 while ((rte_atomic16_read(&tp->nb_dequeued) < 1676 op_params->num_to_process) && 1677 (rte_atomic16_read(&tp->processing_status) != 1678 TEST_FAILED)) 1679 rte_pause(); 1680 1681 ret |= rte_atomic16_read(&tp->processing_status); 1682 } 1683 1684 /* Print throughput if test passed */ 1685 if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE) 1686 print_throughput(t_params, num_lcores); 1687 1688 return ret; 1689 } 1690 1691 static int 1692 operation_latency_test_dec(struct rte_mempool *mempool, 1693 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 1694 int vector_mask, uint16_t dev_id, uint16_t queue_id, 1695 const uint16_t num_to_process, uint16_t burst_sz, 1696 uint64_t *total_time) 1697 { 1698 int ret = TEST_SUCCESS; 1699 uint16_t i, j, dequeued; 1700 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1701 uint64_t start_time = 0; 1702 1703 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1704 uint16_t enq = 0, deq = 0; 1705 bool first_time = true; 1706 1707 if (unlikely(num_to_process - dequeued < burst_sz)) 1708 burst_sz = num_to_process - dequeued; 1709 1710 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1711 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1712 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1713 bufs->inputs, 1714 bufs->hard_outputs, 1715 bufs->soft_outputs, 1716 ref_op); 1717 1718 /* Set counter to validate the ordering */ 1719 for (j = 0; j < burst_sz; ++j) 1720 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1721 1722 start_time = rte_rdtsc_precise(); 1723 1724 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 1725 burst_sz); 1726 TEST_ASSERT(enq == burst_sz, 1727 "Error enqueueing burst, expected %u, got %u", 1728 burst_sz, enq); 1729 1730 /* Dequeue */ 1731 do { 1732 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1733 &ops_deq[deq], burst_sz - deq); 1734 if (likely(first_time && (deq > 0))) { 1735 *total_time += rte_rdtsc_precise() - start_time; 1736 first_time = false; 1737 } 1738 } while (unlikely(burst_sz != deq)); 1739 1740 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1741 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 1742 vector_mask); 1743 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1744 } 1745 1746 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1747 dequeued += deq; 1748 } 1749 1750 return i; 1751 } 1752 1753 static int 1754 operation_latency_test_enc(struct rte_mempool *mempool, 1755 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 1756 uint16_t dev_id, uint16_t queue_id, 1757 const uint16_t num_to_process, uint16_t burst_sz, 1758 uint64_t *total_time) 1759 { 1760 int ret = TEST_SUCCESS; 1761 uint16_t i, j, dequeued; 1762 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1763 uint64_t start_time = 0; 1764 1765 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1766 uint16_t enq = 0, deq = 0; 1767 bool first_time = true; 1768 1769 if (unlikely(num_to_process - dequeued < burst_sz)) 1770 burst_sz = num_to_process - dequeued; 1771 1772 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1773 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1774 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1775 bufs->inputs, 1776 bufs->hard_outputs, 1777 ref_op); 1778 1779 /* Set counter to validate the ordering */ 1780 for (j = 0; j < burst_sz; ++j) 1781 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 1782 1783 start_time = rte_rdtsc_precise(); 1784 1785 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 1786 burst_sz); 1787 TEST_ASSERT(enq == burst_sz, 1788 "Error enqueueing burst, expected %u, got %u", 1789 burst_sz, enq); 1790 1791 /* Dequeue */ 1792 do { 1793 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1794 &ops_deq[deq], burst_sz - deq); 1795 if (likely(first_time && (deq > 0))) { 1796 *total_time += rte_rdtsc_precise() - start_time; 1797 first_time = false; 1798 } 1799 } while (unlikely(burst_sz != deq)); 1800 1801 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 1802 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 1803 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 1804 } 1805 1806 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1807 dequeued += deq; 1808 } 1809 1810 return i; 1811 } 1812 1813 static int 1814 operation_latency_test(struct active_device *ad, 1815 struct test_op_params *op_params) 1816 { 1817 int iter; 1818 uint16_t burst_sz = op_params->burst_sz; 1819 const uint16_t num_to_process = op_params->num_to_process; 1820 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1821 const uint16_t queue_id = ad->queue_ids[0]; 1822 struct test_buffers *bufs = NULL; 1823 struct rte_bbdev_info info; 1824 uint64_t total_time = 0; 1825 const char *op_type_str; 1826 1827 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1828 "BURST_SIZE should be <= %u", MAX_BURST); 1829 1830 rte_bbdev_info_get(ad->dev_id, &info); 1831 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1832 1833 op_type_str = rte_bbdev_op_type_str(op_type); 1834 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1835 1836 printf( 1837 "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1838 info.dev_name, burst_sz, num_to_process, op_type_str); 1839 1840 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1841 iter = operation_latency_test_dec(op_params->mp, bufs, 1842 op_params->ref_dec_op, op_params->vector_mask, 1843 ad->dev_id, queue_id, num_to_process, 1844 burst_sz, &total_time); 1845 else 1846 iter = operation_latency_test_enc(op_params->mp, bufs, 1847 op_params->ref_enc_op, ad->dev_id, queue_id, 1848 num_to_process, burst_sz, &total_time); 1849 1850 if (iter <= 0) 1851 return TEST_FAILED; 1852 1853 printf("\toperation avg. latency: %lg cycles, %lg us\n", 1854 (double)total_time / (double)iter, 1855 (double)(total_time * 1000000) / (double)iter / 1856 (double)rte_get_tsc_hz()); 1857 1858 return TEST_SUCCESS; 1859 } 1860 1861 static int 1862 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 1863 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 1864 uint16_t queue_id, const uint16_t num_to_process, 1865 uint16_t burst_sz, uint64_t *enq_total_time, 1866 uint64_t *deq_total_time) 1867 { 1868 int i, dequeued; 1869 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1870 uint64_t enq_start_time, deq_start_time; 1871 1872 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1873 uint16_t enq = 0, deq = 0; 1874 1875 if (unlikely(num_to_process - dequeued < burst_sz)) 1876 burst_sz = num_to_process - dequeued; 1877 1878 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 1879 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1880 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 1881 bufs->inputs, 1882 bufs->hard_outputs, 1883 bufs->soft_outputs, 1884 ref_op); 1885 1886 /* Start time measurment for enqueue function offload latency */ 1887 enq_start_time = rte_rdtsc(); 1888 do { 1889 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 1890 &ops_enq[enq], burst_sz - enq); 1891 } while (unlikely(burst_sz != enq)); 1892 *enq_total_time += rte_rdtsc() - enq_start_time; 1893 1894 /* ensure enqueue has been completed */ 1895 rte_delay_ms(10); 1896 1897 /* Start time measurment for dequeue function offload latency */ 1898 deq_start_time = rte_rdtsc(); 1899 do { 1900 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1901 &ops_deq[deq], burst_sz - deq); 1902 } while (unlikely(burst_sz != deq)); 1903 *deq_total_time += rte_rdtsc() - deq_start_time; 1904 1905 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 1906 dequeued += deq; 1907 } 1908 1909 return i; 1910 } 1911 1912 static int 1913 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 1914 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 1915 uint16_t queue_id, const uint16_t num_to_process, 1916 uint16_t burst_sz, uint64_t *enq_total_time, 1917 uint64_t *deq_total_time) 1918 { 1919 int i, dequeued; 1920 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 1921 uint64_t enq_start_time, deq_start_time; 1922 1923 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 1924 uint16_t enq = 0, deq = 0; 1925 1926 if (unlikely(num_to_process - dequeued < burst_sz)) 1927 burst_sz = num_to_process - dequeued; 1928 1929 rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 1930 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1931 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 1932 bufs->inputs, 1933 bufs->hard_outputs, 1934 ref_op); 1935 1936 /* Start time measurment for enqueue function offload latency */ 1937 enq_start_time = rte_rdtsc(); 1938 do { 1939 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 1940 &ops_enq[enq], burst_sz - enq); 1941 } while (unlikely(burst_sz != enq)); 1942 *enq_total_time += rte_rdtsc() - enq_start_time; 1943 1944 /* ensure enqueue has been completed */ 1945 rte_delay_ms(10); 1946 1947 /* Start time measurment for dequeue function offload latency */ 1948 deq_start_time = rte_rdtsc(); 1949 do { 1950 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1951 &ops_deq[deq], burst_sz - deq); 1952 } while (unlikely(burst_sz != deq)); 1953 *deq_total_time += rte_rdtsc() - deq_start_time; 1954 1955 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 1956 dequeued += deq; 1957 } 1958 1959 return i; 1960 } 1961 1962 static int 1963 offload_latency_test(struct active_device *ad, 1964 struct test_op_params *op_params) 1965 { 1966 int iter; 1967 uint64_t enq_total_time = 0, deq_total_time = 0; 1968 uint16_t burst_sz = op_params->burst_sz; 1969 const uint16_t num_to_process = op_params->num_to_process; 1970 const enum rte_bbdev_op_type op_type = test_vector.op_type; 1971 const uint16_t queue_id = ad->queue_ids[0]; 1972 struct test_buffers *bufs = NULL; 1973 struct rte_bbdev_info info; 1974 const char *op_type_str; 1975 1976 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1977 "BURST_SIZE should be <= %u", MAX_BURST); 1978 1979 rte_bbdev_info_get(ad->dev_id, &info); 1980 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1981 1982 op_type_str = rte_bbdev_op_type_str(op_type); 1983 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 1984 1985 printf( 1986 "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 1987 info.dev_name, burst_sz, num_to_process, op_type_str); 1988 1989 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 1990 iter = offload_latency_test_dec(op_params->mp, bufs, 1991 op_params->ref_dec_op, ad->dev_id, queue_id, 1992 num_to_process, burst_sz, &enq_total_time, 1993 &deq_total_time); 1994 else 1995 iter = offload_latency_test_enc(op_params->mp, bufs, 1996 op_params->ref_enc_op, ad->dev_id, queue_id, 1997 num_to_process, burst_sz, &enq_total_time, 1998 &deq_total_time); 1999 2000 if (iter <= 0) 2001 return TEST_FAILED; 2002 2003 printf("\tenq offload avg. latency: %lg cycles, %lg us\n", 2004 (double)enq_total_time / (double)iter, 2005 (double)(enq_total_time * 1000000) / (double)iter / 2006 (double)rte_get_tsc_hz()); 2007 2008 printf("\tdeq offload avg. latency: %lg cycles, %lg us\n", 2009 (double)deq_total_time / (double)iter, 2010 (double)(deq_total_time * 1000000) / (double)iter / 2011 (double)rte_get_tsc_hz()); 2012 2013 return TEST_SUCCESS; 2014 } 2015 2016 static int 2017 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 2018 const uint16_t num_to_process, uint16_t burst_sz, 2019 uint64_t *deq_total_time) 2020 { 2021 int i, deq_total; 2022 struct rte_bbdev_dec_op *ops[MAX_BURST]; 2023 uint64_t deq_start_time; 2024 2025 /* Test deq offload latency from an empty queue */ 2026 deq_start_time = rte_rdtsc_precise(); 2027 for (i = 0, deq_total = 0; deq_total < num_to_process; 2028 ++i, deq_total += burst_sz) { 2029 if (unlikely(num_to_process - deq_total < burst_sz)) 2030 burst_sz = num_to_process - deq_total; 2031 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 2032 } 2033 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 2034 2035 return i; 2036 } 2037 2038 static int 2039 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 2040 const uint16_t num_to_process, uint16_t burst_sz, 2041 uint64_t *deq_total_time) 2042 { 2043 int i, deq_total; 2044 struct rte_bbdev_enc_op *ops[MAX_BURST]; 2045 uint64_t deq_start_time; 2046 2047 /* Test deq offload latency from an empty queue */ 2048 deq_start_time = rte_rdtsc_precise(); 2049 for (i = 0, deq_total = 0; deq_total < num_to_process; 2050 ++i, deq_total += burst_sz) { 2051 if (unlikely(num_to_process - deq_total < burst_sz)) 2052 burst_sz = num_to_process - deq_total; 2053 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 2054 } 2055 *deq_total_time = rte_rdtsc_precise() - deq_start_time; 2056 2057 return i; 2058 } 2059 2060 static int 2061 offload_latency_empty_q_test(struct active_device *ad, 2062 struct test_op_params *op_params) 2063 { 2064 int iter; 2065 uint64_t deq_total_time = 0; 2066 uint16_t burst_sz = op_params->burst_sz; 2067 const uint16_t num_to_process = op_params->num_to_process; 2068 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2069 const uint16_t queue_id = ad->queue_ids[0]; 2070 struct rte_bbdev_info info; 2071 const char *op_type_str; 2072 2073 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2074 "BURST_SIZE should be <= %u", MAX_BURST); 2075 2076 rte_bbdev_info_get(ad->dev_id, &info); 2077 2078 op_type_str = rte_bbdev_op_type_str(op_type); 2079 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2080 2081 printf( 2082 "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", 2083 info.dev_name, burst_sz, num_to_process, op_type_str); 2084 2085 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2086 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 2087 num_to_process, burst_sz, &deq_total_time); 2088 else 2089 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 2090 num_to_process, burst_sz, &deq_total_time); 2091 2092 if (iter <= 0) 2093 return TEST_FAILED; 2094 2095 printf("\tempty deq offload avg. latency: %lg cycles, %lg us\n", 2096 (double)deq_total_time / (double)iter, 2097 (double)(deq_total_time * 1000000) / (double)iter / 2098 (double)rte_get_tsc_hz()); 2099 2100 return TEST_SUCCESS; 2101 } 2102 2103 static int 2104 throughput_tc(void) 2105 { 2106 return run_test_case(throughput_test); 2107 } 2108 2109 static int 2110 offload_latency_tc(void) 2111 { 2112 return run_test_case(offload_latency_test); 2113 } 2114 2115 static int 2116 offload_latency_empty_q_tc(void) 2117 { 2118 return run_test_case(offload_latency_empty_q_test); 2119 } 2120 2121 static int 2122 operation_latency_tc(void) 2123 { 2124 return run_test_case(operation_latency_test); 2125 } 2126 2127 static int 2128 interrupt_tc(void) 2129 { 2130 return run_test_case(throughput_test); 2131 } 2132 2133 static struct unit_test_suite bbdev_throughput_testsuite = { 2134 .suite_name = "BBdev Throughput Tests", 2135 .setup = testsuite_setup, 2136 .teardown = testsuite_teardown, 2137 .unit_test_cases = { 2138 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 2139 TEST_CASES_END() /**< NULL terminate unit test array */ 2140 } 2141 }; 2142 2143 static struct unit_test_suite bbdev_validation_testsuite = { 2144 .suite_name = "BBdev Validation Tests", 2145 .setup = testsuite_setup, 2146 .teardown = testsuite_teardown, 2147 .unit_test_cases = { 2148 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2149 TEST_CASES_END() /**< NULL terminate unit test array */ 2150 } 2151 }; 2152 2153 static struct unit_test_suite bbdev_latency_testsuite = { 2154 .suite_name = "BBdev Latency Tests", 2155 .setup = testsuite_setup, 2156 .teardown = testsuite_teardown, 2157 .unit_test_cases = { 2158 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_tc), 2159 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 2160 TEST_CASE_ST(ut_setup, ut_teardown, operation_latency_tc), 2161 TEST_CASES_END() /**< NULL terminate unit test array */ 2162 } 2163 }; 2164 2165 static struct unit_test_suite bbdev_interrupt_testsuite = { 2166 .suite_name = "BBdev Interrupt Tests", 2167 .setup = interrupt_testsuite_setup, 2168 .teardown = testsuite_teardown, 2169 .unit_test_cases = { 2170 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 2171 TEST_CASES_END() /**< NULL terminate unit test array */ 2172 } 2173 }; 2174 2175 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 2176 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 2177 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 2178 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 2179