1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 22 #include <fpga_lte_fec.h> 23 #endif 24 25 #include "main.h" 26 #include "test_bbdev_vector.h" 27 28 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 29 30 #define MAX_QUEUES RTE_MAX_LCORE 31 #define TEST_REPETITIONS 1000 32 33 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 34 #define FPGA_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 35 #define FPGA_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 36 #define VF_UL_QUEUE_VALUE 4 37 #define VF_DL_QUEUE_VALUE 4 38 #define UL_BANDWIDTH 3 39 #define DL_BANDWIDTH 3 40 #define UL_LOAD_BALANCE 128 41 #define DL_LOAD_BALANCE 128 42 #define FLR_TIMEOUT 610 43 #endif 44 45 #define OPS_CACHE_SIZE 256U 46 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 47 48 #define SYNC_WAIT 0 49 #define SYNC_START 1 50 51 #define INVALID_QUEUE_ID -1 52 53 static struct test_bbdev_vector test_vector; 54 55 /* Switch between PMD and Interrupt for throughput TC */ 56 static bool intr_enabled; 57 58 /* Represents tested active devices */ 59 static struct active_device { 60 const char *driver_name; 61 uint8_t dev_id; 62 uint16_t supported_ops; 63 uint16_t queue_ids[MAX_QUEUES]; 64 uint16_t nb_queues; 65 struct rte_mempool *ops_mempool; 66 struct rte_mempool *in_mbuf_pool; 67 struct rte_mempool *hard_out_mbuf_pool; 68 struct rte_mempool *soft_out_mbuf_pool; 69 struct rte_mempool *harq_in_mbuf_pool; 70 struct rte_mempool *harq_out_mbuf_pool; 71 } active_devs[RTE_BBDEV_MAX_DEVS]; 72 73 static uint8_t nb_active_devs; 74 75 /* Data buffers used by BBDEV ops */ 76 struct test_buffers { 77 struct rte_bbdev_op_data *inputs; 78 struct rte_bbdev_op_data *hard_outputs; 79 struct rte_bbdev_op_data *soft_outputs; 80 struct rte_bbdev_op_data *harq_inputs; 81 struct rte_bbdev_op_data *harq_outputs; 82 }; 83 84 /* Operation parameters specific for given test case */ 85 struct test_op_params { 86 struct rte_mempool *mp; 87 struct rte_bbdev_dec_op *ref_dec_op; 88 struct rte_bbdev_enc_op *ref_enc_op; 89 uint16_t burst_sz; 90 uint16_t num_to_process; 91 uint16_t num_lcores; 92 int vector_mask; 93 rte_atomic16_t sync; 94 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 95 }; 96 97 /* Contains per lcore params */ 98 struct thread_params { 99 uint8_t dev_id; 100 uint16_t queue_id; 101 uint32_t lcore_id; 102 uint64_t start_time; 103 double ops_per_sec; 104 double mbps; 105 uint8_t iter_count; 106 rte_atomic16_t nb_dequeued; 107 rte_atomic16_t processing_status; 108 rte_atomic16_t burst_sz; 109 struct test_op_params *op_params; 110 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 111 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 112 }; 113 114 #ifdef RTE_BBDEV_OFFLOAD_COST 115 /* Stores time statistics */ 116 struct test_time_stats { 117 /* Stores software enqueue total working time */ 118 uint64_t enq_sw_total_time; 119 /* Stores minimum value of software enqueue working time */ 120 uint64_t enq_sw_min_time; 121 /* Stores maximum value of software enqueue working time */ 122 uint64_t enq_sw_max_time; 123 /* Stores turbo enqueue total working time */ 124 uint64_t enq_acc_total_time; 125 /* Stores minimum value of accelerator enqueue working time */ 126 uint64_t enq_acc_min_time; 127 /* Stores maximum value of accelerator enqueue working time */ 128 uint64_t enq_acc_max_time; 129 /* Stores dequeue total working time */ 130 uint64_t deq_total_time; 131 /* Stores minimum value of dequeue working time */ 132 uint64_t deq_min_time; 133 /* Stores maximum value of dequeue working time */ 134 uint64_t deq_max_time; 135 }; 136 #endif 137 138 typedef int (test_case_function)(struct active_device *ad, 139 struct test_op_params *op_params); 140 141 static inline void 142 mbuf_reset(struct rte_mbuf *m) 143 { 144 m->pkt_len = 0; 145 146 do { 147 m->data_len = 0; 148 m = m->next; 149 } while (m != NULL); 150 } 151 152 /* Read flag value 0/1 from bitmap */ 153 static inline bool 154 check_bit(uint32_t bitmap, uint32_t bitmask) 155 { 156 return bitmap & bitmask; 157 } 158 159 static inline void 160 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 161 { 162 ad->supported_ops |= (1 << op_type); 163 } 164 165 static inline bool 166 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 167 { 168 return ad->supported_ops & (1 << op_type); 169 } 170 171 static inline bool 172 flags_match(uint32_t flags_req, uint32_t flags_present) 173 { 174 return (flags_req & flags_present) == flags_req; 175 } 176 177 static void 178 clear_soft_out_cap(uint32_t *op_flags) 179 { 180 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 181 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 182 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 183 } 184 185 static int 186 check_dev_cap(const struct rte_bbdev_info *dev_info) 187 { 188 unsigned int i; 189 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 190 nb_harq_inputs, nb_harq_outputs; 191 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 192 193 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 194 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 195 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 196 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 197 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 198 199 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 200 if (op_cap->type != test_vector.op_type) 201 continue; 202 203 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 204 const struct rte_bbdev_op_cap_turbo_dec *cap = 205 &op_cap->cap.turbo_dec; 206 /* Ignore lack of soft output capability, just skip 207 * checking if soft output is valid. 208 */ 209 if ((test_vector.turbo_dec.op_flags & 210 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 211 !(cap->capability_flags & 212 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 213 printf( 214 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 215 dev_info->dev_name); 216 clear_soft_out_cap( 217 &test_vector.turbo_dec.op_flags); 218 } 219 220 if (!flags_match(test_vector.turbo_dec.op_flags, 221 cap->capability_flags)) 222 return TEST_FAILED; 223 if (nb_inputs > cap->num_buffers_src) { 224 printf("Too many inputs defined: %u, max: %u\n", 225 nb_inputs, cap->num_buffers_src); 226 return TEST_FAILED; 227 } 228 if (nb_soft_outputs > cap->num_buffers_soft_out && 229 (test_vector.turbo_dec.op_flags & 230 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 231 printf( 232 "Too many soft outputs defined: %u, max: %u\n", 233 nb_soft_outputs, 234 cap->num_buffers_soft_out); 235 return TEST_FAILED; 236 } 237 if (nb_hard_outputs > cap->num_buffers_hard_out) { 238 printf( 239 "Too many hard outputs defined: %u, max: %u\n", 240 nb_hard_outputs, 241 cap->num_buffers_hard_out); 242 return TEST_FAILED; 243 } 244 if (intr_enabled && !(cap->capability_flags & 245 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 246 printf( 247 "Dequeue interrupts are not supported!\n"); 248 return TEST_FAILED; 249 } 250 251 return TEST_SUCCESS; 252 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 253 const struct rte_bbdev_op_cap_turbo_enc *cap = 254 &op_cap->cap.turbo_enc; 255 256 if (!flags_match(test_vector.turbo_enc.op_flags, 257 cap->capability_flags)) 258 return TEST_FAILED; 259 if (nb_inputs > cap->num_buffers_src) { 260 printf("Too many inputs defined: %u, max: %u\n", 261 nb_inputs, cap->num_buffers_src); 262 return TEST_FAILED; 263 } 264 if (nb_hard_outputs > cap->num_buffers_dst) { 265 printf( 266 "Too many hard outputs defined: %u, max: %u\n", 267 nb_hard_outputs, cap->num_buffers_dst); 268 return TEST_FAILED; 269 } 270 if (intr_enabled && !(cap->capability_flags & 271 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 272 printf( 273 "Dequeue interrupts are not supported!\n"); 274 return TEST_FAILED; 275 } 276 277 return TEST_SUCCESS; 278 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 279 const struct rte_bbdev_op_cap_ldpc_enc *cap = 280 &op_cap->cap.ldpc_enc; 281 282 if (!flags_match(test_vector.ldpc_enc.op_flags, 283 cap->capability_flags)){ 284 printf("Flag Mismatch\n"); 285 return TEST_FAILED; 286 } 287 if (nb_inputs > cap->num_buffers_src) { 288 printf("Too many inputs defined: %u, max: %u\n", 289 nb_inputs, cap->num_buffers_src); 290 return TEST_FAILED; 291 } 292 if (nb_hard_outputs > cap->num_buffers_dst) { 293 printf( 294 "Too many hard outputs defined: %u, max: %u\n", 295 nb_hard_outputs, cap->num_buffers_dst); 296 return TEST_FAILED; 297 } 298 if (intr_enabled && !(cap->capability_flags & 299 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 300 printf( 301 "Dequeue interrupts are not supported!\n"); 302 return TEST_FAILED; 303 } 304 305 return TEST_SUCCESS; 306 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 307 const struct rte_bbdev_op_cap_ldpc_dec *cap = 308 &op_cap->cap.ldpc_dec; 309 310 if (!flags_match(test_vector.ldpc_dec.op_flags, 311 cap->capability_flags)){ 312 printf("Flag Mismatch\n"); 313 return TEST_FAILED; 314 } 315 if (nb_inputs > cap->num_buffers_src) { 316 printf("Too many inputs defined: %u, max: %u\n", 317 nb_inputs, cap->num_buffers_src); 318 return TEST_FAILED; 319 } 320 if (nb_hard_outputs > cap->num_buffers_hard_out) { 321 printf( 322 "Too many hard outputs defined: %u, max: %u\n", 323 nb_hard_outputs, 324 cap->num_buffers_hard_out); 325 return TEST_FAILED; 326 } 327 if (nb_harq_inputs > cap->num_buffers_hard_out) { 328 printf( 329 "Too many HARQ inputs defined: %u, max: %u\n", 330 nb_hard_outputs, 331 cap->num_buffers_hard_out); 332 return TEST_FAILED; 333 } 334 if (nb_harq_outputs > cap->num_buffers_hard_out) { 335 printf( 336 "Too many HARQ outputs defined: %u, max: %u\n", 337 nb_hard_outputs, 338 cap->num_buffers_hard_out); 339 return TEST_FAILED; 340 } 341 if (intr_enabled && !(cap->capability_flags & 342 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 343 printf( 344 "Dequeue interrupts are not supported!\n"); 345 return TEST_FAILED; 346 } 347 348 return TEST_SUCCESS; 349 } 350 } 351 352 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 353 return TEST_SUCCESS; /* Special case for NULL device */ 354 355 return TEST_FAILED; 356 } 357 358 /* calculates optimal mempool size not smaller than the val */ 359 static unsigned int 360 optimal_mempool_size(unsigned int val) 361 { 362 return rte_align32pow2(val + 1) - 1; 363 } 364 365 /* allocates mbuf mempool for inputs and outputs */ 366 static struct rte_mempool * 367 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 368 int socket_id, unsigned int mbuf_pool_size, 369 const char *op_type_str) 370 { 371 unsigned int i; 372 uint32_t max_seg_sz = 0; 373 char pool_name[RTE_MEMPOOL_NAMESIZE]; 374 375 /* find max input segment size */ 376 for (i = 0; i < entries->nb_segments; ++i) 377 if (entries->segments[i].length > max_seg_sz) 378 max_seg_sz = entries->segments[i].length; 379 380 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 381 dev_id); 382 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 383 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 384 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 385 } 386 387 static int 388 create_mempools(struct active_device *ad, int socket_id, 389 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 390 { 391 struct rte_mempool *mp; 392 unsigned int ops_pool_size, mbuf_pool_size = 0; 393 char pool_name[RTE_MEMPOOL_NAMESIZE]; 394 const char *op_type_str; 395 enum rte_bbdev_op_type op_type = org_op_type; 396 397 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 398 struct op_data_entries *hard_out = 399 &test_vector.entries[DATA_HARD_OUTPUT]; 400 struct op_data_entries *soft_out = 401 &test_vector.entries[DATA_SOFT_OUTPUT]; 402 struct op_data_entries *harq_in = 403 &test_vector.entries[DATA_HARQ_INPUT]; 404 struct op_data_entries *harq_out = 405 &test_vector.entries[DATA_HARQ_OUTPUT]; 406 407 /* allocate ops mempool */ 408 ops_pool_size = optimal_mempool_size(RTE_MAX( 409 /* Ops used plus 1 reference op */ 410 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 411 /* Minimal cache size plus 1 reference op */ 412 (unsigned int)(1.5 * rte_lcore_count() * 413 OPS_CACHE_SIZE + 1)), 414 OPS_POOL_SIZE_MIN)); 415 416 if (org_op_type == RTE_BBDEV_OP_NONE) 417 op_type = RTE_BBDEV_OP_TURBO_ENC; 418 419 op_type_str = rte_bbdev_op_type_str(op_type); 420 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 421 422 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 423 ad->dev_id); 424 mp = rte_bbdev_op_pool_create(pool_name, op_type, 425 ops_pool_size, OPS_CACHE_SIZE, socket_id); 426 TEST_ASSERT_NOT_NULL(mp, 427 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 428 ops_pool_size, 429 ad->dev_id, 430 socket_id); 431 ad->ops_mempool = mp; 432 433 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 434 if (org_op_type == RTE_BBDEV_OP_NONE) 435 return TEST_SUCCESS; 436 437 /* Inputs */ 438 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 439 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 440 TEST_ASSERT_NOT_NULL(mp, 441 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 442 mbuf_pool_size, 443 ad->dev_id, 444 socket_id); 445 ad->in_mbuf_pool = mp; 446 447 /* Hard outputs */ 448 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 449 hard_out->nb_segments); 450 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 451 "hard_out"); 452 TEST_ASSERT_NOT_NULL(mp, 453 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 454 mbuf_pool_size, 455 ad->dev_id, 456 socket_id); 457 ad->hard_out_mbuf_pool = mp; 458 459 460 /* Soft outputs */ 461 if (soft_out->nb_segments > 0) { 462 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 463 soft_out->nb_segments); 464 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 465 mbuf_pool_size, 466 "soft_out"); 467 TEST_ASSERT_NOT_NULL(mp, 468 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 469 mbuf_pool_size, 470 ad->dev_id, 471 socket_id); 472 ad->soft_out_mbuf_pool = mp; 473 } 474 475 /* HARQ inputs */ 476 if (harq_in->nb_segments > 0) { 477 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 478 harq_in->nb_segments); 479 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 480 mbuf_pool_size, 481 "harq_in"); 482 TEST_ASSERT_NOT_NULL(mp, 483 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 484 mbuf_pool_size, 485 ad->dev_id, 486 socket_id); 487 ad->harq_in_mbuf_pool = mp; 488 } 489 490 /* HARQ outputs */ 491 if (harq_out->nb_segments > 0) { 492 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 493 harq_out->nb_segments); 494 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 495 mbuf_pool_size, 496 "harq_out"); 497 TEST_ASSERT_NOT_NULL(mp, 498 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 499 mbuf_pool_size, 500 ad->dev_id, 501 socket_id); 502 ad->harq_out_mbuf_pool = mp; 503 } 504 505 return TEST_SUCCESS; 506 } 507 508 static int 509 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 510 struct test_bbdev_vector *vector) 511 { 512 int ret; 513 unsigned int queue_id; 514 struct rte_bbdev_queue_conf qconf; 515 struct active_device *ad = &active_devs[nb_active_devs]; 516 unsigned int nb_queues; 517 enum rte_bbdev_op_type op_type = vector->op_type; 518 519 /* Configure fpga lte fec with PF & VF values 520 * if '-i' flag is set and using fpga device 521 */ 522 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 523 if ((get_init_device() == true) && 524 (!strcmp(info->drv.driver_name, FPGA_PF_DRIVER_NAME))) { 525 struct fpga_lte_fec_conf conf; 526 unsigned int i; 527 528 printf("Configure FPGA FEC Driver %s with default values\n", 529 info->drv.driver_name); 530 531 /* clear default configuration before initialization */ 532 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf)); 533 534 /* Set PF mode : 535 * true if PF is used for data plane 536 * false for VFs 537 */ 538 conf.pf_mode_en = true; 539 540 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 541 /* Number of UL queues per VF (fpga supports 8 VFs) */ 542 conf.vf_ul_queues_number[i] = VF_UL_QUEUE_VALUE; 543 /* Number of DL queues per VF (fpga supports 8 VFs) */ 544 conf.vf_dl_queues_number[i] = VF_DL_QUEUE_VALUE; 545 } 546 547 /* UL bandwidth. Needed for schedule algorithm */ 548 conf.ul_bandwidth = UL_BANDWIDTH; 549 /* DL bandwidth */ 550 conf.dl_bandwidth = DL_BANDWIDTH; 551 552 /* UL & DL load Balance Factor to 64 */ 553 conf.ul_load_balance = UL_LOAD_BALANCE; 554 conf.dl_load_balance = DL_LOAD_BALANCE; 555 556 /**< FLR timeout value */ 557 conf.flr_time_out = FLR_TIMEOUT; 558 559 /* setup FPGA PF with configuration information */ 560 ret = fpga_lte_fec_configure(info->dev_name, &conf); 561 TEST_ASSERT_SUCCESS(ret, 562 "Failed to configure 4G FPGA PF for bbdev %s", 563 info->dev_name); 564 } 565 #endif 566 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 567 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 568 569 /* setup device */ 570 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 571 if (ret < 0) { 572 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 573 dev_id, nb_queues, info->socket_id, ret); 574 return TEST_FAILED; 575 } 576 577 /* configure interrupts if needed */ 578 if (intr_enabled) { 579 ret = rte_bbdev_intr_enable(dev_id); 580 if (ret < 0) { 581 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 582 ret); 583 return TEST_FAILED; 584 } 585 } 586 587 /* setup device queues */ 588 qconf.socket = info->socket_id; 589 qconf.queue_size = info->drv.default_queue_conf.queue_size; 590 qconf.priority = 0; 591 qconf.deferred_start = 0; 592 qconf.op_type = op_type; 593 594 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 595 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 596 if (ret != 0) { 597 printf( 598 "Allocated all queues (id=%u) at prio%u on dev%u\n", 599 queue_id, qconf.priority, dev_id); 600 qconf.priority++; 601 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 602 &qconf); 603 } 604 if (ret != 0) { 605 printf("All queues on dev %u allocated: %u\n", 606 dev_id, queue_id); 607 break; 608 } 609 ad->queue_ids[queue_id] = queue_id; 610 } 611 TEST_ASSERT(queue_id != 0, 612 "ERROR Failed to configure any queues on dev %u", 613 dev_id); 614 ad->nb_queues = queue_id; 615 616 set_avail_op(ad, op_type); 617 618 return TEST_SUCCESS; 619 } 620 621 static int 622 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 623 struct test_bbdev_vector *vector) 624 { 625 int ret; 626 627 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 628 active_devs[nb_active_devs].dev_id = dev_id; 629 630 ret = add_bbdev_dev(dev_id, info, vector); 631 if (ret == TEST_SUCCESS) 632 ++nb_active_devs; 633 return ret; 634 } 635 636 static uint8_t 637 populate_active_devices(void) 638 { 639 int ret; 640 uint8_t dev_id; 641 uint8_t nb_devs_added = 0; 642 struct rte_bbdev_info info; 643 644 RTE_BBDEV_FOREACH(dev_id) { 645 rte_bbdev_info_get(dev_id, &info); 646 647 if (check_dev_cap(&info)) { 648 printf( 649 "Device %d (%s) does not support specified capabilities\n", 650 dev_id, info.dev_name); 651 continue; 652 } 653 654 ret = add_active_device(dev_id, &info, &test_vector); 655 if (ret != 0) { 656 printf("Adding active bbdev %s skipped\n", 657 info.dev_name); 658 continue; 659 } 660 nb_devs_added++; 661 } 662 663 return nb_devs_added; 664 } 665 666 static int 667 read_test_vector(void) 668 { 669 int ret; 670 671 memset(&test_vector, 0, sizeof(test_vector)); 672 printf("Test vector file = %s\n", get_vector_filename()); 673 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 674 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 675 get_vector_filename()); 676 677 return TEST_SUCCESS; 678 } 679 680 static int 681 testsuite_setup(void) 682 { 683 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 684 685 if (populate_active_devices() == 0) { 686 printf("No suitable devices found!\n"); 687 return TEST_SKIPPED; 688 } 689 690 return TEST_SUCCESS; 691 } 692 693 static int 694 interrupt_testsuite_setup(void) 695 { 696 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 697 698 /* Enable interrupts */ 699 intr_enabled = true; 700 701 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 702 if (populate_active_devices() == 0 || 703 test_vector.op_type == RTE_BBDEV_OP_NONE) { 704 intr_enabled = false; 705 printf("No suitable devices found!\n"); 706 return TEST_SKIPPED; 707 } 708 709 return TEST_SUCCESS; 710 } 711 712 static void 713 testsuite_teardown(void) 714 { 715 uint8_t dev_id; 716 717 /* Unconfigure devices */ 718 RTE_BBDEV_FOREACH(dev_id) 719 rte_bbdev_close(dev_id); 720 721 /* Clear active devices structs. */ 722 memset(active_devs, 0, sizeof(active_devs)); 723 nb_active_devs = 0; 724 } 725 726 static int 727 ut_setup(void) 728 { 729 uint8_t i, dev_id; 730 731 for (i = 0; i < nb_active_devs; i++) { 732 dev_id = active_devs[i].dev_id; 733 /* reset bbdev stats */ 734 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 735 "Failed to reset stats of bbdev %u", dev_id); 736 /* start the device */ 737 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 738 "Failed to start bbdev %u", dev_id); 739 } 740 741 return TEST_SUCCESS; 742 } 743 744 static void 745 ut_teardown(void) 746 { 747 uint8_t i, dev_id; 748 struct rte_bbdev_stats stats; 749 750 for (i = 0; i < nb_active_devs; i++) { 751 dev_id = active_devs[i].dev_id; 752 /* read stats and print */ 753 rte_bbdev_stats_get(dev_id, &stats); 754 /* Stop the device */ 755 rte_bbdev_stop(dev_id); 756 } 757 } 758 759 static int 760 init_op_data_objs(struct rte_bbdev_op_data *bufs, 761 struct op_data_entries *ref_entries, 762 struct rte_mempool *mbuf_pool, const uint16_t n, 763 enum op_data_type op_type, uint16_t min_alignment) 764 { 765 int ret; 766 unsigned int i, j; 767 bool large_input = false; 768 769 for (i = 0; i < n; ++i) { 770 char *data; 771 struct op_data_buf *seg = &ref_entries->segments[0]; 772 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 773 TEST_ASSERT_NOT_NULL(m_head, 774 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 775 op_type, n * ref_entries->nb_segments, 776 mbuf_pool->size); 777 778 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) { 779 /* 780 * Special case when DPDK mbuf cannot handle 781 * the required input size 782 */ 783 printf("Warning: Larger input size than DPDK mbuf %d\n", 784 seg->length); 785 large_input = true; 786 } 787 bufs[i].data = m_head; 788 bufs[i].offset = 0; 789 bufs[i].length = 0; 790 791 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 792 if ((op_type == DATA_INPUT) && large_input) { 793 /* Allocate a fake overused mbuf */ 794 data = rte_malloc(NULL, seg->length, 0); 795 memcpy(data, seg->addr, seg->length); 796 m_head->buf_addr = data; 797 m_head->buf_iova = rte_malloc_virt2iova(data); 798 m_head->data_off = 0; 799 m_head->data_len = seg->length; 800 } else { 801 data = rte_pktmbuf_append(m_head, seg->length); 802 TEST_ASSERT_NOT_NULL(data, 803 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 804 seg->length, op_type); 805 806 TEST_ASSERT(data == RTE_PTR_ALIGN( 807 data, min_alignment), 808 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 809 data, min_alignment); 810 rte_memcpy(data, seg->addr, seg->length); 811 } 812 813 bufs[i].length += seg->length; 814 815 for (j = 1; j < ref_entries->nb_segments; ++j) { 816 struct rte_mbuf *m_tail = 817 rte_pktmbuf_alloc(mbuf_pool); 818 TEST_ASSERT_NOT_NULL(m_tail, 819 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 820 op_type, 821 n * ref_entries->nb_segments, 822 mbuf_pool->size); 823 seg += 1; 824 825 data = rte_pktmbuf_append(m_tail, seg->length); 826 TEST_ASSERT_NOT_NULL(data, 827 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 828 seg->length, op_type); 829 830 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 831 min_alignment), 832 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 833 data, min_alignment); 834 rte_memcpy(data, seg->addr, seg->length); 835 bufs[i].length += seg->length; 836 837 ret = rte_pktmbuf_chain(m_head, m_tail); 838 TEST_ASSERT_SUCCESS(ret, 839 "Couldn't chain mbufs from %d data type mbuf pool", 840 op_type); 841 } 842 } else { 843 844 /* allocate chained-mbuf for output buffer */ 845 for (j = 1; j < ref_entries->nb_segments; ++j) { 846 struct rte_mbuf *m_tail = 847 rte_pktmbuf_alloc(mbuf_pool); 848 TEST_ASSERT_NOT_NULL(m_tail, 849 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 850 op_type, 851 n * ref_entries->nb_segments, 852 mbuf_pool->size); 853 854 ret = rte_pktmbuf_chain(m_head, m_tail); 855 TEST_ASSERT_SUCCESS(ret, 856 "Couldn't chain mbufs from %d data type mbuf pool", 857 op_type); 858 } 859 } 860 } 861 862 return 0; 863 } 864 865 static int 866 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 867 const int socket) 868 { 869 int i; 870 871 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 872 if (*buffers == NULL) { 873 printf("WARNING: Failed to allocate op_data on socket %d\n", 874 socket); 875 /* try to allocate memory on other detected sockets */ 876 for (i = 0; i < socket; i++) { 877 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 878 if (*buffers != NULL) 879 break; 880 } 881 } 882 883 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 884 } 885 886 static void 887 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 888 const uint16_t n, const int8_t max_llr_modulus) 889 { 890 uint16_t i, byte_idx; 891 892 for (i = 0; i < n; ++i) { 893 struct rte_mbuf *m = input_ops[i].data; 894 while (m != NULL) { 895 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 896 input_ops[i].offset); 897 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 898 ++byte_idx) 899 llr[byte_idx] = round((double)max_llr_modulus * 900 llr[byte_idx] / INT8_MAX); 901 902 m = m->next; 903 } 904 } 905 } 906 907 static void 908 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 909 const uint16_t n, const int8_t llr_size, 910 const int8_t llr_decimals) 911 { 912 if (input_ops == NULL) 913 return; 914 915 uint16_t i, byte_idx; 916 917 int16_t llr_max, llr_min, llr_tmp; 918 llr_max = (1 << (llr_size - 1)) - 1; 919 llr_min = -llr_max; 920 for (i = 0; i < n; ++i) { 921 struct rte_mbuf *m = input_ops[i].data; 922 while (m != NULL) { 923 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 924 input_ops[i].offset); 925 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 926 ++byte_idx) { 927 928 llr_tmp = llr[byte_idx]; 929 if (llr_decimals == 2) 930 llr_tmp *= 2; 931 else if (llr_decimals == 0) 932 llr_tmp /= 2; 933 llr_tmp = RTE_MIN(llr_max, 934 RTE_MAX(llr_min, llr_tmp)); 935 llr[byte_idx] = (int8_t) llr_tmp; 936 } 937 938 m = m->next; 939 } 940 } 941 } 942 943 944 945 static int 946 fill_queue_buffers(struct test_op_params *op_params, 947 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 948 struct rte_mempool *soft_out_mp, 949 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 950 uint16_t queue_id, 951 const struct rte_bbdev_op_cap *capabilities, 952 uint16_t min_alignment, const int socket_id) 953 { 954 int ret; 955 enum op_data_type type; 956 const uint16_t n = op_params->num_to_process; 957 958 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 959 in_mp, 960 soft_out_mp, 961 hard_out_mp, 962 harq_in_mp, 963 harq_out_mp, 964 }; 965 966 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 967 &op_params->q_bufs[socket_id][queue_id].inputs, 968 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 969 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 970 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 971 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 972 }; 973 974 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 975 struct op_data_entries *ref_entries = 976 &test_vector.entries[type]; 977 if (ref_entries->nb_segments == 0) 978 continue; 979 980 ret = allocate_buffers_on_socket(queue_ops[type], 981 n * sizeof(struct rte_bbdev_op_data), 982 socket_id); 983 TEST_ASSERT_SUCCESS(ret, 984 "Couldn't allocate memory for rte_bbdev_op_data structs"); 985 986 ret = init_op_data_objs(*queue_ops[type], ref_entries, 987 mbuf_pools[type], n, type, min_alignment); 988 TEST_ASSERT_SUCCESS(ret, 989 "Couldn't init rte_bbdev_op_data structs"); 990 } 991 992 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 993 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 994 capabilities->cap.turbo_dec.max_llr_modulus); 995 996 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 997 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 998 capabilities->cap.ldpc_dec.llr_size, 999 capabilities->cap.ldpc_dec.llr_decimals); 1000 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1001 capabilities->cap.ldpc_dec.llr_size, 1002 capabilities->cap.ldpc_dec.llr_decimals); 1003 } 1004 1005 return 0; 1006 } 1007 1008 static void 1009 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1010 { 1011 unsigned int i, j; 1012 1013 rte_mempool_free(ad->ops_mempool); 1014 rte_mempool_free(ad->in_mbuf_pool); 1015 rte_mempool_free(ad->hard_out_mbuf_pool); 1016 rte_mempool_free(ad->soft_out_mbuf_pool); 1017 rte_mempool_free(ad->harq_in_mbuf_pool); 1018 rte_mempool_free(ad->harq_out_mbuf_pool); 1019 1020 for (i = 0; i < rte_lcore_count(); ++i) { 1021 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1022 rte_free(op_params->q_bufs[j][i].inputs); 1023 rte_free(op_params->q_bufs[j][i].hard_outputs); 1024 rte_free(op_params->q_bufs[j][i].soft_outputs); 1025 rte_free(op_params->q_bufs[j][i].harq_inputs); 1026 rte_free(op_params->q_bufs[j][i].harq_outputs); 1027 } 1028 } 1029 } 1030 1031 static void 1032 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1033 unsigned int start_idx, 1034 struct rte_bbdev_op_data *inputs, 1035 struct rte_bbdev_op_data *hard_outputs, 1036 struct rte_bbdev_op_data *soft_outputs, 1037 struct rte_bbdev_dec_op *ref_op) 1038 { 1039 unsigned int i; 1040 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1041 1042 for (i = 0; i < n; ++i) { 1043 if (turbo_dec->code_block_mode == 0) { 1044 ops[i]->turbo_dec.tb_params.ea = 1045 turbo_dec->tb_params.ea; 1046 ops[i]->turbo_dec.tb_params.eb = 1047 turbo_dec->tb_params.eb; 1048 ops[i]->turbo_dec.tb_params.k_pos = 1049 turbo_dec->tb_params.k_pos; 1050 ops[i]->turbo_dec.tb_params.k_neg = 1051 turbo_dec->tb_params.k_neg; 1052 ops[i]->turbo_dec.tb_params.c = 1053 turbo_dec->tb_params.c; 1054 ops[i]->turbo_dec.tb_params.c_neg = 1055 turbo_dec->tb_params.c_neg; 1056 ops[i]->turbo_dec.tb_params.cab = 1057 turbo_dec->tb_params.cab; 1058 ops[i]->turbo_dec.tb_params.r = 1059 turbo_dec->tb_params.r; 1060 } else { 1061 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1062 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1063 } 1064 1065 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1066 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1067 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1068 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1069 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1070 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1071 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1072 1073 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1074 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1075 if (soft_outputs != NULL) 1076 ops[i]->turbo_dec.soft_output = 1077 soft_outputs[start_idx + i]; 1078 } 1079 } 1080 1081 static void 1082 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1083 unsigned int start_idx, 1084 struct rte_bbdev_op_data *inputs, 1085 struct rte_bbdev_op_data *outputs, 1086 struct rte_bbdev_enc_op *ref_op) 1087 { 1088 unsigned int i; 1089 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1090 for (i = 0; i < n; ++i) { 1091 if (turbo_enc->code_block_mode == 0) { 1092 ops[i]->turbo_enc.tb_params.ea = 1093 turbo_enc->tb_params.ea; 1094 ops[i]->turbo_enc.tb_params.eb = 1095 turbo_enc->tb_params.eb; 1096 ops[i]->turbo_enc.tb_params.k_pos = 1097 turbo_enc->tb_params.k_pos; 1098 ops[i]->turbo_enc.tb_params.k_neg = 1099 turbo_enc->tb_params.k_neg; 1100 ops[i]->turbo_enc.tb_params.c = 1101 turbo_enc->tb_params.c; 1102 ops[i]->turbo_enc.tb_params.c_neg = 1103 turbo_enc->tb_params.c_neg; 1104 ops[i]->turbo_enc.tb_params.cab = 1105 turbo_enc->tb_params.cab; 1106 ops[i]->turbo_enc.tb_params.ncb_pos = 1107 turbo_enc->tb_params.ncb_pos; 1108 ops[i]->turbo_enc.tb_params.ncb_neg = 1109 turbo_enc->tb_params.ncb_neg; 1110 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1111 } else { 1112 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1113 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1114 ops[i]->turbo_enc.cb_params.ncb = 1115 turbo_enc->cb_params.ncb; 1116 } 1117 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1118 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1119 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1120 1121 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1122 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1123 } 1124 } 1125 1126 static void 1127 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1128 unsigned int start_idx, 1129 struct rte_bbdev_op_data *inputs, 1130 struct rte_bbdev_op_data *hard_outputs, 1131 struct rte_bbdev_op_data *soft_outputs, 1132 struct rte_bbdev_op_data *harq_inputs, 1133 struct rte_bbdev_op_data *harq_outputs, 1134 struct rte_bbdev_dec_op *ref_op) 1135 { 1136 unsigned int i; 1137 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1138 1139 for (i = 0; i < n; ++i) { 1140 if (ldpc_dec->code_block_mode == 0) { 1141 ops[i]->ldpc_dec.tb_params.ea = 1142 ldpc_dec->tb_params.ea; 1143 ops[i]->ldpc_dec.tb_params.eb = 1144 ldpc_dec->tb_params.eb; 1145 ops[i]->ldpc_dec.tb_params.c = 1146 ldpc_dec->tb_params.c; 1147 ops[i]->ldpc_dec.tb_params.cab = 1148 ldpc_dec->tb_params.cab; 1149 ops[i]->ldpc_dec.tb_params.r = 1150 ldpc_dec->tb_params.r; 1151 } else { 1152 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1153 } 1154 1155 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1156 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1157 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1158 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1159 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1160 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1161 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1162 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1163 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1164 1165 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i]; 1166 ops[i]->ldpc_dec.input = inputs[start_idx + i]; 1167 if (soft_outputs != NULL) 1168 ops[i]->ldpc_dec.soft_output = 1169 soft_outputs[start_idx + i]; 1170 if (harq_inputs != NULL) 1171 ops[i]->ldpc_dec.harq_combined_input = 1172 harq_inputs[start_idx + i]; 1173 if (harq_outputs != NULL) 1174 ops[i]->ldpc_dec.harq_combined_output = 1175 harq_outputs[start_idx + i]; 1176 } 1177 } 1178 1179 1180 static void 1181 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1182 unsigned int start_idx, 1183 struct rte_bbdev_op_data *inputs, 1184 struct rte_bbdev_op_data *outputs, 1185 struct rte_bbdev_enc_op *ref_op) 1186 { 1187 unsigned int i; 1188 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1189 for (i = 0; i < n; ++i) { 1190 if (ldpc_enc->code_block_mode == 0) { 1191 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1192 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1193 ops[i]->ldpc_enc.tb_params.cab = 1194 ldpc_enc->tb_params.cab; 1195 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1196 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1197 } else { 1198 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1199 } 1200 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1201 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1202 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1203 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1204 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1205 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1206 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1207 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1208 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1209 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1210 } 1211 } 1212 1213 static int 1214 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1215 unsigned int order_idx, const int expected_status) 1216 { 1217 TEST_ASSERT(op->status == expected_status, 1218 "op_status (%d) != expected_status (%d)", 1219 op->status, expected_status); 1220 1221 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1222 "Ordering error, expected %p, got %p", 1223 (void *)(uintptr_t)order_idx, op->opaque_data); 1224 1225 return TEST_SUCCESS; 1226 } 1227 1228 static int 1229 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1230 unsigned int order_idx, const int expected_status) 1231 { 1232 TEST_ASSERT(op->status == expected_status, 1233 "op_status (%d) != expected_status (%d)", 1234 op->status, expected_status); 1235 1236 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1237 "Ordering error, expected %p, got %p", 1238 (void *)(uintptr_t)order_idx, op->opaque_data); 1239 1240 return TEST_SUCCESS; 1241 } 1242 1243 static inline int 1244 validate_op_chain(struct rte_bbdev_op_data *op, 1245 struct op_data_entries *orig_op) 1246 { 1247 uint8_t i; 1248 struct rte_mbuf *m = op->data; 1249 uint8_t nb_dst_segments = orig_op->nb_segments; 1250 uint32_t total_data_size = 0; 1251 1252 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1253 "Number of segments differ in original (%u) and filled (%u) op", 1254 nb_dst_segments, m->nb_segs); 1255 1256 /* Validate each mbuf segment length */ 1257 for (i = 0; i < nb_dst_segments; ++i) { 1258 /* Apply offset to the first mbuf segment */ 1259 uint16_t offset = (i == 0) ? op->offset : 0; 1260 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1261 total_data_size += orig_op->segments[i].length; 1262 1263 TEST_ASSERT(orig_op->segments[i].length == data_len, 1264 "Length of segment differ in original (%u) and filled (%u) op", 1265 orig_op->segments[i].length, data_len); 1266 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1267 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1268 data_len, 1269 "Output buffers (CB=%u) are not equal", i); 1270 m = m->next; 1271 } 1272 1273 /* Validate total mbuf pkt length */ 1274 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1275 TEST_ASSERT(total_data_size == pkt_len, 1276 "Length of data differ in original (%u) and filled (%u) op", 1277 total_data_size, pkt_len); 1278 1279 return TEST_SUCCESS; 1280 } 1281 1282 static int 1283 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1284 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1285 { 1286 unsigned int i; 1287 int ret; 1288 struct op_data_entries *hard_data_orig = 1289 &test_vector.entries[DATA_HARD_OUTPUT]; 1290 struct op_data_entries *soft_data_orig = 1291 &test_vector.entries[DATA_SOFT_OUTPUT]; 1292 struct rte_bbdev_op_turbo_dec *ops_td; 1293 struct rte_bbdev_op_data *hard_output; 1294 struct rte_bbdev_op_data *soft_output; 1295 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 1296 1297 for (i = 0; i < n; ++i) { 1298 ops_td = &ops[i]->turbo_dec; 1299 hard_output = &ops_td->hard_output; 1300 soft_output = &ops_td->soft_output; 1301 1302 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1303 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1304 "Returned iter_count (%d) > expected iter_count (%d)", 1305 ops_td->iter_count, ref_td->iter_count); 1306 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1307 TEST_ASSERT_SUCCESS(ret, 1308 "Checking status and ordering for decoder failed"); 1309 1310 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1311 hard_data_orig), 1312 "Hard output buffers (CB=%u) are not equal", 1313 i); 1314 1315 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 1316 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1317 soft_data_orig), 1318 "Soft output buffers (CB=%u) are not equal", 1319 i); 1320 } 1321 1322 return TEST_SUCCESS; 1323 } 1324 1325 1326 static int 1327 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1328 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1329 { 1330 unsigned int i; 1331 int ret; 1332 struct op_data_entries *hard_data_orig = 1333 &test_vector.entries[DATA_HARD_OUTPUT]; 1334 struct op_data_entries *soft_data_orig = 1335 &test_vector.entries[DATA_SOFT_OUTPUT]; 1336 struct op_data_entries *harq_data_orig = 1337 &test_vector.entries[DATA_HARQ_OUTPUT]; 1338 struct rte_bbdev_op_ldpc_dec *ops_td; 1339 struct rte_bbdev_op_data *hard_output; 1340 struct rte_bbdev_op_data *harq_output; 1341 struct rte_bbdev_op_data *soft_output; 1342 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 1343 1344 for (i = 0; i < n; ++i) { 1345 ops_td = &ops[i]->ldpc_dec; 1346 hard_output = &ops_td->hard_output; 1347 harq_output = &ops_td->harq_combined_output; 1348 soft_output = &ops_td->soft_output; 1349 1350 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1351 TEST_ASSERT_SUCCESS(ret, 1352 "Checking status and ordering for decoder failed"); 1353 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1354 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1355 "Returned iter_count (%d) > expected iter_count (%d)", 1356 ops_td->iter_count, ref_td->iter_count); 1357 /* We can ignore data when the decoding failed to converge */ 1358 if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0) 1359 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1360 hard_data_orig), 1361 "Hard output buffers (CB=%u) are not equal", 1362 i); 1363 1364 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 1365 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1366 soft_data_orig), 1367 "Soft output buffers (CB=%u) are not equal", 1368 i); 1369 if (ref_op->ldpc_dec.op_flags & 1370 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 1371 ldpc_input_llr_scaling(harq_output, 1, 8, 0); 1372 TEST_ASSERT_SUCCESS(validate_op_chain(harq_output, 1373 harq_data_orig), 1374 "HARQ output buffers (CB=%u) are not equal", 1375 i); 1376 } 1377 } 1378 1379 return TEST_SUCCESS; 1380 } 1381 1382 1383 static int 1384 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1385 struct rte_bbdev_enc_op *ref_op) 1386 { 1387 unsigned int i; 1388 int ret; 1389 struct op_data_entries *hard_data_orig = 1390 &test_vector.entries[DATA_HARD_OUTPUT]; 1391 1392 for (i = 0; i < n; ++i) { 1393 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1394 TEST_ASSERT_SUCCESS(ret, 1395 "Checking status and ordering for encoder failed"); 1396 TEST_ASSERT_SUCCESS(validate_op_chain( 1397 &ops[i]->turbo_enc.output, 1398 hard_data_orig), 1399 "Output buffers (CB=%u) are not equal", 1400 i); 1401 } 1402 1403 return TEST_SUCCESS; 1404 } 1405 1406 static int 1407 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1408 struct rte_bbdev_enc_op *ref_op) 1409 { 1410 unsigned int i; 1411 int ret; 1412 struct op_data_entries *hard_data_orig = 1413 &test_vector.entries[DATA_HARD_OUTPUT]; 1414 1415 for (i = 0; i < n; ++i) { 1416 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1417 TEST_ASSERT_SUCCESS(ret, 1418 "Checking status and ordering for encoder failed"); 1419 TEST_ASSERT_SUCCESS(validate_op_chain( 1420 &ops[i]->ldpc_enc.output, 1421 hard_data_orig), 1422 "Output buffers (CB=%u) are not equal", 1423 i); 1424 } 1425 1426 return TEST_SUCCESS; 1427 } 1428 1429 static void 1430 create_reference_dec_op(struct rte_bbdev_dec_op *op) 1431 { 1432 unsigned int i; 1433 struct op_data_entries *entry; 1434 1435 op->turbo_dec = test_vector.turbo_dec; 1436 entry = &test_vector.entries[DATA_INPUT]; 1437 for (i = 0; i < entry->nb_segments; ++i) 1438 op->turbo_dec.input.length += 1439 entry->segments[i].length; 1440 } 1441 1442 static void 1443 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 1444 { 1445 unsigned int i; 1446 struct op_data_entries *entry; 1447 1448 op->ldpc_dec = test_vector.ldpc_dec; 1449 entry = &test_vector.entries[DATA_INPUT]; 1450 for (i = 0; i < entry->nb_segments; ++i) 1451 op->ldpc_dec.input.length += 1452 entry->segments[i].length; 1453 if (test_vector.ldpc_dec.op_flags & 1454 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 1455 entry = &test_vector.entries[DATA_HARQ_INPUT]; 1456 for (i = 0; i < entry->nb_segments; ++i) 1457 op->ldpc_dec.harq_combined_input.length += 1458 entry->segments[i].length; 1459 } 1460 } 1461 1462 1463 static void 1464 create_reference_enc_op(struct rte_bbdev_enc_op *op) 1465 { 1466 unsigned int i; 1467 struct op_data_entries *entry; 1468 1469 op->turbo_enc = test_vector.turbo_enc; 1470 entry = &test_vector.entries[DATA_INPUT]; 1471 for (i = 0; i < entry->nb_segments; ++i) 1472 op->turbo_enc.input.length += 1473 entry->segments[i].length; 1474 } 1475 1476 static void 1477 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 1478 { 1479 unsigned int i; 1480 struct op_data_entries *entry; 1481 1482 op->ldpc_enc = test_vector.ldpc_enc; 1483 entry = &test_vector.entries[DATA_INPUT]; 1484 for (i = 0; i < entry->nb_segments; ++i) 1485 op->ldpc_enc.input.length += 1486 entry->segments[i].length; 1487 } 1488 1489 static uint32_t 1490 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 1491 { 1492 uint8_t i; 1493 uint32_t c, r, tb_size = 0; 1494 1495 if (op->turbo_dec.code_block_mode) { 1496 tb_size = op->turbo_dec.tb_params.k_neg; 1497 } else { 1498 c = op->turbo_dec.tb_params.c; 1499 r = op->turbo_dec.tb_params.r; 1500 for (i = 0; i < c-r; i++) 1501 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 1502 op->turbo_dec.tb_params.k_neg : 1503 op->turbo_dec.tb_params.k_pos; 1504 } 1505 return tb_size; 1506 } 1507 1508 static uint32_t 1509 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 1510 { 1511 uint8_t i; 1512 uint32_t c, r, tb_size = 0; 1513 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 1514 1515 if (op->ldpc_dec.code_block_mode) { 1516 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 1517 } else { 1518 c = op->ldpc_dec.tb_params.c; 1519 r = op->ldpc_dec.tb_params.r; 1520 for (i = 0; i < c-r; i++) 1521 tb_size += sys_cols * op->ldpc_dec.z_c 1522 - op->ldpc_dec.n_filler; 1523 } 1524 return tb_size; 1525 } 1526 1527 static uint32_t 1528 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 1529 { 1530 uint8_t i; 1531 uint32_t c, r, tb_size = 0; 1532 1533 if (op->turbo_enc.code_block_mode) { 1534 tb_size = op->turbo_enc.tb_params.k_neg; 1535 } else { 1536 c = op->turbo_enc.tb_params.c; 1537 r = op->turbo_enc.tb_params.r; 1538 for (i = 0; i < c-r; i++) 1539 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 1540 op->turbo_enc.tb_params.k_neg : 1541 op->turbo_enc.tb_params.k_pos; 1542 } 1543 return tb_size; 1544 } 1545 1546 static uint32_t 1547 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 1548 { 1549 uint8_t i; 1550 uint32_t c, r, tb_size = 0; 1551 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 1552 1553 if (op->turbo_enc.code_block_mode) { 1554 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 1555 } else { 1556 c = op->turbo_enc.tb_params.c; 1557 r = op->turbo_enc.tb_params.r; 1558 for (i = 0; i < c-r; i++) 1559 tb_size += sys_cols * op->ldpc_enc.z_c 1560 - op->ldpc_enc.n_filler; 1561 } 1562 return tb_size; 1563 } 1564 1565 1566 static int 1567 init_test_op_params(struct test_op_params *op_params, 1568 enum rte_bbdev_op_type op_type, const int expected_status, 1569 const int vector_mask, struct rte_mempool *ops_mp, 1570 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 1571 { 1572 int ret = 0; 1573 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1574 op_type == RTE_BBDEV_OP_LDPC_DEC) 1575 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 1576 &op_params->ref_dec_op, 1); 1577 else 1578 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 1579 &op_params->ref_enc_op, 1); 1580 1581 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 1582 1583 op_params->mp = ops_mp; 1584 op_params->burst_sz = burst_sz; 1585 op_params->num_to_process = num_to_process; 1586 op_params->num_lcores = num_lcores; 1587 op_params->vector_mask = vector_mask; 1588 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1589 op_type == RTE_BBDEV_OP_LDPC_DEC) 1590 op_params->ref_dec_op->status = expected_status; 1591 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 1592 || op_type == RTE_BBDEV_OP_LDPC_ENC) 1593 op_params->ref_enc_op->status = expected_status; 1594 return 0; 1595 } 1596 1597 static int 1598 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 1599 struct test_op_params *op_params) 1600 { 1601 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 1602 unsigned int i; 1603 struct active_device *ad; 1604 unsigned int burst_sz = get_burst_sz(); 1605 enum rte_bbdev_op_type op_type = test_vector.op_type; 1606 const struct rte_bbdev_op_cap *capabilities = NULL; 1607 1608 ad = &active_devs[dev_id]; 1609 1610 /* Check if device supports op_type */ 1611 if (!is_avail_op(ad, test_vector.op_type)) 1612 return TEST_SUCCESS; 1613 1614 struct rte_bbdev_info info; 1615 rte_bbdev_info_get(ad->dev_id, &info); 1616 socket_id = GET_SOCKET(info.socket_id); 1617 1618 f_ret = create_mempools(ad, socket_id, op_type, 1619 get_num_ops()); 1620 if (f_ret != TEST_SUCCESS) { 1621 printf("Couldn't create mempools"); 1622 goto fail; 1623 } 1624 if (op_type == RTE_BBDEV_OP_NONE) 1625 op_type = RTE_BBDEV_OP_TURBO_ENC; 1626 1627 f_ret = init_test_op_params(op_params, test_vector.op_type, 1628 test_vector.expected_status, 1629 test_vector.mask, 1630 ad->ops_mempool, 1631 burst_sz, 1632 get_num_ops(), 1633 get_num_lcores()); 1634 if (f_ret != TEST_SUCCESS) { 1635 printf("Couldn't init test op params"); 1636 goto fail; 1637 } 1638 1639 1640 /* Find capabilities */ 1641 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 1642 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 1643 if (cap->type == test_vector.op_type) { 1644 capabilities = cap; 1645 break; 1646 } 1647 cap++; 1648 } 1649 TEST_ASSERT_NOT_NULL(capabilities, 1650 "Couldn't find capabilities"); 1651 1652 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1653 create_reference_dec_op(op_params->ref_dec_op); 1654 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1655 create_reference_enc_op(op_params->ref_enc_op); 1656 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 1657 create_reference_ldpc_enc_op(op_params->ref_enc_op); 1658 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1659 create_reference_ldpc_dec_op(op_params->ref_dec_op); 1660 1661 for (i = 0; i < ad->nb_queues; ++i) { 1662 f_ret = fill_queue_buffers(op_params, 1663 ad->in_mbuf_pool, 1664 ad->hard_out_mbuf_pool, 1665 ad->soft_out_mbuf_pool, 1666 ad->harq_in_mbuf_pool, 1667 ad->harq_out_mbuf_pool, 1668 ad->queue_ids[i], 1669 capabilities, 1670 info.drv.min_alignment, 1671 socket_id); 1672 if (f_ret != TEST_SUCCESS) { 1673 printf("Couldn't init queue buffers"); 1674 goto fail; 1675 } 1676 } 1677 1678 /* Run test case function */ 1679 t_ret = test_case_func(ad, op_params); 1680 1681 /* Free active device resources and return */ 1682 free_buffers(ad, op_params); 1683 return t_ret; 1684 1685 fail: 1686 free_buffers(ad, op_params); 1687 return TEST_FAILED; 1688 } 1689 1690 /* Run given test function per active device per supported op type 1691 * per burst size. 1692 */ 1693 static int 1694 run_test_case(test_case_function *test_case_func) 1695 { 1696 int ret = 0; 1697 uint8_t dev; 1698 1699 /* Alloc op_params */ 1700 struct test_op_params *op_params = rte_zmalloc(NULL, 1701 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1702 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1703 RTE_ALIGN(sizeof(struct test_op_params), 1704 RTE_CACHE_LINE_SIZE)); 1705 1706 /* For each device run test case function */ 1707 for (dev = 0; dev < nb_active_devs; ++dev) 1708 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1709 1710 rte_free(op_params); 1711 1712 return ret; 1713 } 1714 1715 static void 1716 dequeue_event_callback(uint16_t dev_id, 1717 enum rte_bbdev_event_type event, void *cb_arg, 1718 void *ret_param) 1719 { 1720 int ret; 1721 uint16_t i; 1722 uint64_t total_time; 1723 uint16_t deq, burst_sz, num_ops; 1724 uint16_t queue_id = *(uint16_t *) ret_param; 1725 struct rte_bbdev_info info; 1726 double tb_len_bits; 1727 struct thread_params *tp = cb_arg; 1728 1729 /* Find matching thread params using queue_id */ 1730 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1731 if (tp->queue_id == queue_id) 1732 break; 1733 1734 if (i == MAX_QUEUES) { 1735 printf("%s: Queue_id from interrupt details was not found!\n", 1736 __func__); 1737 return; 1738 } 1739 1740 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1741 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1742 printf( 1743 "Dequeue interrupt handler called for incorrect event!\n"); 1744 return; 1745 } 1746 1747 burst_sz = rte_atomic16_read(&tp->burst_sz); 1748 num_ops = tp->op_params->num_to_process; 1749 1750 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 1751 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1752 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1753 &tp->dec_ops[ 1754 rte_atomic16_read(&tp->nb_dequeued)], 1755 burst_sz); 1756 else 1757 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1758 &tp->enc_ops[ 1759 rte_atomic16_read(&tp->nb_dequeued)], 1760 burst_sz); 1761 1762 if (deq < burst_sz) { 1763 printf( 1764 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1765 burst_sz, deq); 1766 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1767 return; 1768 } 1769 1770 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) { 1771 rte_atomic16_add(&tp->nb_dequeued, deq); 1772 return; 1773 } 1774 1775 total_time = rte_rdtsc_precise() - tp->start_time; 1776 1777 rte_bbdev_info_get(dev_id, &info); 1778 1779 ret = TEST_SUCCESS; 1780 1781 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1782 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1783 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 1784 tp->op_params->vector_mask); 1785 /* get the max of iter_count for all dequeued ops */ 1786 for (i = 0; i < num_ops; ++i) 1787 tp->iter_count = RTE_MAX( 1788 tp->dec_ops[i]->turbo_dec.iter_count, 1789 tp->iter_count); 1790 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1791 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 1792 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1793 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 1794 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1795 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 1796 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1797 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 1798 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1799 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1800 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1801 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 1802 tp->op_params->vector_mask); 1803 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1804 } 1805 1806 if (ret) { 1807 printf("Buffers validation failed\n"); 1808 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1809 } 1810 1811 switch (test_vector.op_type) { 1812 case RTE_BBDEV_OP_TURBO_DEC: 1813 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 1814 break; 1815 case RTE_BBDEV_OP_TURBO_ENC: 1816 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 1817 break; 1818 case RTE_BBDEV_OP_LDPC_DEC: 1819 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 1820 break; 1821 case RTE_BBDEV_OP_LDPC_ENC: 1822 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 1823 break; 1824 case RTE_BBDEV_OP_NONE: 1825 tb_len_bits = 0.0; 1826 break; 1827 default: 1828 printf("Unknown op type: %d\n", test_vector.op_type); 1829 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1830 return; 1831 } 1832 1833 tp->ops_per_sec += ((double)num_ops) / 1834 ((double)total_time / (double)rte_get_tsc_hz()); 1835 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 1836 ((double)total_time / (double)rte_get_tsc_hz()); 1837 1838 rte_atomic16_add(&tp->nb_dequeued, deq); 1839 } 1840 1841 static int 1842 throughput_intr_lcore_dec(void *arg) 1843 { 1844 struct thread_params *tp = arg; 1845 unsigned int enqueued; 1846 const uint16_t queue_id = tp->queue_id; 1847 const uint16_t burst_sz = tp->op_params->burst_sz; 1848 const uint16_t num_to_process = tp->op_params->num_to_process; 1849 struct rte_bbdev_dec_op *ops[num_to_process]; 1850 struct test_buffers *bufs = NULL; 1851 struct rte_bbdev_info info; 1852 int ret, i, j; 1853 uint16_t num_to_enq, enq; 1854 1855 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1856 "BURST_SIZE should be <= %u", MAX_BURST); 1857 1858 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1859 "Failed to enable interrupts for dev: %u, queue_id: %u", 1860 tp->dev_id, queue_id); 1861 1862 rte_bbdev_info_get(tp->dev_id, &info); 1863 1864 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1865 "NUM_OPS cannot exceed %u for this device", 1866 info.drv.queue_size_lim); 1867 1868 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1869 1870 rte_atomic16_clear(&tp->processing_status); 1871 rte_atomic16_clear(&tp->nb_dequeued); 1872 1873 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1874 rte_pause(); 1875 1876 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1877 num_to_process); 1878 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1879 num_to_process); 1880 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1881 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 1882 bufs->hard_outputs, bufs->soft_outputs, 1883 tp->op_params->ref_dec_op); 1884 1885 /* Set counter to validate the ordering */ 1886 for (j = 0; j < num_to_process; ++j) 1887 ops[j]->opaque_data = (void *)(uintptr_t)j; 1888 1889 for (j = 0; j < TEST_REPETITIONS; ++j) { 1890 for (i = 0; i < num_to_process; ++i) 1891 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 1892 1893 tp->start_time = rte_rdtsc_precise(); 1894 for (enqueued = 0; enqueued < num_to_process;) { 1895 num_to_enq = burst_sz; 1896 1897 if (unlikely(num_to_process - enqueued < num_to_enq)) 1898 num_to_enq = num_to_process - enqueued; 1899 1900 enq = 0; 1901 do { 1902 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1903 queue_id, &ops[enqueued], 1904 num_to_enq); 1905 } while (unlikely(num_to_enq != enq)); 1906 enqueued += enq; 1907 1908 /* Write to thread burst_sz current number of enqueued 1909 * descriptors. It ensures that proper number of 1910 * descriptors will be dequeued in callback 1911 * function - needed for last batch in case where 1912 * the number of operations is not a multiple of 1913 * burst size. 1914 */ 1915 rte_atomic16_set(&tp->burst_sz, num_to_enq); 1916 1917 /* Wait until processing of previous batch is 1918 * completed 1919 */ 1920 while (rte_atomic16_read(&tp->nb_dequeued) != 1921 (int16_t) enqueued) 1922 rte_pause(); 1923 } 1924 if (j != TEST_REPETITIONS - 1) 1925 rte_atomic16_clear(&tp->nb_dequeued); 1926 } 1927 1928 return TEST_SUCCESS; 1929 } 1930 1931 static int 1932 throughput_intr_lcore_enc(void *arg) 1933 { 1934 struct thread_params *tp = arg; 1935 unsigned int enqueued; 1936 const uint16_t queue_id = tp->queue_id; 1937 const uint16_t burst_sz = tp->op_params->burst_sz; 1938 const uint16_t num_to_process = tp->op_params->num_to_process; 1939 struct rte_bbdev_enc_op *ops[num_to_process]; 1940 struct test_buffers *bufs = NULL; 1941 struct rte_bbdev_info info; 1942 int ret, i, j; 1943 uint16_t num_to_enq, enq; 1944 1945 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1946 "BURST_SIZE should be <= %u", MAX_BURST); 1947 1948 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1949 "Failed to enable interrupts for dev: %u, queue_id: %u", 1950 tp->dev_id, queue_id); 1951 1952 rte_bbdev_info_get(tp->dev_id, &info); 1953 1954 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1955 "NUM_OPS cannot exceed %u for this device", 1956 info.drv.queue_size_lim); 1957 1958 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1959 1960 rte_atomic16_clear(&tp->processing_status); 1961 rte_atomic16_clear(&tp->nb_dequeued); 1962 1963 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1964 rte_pause(); 1965 1966 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1967 num_to_process); 1968 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1969 num_to_process); 1970 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1971 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 1972 bufs->hard_outputs, tp->op_params->ref_enc_op); 1973 1974 /* Set counter to validate the ordering */ 1975 for (j = 0; j < num_to_process; ++j) 1976 ops[j]->opaque_data = (void *)(uintptr_t)j; 1977 1978 for (j = 0; j < TEST_REPETITIONS; ++j) { 1979 for (i = 0; i < num_to_process; ++i) 1980 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 1981 1982 tp->start_time = rte_rdtsc_precise(); 1983 for (enqueued = 0; enqueued < num_to_process;) { 1984 num_to_enq = burst_sz; 1985 1986 if (unlikely(num_to_process - enqueued < num_to_enq)) 1987 num_to_enq = num_to_process - enqueued; 1988 1989 enq = 0; 1990 do { 1991 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1992 queue_id, &ops[enqueued], 1993 num_to_enq); 1994 } while (unlikely(enq != num_to_enq)); 1995 enqueued += enq; 1996 1997 /* Write to thread burst_sz current number of enqueued 1998 * descriptors. It ensures that proper number of 1999 * descriptors will be dequeued in callback 2000 * function - needed for last batch in case where 2001 * the number of operations is not a multiple of 2002 * burst size. 2003 */ 2004 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2005 2006 /* Wait until processing of previous batch is 2007 * completed 2008 */ 2009 while (rte_atomic16_read(&tp->nb_dequeued) != 2010 (int16_t) enqueued) 2011 rte_pause(); 2012 } 2013 if (j != TEST_REPETITIONS - 1) 2014 rte_atomic16_clear(&tp->nb_dequeued); 2015 } 2016 2017 return TEST_SUCCESS; 2018 } 2019 2020 static int 2021 throughput_pmd_lcore_dec(void *arg) 2022 { 2023 struct thread_params *tp = arg; 2024 uint16_t enq, deq; 2025 uint64_t total_time = 0, start_time; 2026 const uint16_t queue_id = tp->queue_id; 2027 const uint16_t burst_sz = tp->op_params->burst_sz; 2028 const uint16_t num_ops = tp->op_params->num_to_process; 2029 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2030 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2031 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2032 struct test_buffers *bufs = NULL; 2033 int i, j, ret; 2034 struct rte_bbdev_info info; 2035 uint16_t num_to_enq; 2036 2037 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2038 "BURST_SIZE should be <= %u", MAX_BURST); 2039 2040 rte_bbdev_info_get(tp->dev_id, &info); 2041 2042 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2043 "NUM_OPS cannot exceed %u for this device", 2044 info.drv.queue_size_lim); 2045 2046 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2047 2048 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2049 rte_pause(); 2050 2051 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2052 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2053 2054 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2055 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2056 bufs->hard_outputs, bufs->soft_outputs, ref_op); 2057 2058 /* Set counter to validate the ordering */ 2059 for (j = 0; j < num_ops; ++j) 2060 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2061 2062 for (i = 0; i < TEST_REPETITIONS; ++i) { 2063 2064 for (j = 0; j < num_ops; ++j) 2065 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 2066 2067 start_time = rte_rdtsc_precise(); 2068 2069 for (enq = 0, deq = 0; enq < num_ops;) { 2070 num_to_enq = burst_sz; 2071 2072 if (unlikely(num_ops - enq < num_to_enq)) 2073 num_to_enq = num_ops - enq; 2074 2075 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2076 queue_id, &ops_enq[enq], num_to_enq); 2077 2078 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2079 queue_id, &ops_deq[deq], enq - deq); 2080 } 2081 2082 /* dequeue the remaining */ 2083 while (deq < enq) { 2084 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2085 queue_id, &ops_deq[deq], enq - deq); 2086 } 2087 2088 total_time += rte_rdtsc_precise() - start_time; 2089 } 2090 2091 tp->iter_count = 0; 2092 /* get the max of iter_count for all dequeued ops */ 2093 for (i = 0; i < num_ops; ++i) { 2094 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 2095 tp->iter_count); 2096 } 2097 2098 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2099 ret = validate_dec_op(ops_deq, num_ops, ref_op, 2100 tp->op_params->vector_mask); 2101 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2102 } 2103 2104 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2105 2106 double tb_len_bits = calc_dec_TB_size(ref_op); 2107 2108 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2109 ((double)total_time / (double)rte_get_tsc_hz()); 2110 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2111 1000000.0) / ((double)total_time / 2112 (double)rte_get_tsc_hz()); 2113 2114 return TEST_SUCCESS; 2115 } 2116 2117 static int 2118 throughput_pmd_lcore_ldpc_dec(void *arg) 2119 { 2120 struct thread_params *tp = arg; 2121 uint16_t enq, deq; 2122 uint64_t total_time = 0, start_time; 2123 const uint16_t queue_id = tp->queue_id; 2124 const uint16_t burst_sz = tp->op_params->burst_sz; 2125 const uint16_t num_ops = tp->op_params->num_to_process; 2126 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2127 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2128 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2129 struct test_buffers *bufs = NULL; 2130 int i, j, ret; 2131 struct rte_bbdev_info info; 2132 uint16_t num_to_enq; 2133 2134 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2135 "BURST_SIZE should be <= %u", MAX_BURST); 2136 2137 rte_bbdev_info_get(tp->dev_id, &info); 2138 2139 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2140 "NUM_OPS cannot exceed %u for this device", 2141 info.drv.queue_size_lim); 2142 2143 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2144 2145 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2146 rte_pause(); 2147 2148 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2149 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2150 2151 /* For throughput tests we need to disable early termination */ 2152 if (check_bit(ref_op->ldpc_dec.op_flags, 2153 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 2154 ref_op->ldpc_dec.op_flags -= 2155 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 2156 ref_op->ldpc_dec.iter_max = 6; 2157 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 2158 2159 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2160 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2161 bufs->hard_outputs, bufs->soft_outputs, 2162 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2163 2164 /* Set counter to validate the ordering */ 2165 for (j = 0; j < num_ops; ++j) 2166 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2167 2168 for (i = 0; i < TEST_REPETITIONS; ++i) { 2169 for (j = 0; j < num_ops; ++j) { 2170 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 2171 if (check_bit(ref_op->ldpc_dec.op_flags, 2172 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 2173 mbuf_reset( 2174 ops_enq[j]->ldpc_dec.harq_combined_output.data); 2175 } 2176 2177 start_time = rte_rdtsc_precise(); 2178 2179 for (enq = 0, deq = 0; enq < num_ops;) { 2180 num_to_enq = burst_sz; 2181 2182 if (unlikely(num_ops - enq < num_to_enq)) 2183 num_to_enq = num_ops - enq; 2184 2185 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 2186 queue_id, &ops_enq[enq], num_to_enq); 2187 2188 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2189 queue_id, &ops_deq[deq], enq - deq); 2190 } 2191 2192 /* dequeue the remaining */ 2193 while (deq < enq) { 2194 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2195 queue_id, &ops_deq[deq], enq - deq); 2196 } 2197 2198 total_time += rte_rdtsc_precise() - start_time; 2199 } 2200 2201 tp->iter_count = 0; 2202 /* get the max of iter_count for all dequeued ops */ 2203 for (i = 0; i < num_ops; ++i) { 2204 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 2205 tp->iter_count); 2206 } 2207 2208 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2209 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 2210 tp->op_params->vector_mask); 2211 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2212 } 2213 2214 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2215 2216 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 2217 2218 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2219 ((double)total_time / (double)rte_get_tsc_hz()); 2220 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2221 1000000.0) / ((double)total_time / 2222 (double)rte_get_tsc_hz()); 2223 2224 return TEST_SUCCESS; 2225 } 2226 2227 static int 2228 throughput_pmd_lcore_enc(void *arg) 2229 { 2230 struct thread_params *tp = arg; 2231 uint16_t enq, deq; 2232 uint64_t total_time = 0, start_time; 2233 const uint16_t queue_id = tp->queue_id; 2234 const uint16_t burst_sz = tp->op_params->burst_sz; 2235 const uint16_t num_ops = tp->op_params->num_to_process; 2236 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2237 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2238 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2239 struct test_buffers *bufs = NULL; 2240 int i, j, ret; 2241 struct rte_bbdev_info info; 2242 uint16_t num_to_enq; 2243 2244 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2245 "BURST_SIZE should be <= %u", MAX_BURST); 2246 2247 rte_bbdev_info_get(tp->dev_id, &info); 2248 2249 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2250 "NUM_OPS cannot exceed %u for this device", 2251 info.drv.queue_size_lim); 2252 2253 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2254 2255 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2256 rte_pause(); 2257 2258 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2259 num_ops); 2260 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2261 num_ops); 2262 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2263 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2264 bufs->hard_outputs, ref_op); 2265 2266 /* Set counter to validate the ordering */ 2267 for (j = 0; j < num_ops; ++j) 2268 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2269 2270 for (i = 0; i < TEST_REPETITIONS; ++i) { 2271 2272 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2273 for (j = 0; j < num_ops; ++j) 2274 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2275 2276 start_time = rte_rdtsc_precise(); 2277 2278 for (enq = 0, deq = 0; enq < num_ops;) { 2279 num_to_enq = burst_sz; 2280 2281 if (unlikely(num_ops - enq < num_to_enq)) 2282 num_to_enq = num_ops - enq; 2283 2284 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2285 queue_id, &ops_enq[enq], num_to_enq); 2286 2287 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2288 queue_id, &ops_deq[deq], enq - deq); 2289 } 2290 2291 /* dequeue the remaining */ 2292 while (deq < enq) { 2293 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2294 queue_id, &ops_deq[deq], enq - deq); 2295 } 2296 2297 total_time += rte_rdtsc_precise() - start_time; 2298 } 2299 2300 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2301 ret = validate_enc_op(ops_deq, num_ops, ref_op); 2302 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2303 } 2304 2305 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2306 2307 double tb_len_bits = calc_enc_TB_size(ref_op); 2308 2309 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2310 ((double)total_time / (double)rte_get_tsc_hz()); 2311 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2312 / 1000000.0) / ((double)total_time / 2313 (double)rte_get_tsc_hz()); 2314 2315 return TEST_SUCCESS; 2316 } 2317 2318 static int 2319 throughput_pmd_lcore_ldpc_enc(void *arg) 2320 { 2321 struct thread_params *tp = arg; 2322 uint16_t enq, deq; 2323 uint64_t total_time = 0, start_time; 2324 const uint16_t queue_id = tp->queue_id; 2325 const uint16_t burst_sz = tp->op_params->burst_sz; 2326 const uint16_t num_ops = tp->op_params->num_to_process; 2327 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2328 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2329 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2330 struct test_buffers *bufs = NULL; 2331 int i, j, ret; 2332 struct rte_bbdev_info info; 2333 uint16_t num_to_enq; 2334 2335 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2336 "BURST_SIZE should be <= %u", MAX_BURST); 2337 2338 rte_bbdev_info_get(tp->dev_id, &info); 2339 2340 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2341 "NUM_OPS cannot exceed %u for this device", 2342 info.drv.queue_size_lim); 2343 2344 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2345 2346 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2347 rte_pause(); 2348 2349 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2350 num_ops); 2351 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2352 num_ops); 2353 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2354 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2355 bufs->hard_outputs, ref_op); 2356 2357 /* Set counter to validate the ordering */ 2358 for (j = 0; j < num_ops; ++j) 2359 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2360 2361 for (i = 0; i < TEST_REPETITIONS; ++i) { 2362 2363 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2364 for (j = 0; j < num_ops; ++j) 2365 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2366 2367 start_time = rte_rdtsc_precise(); 2368 2369 for (enq = 0, deq = 0; enq < num_ops;) { 2370 num_to_enq = burst_sz; 2371 2372 if (unlikely(num_ops - enq < num_to_enq)) 2373 num_to_enq = num_ops - enq; 2374 2375 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 2376 queue_id, &ops_enq[enq], num_to_enq); 2377 2378 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2379 queue_id, &ops_deq[deq], enq - deq); 2380 } 2381 2382 /* dequeue the remaining */ 2383 while (deq < enq) { 2384 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2385 queue_id, &ops_deq[deq], enq - deq); 2386 } 2387 2388 total_time += rte_rdtsc_precise() - start_time; 2389 } 2390 2391 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2392 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 2393 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2394 } 2395 2396 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2397 2398 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 2399 2400 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2401 ((double)total_time / (double)rte_get_tsc_hz()); 2402 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2403 / 1000000.0) / ((double)total_time / 2404 (double)rte_get_tsc_hz()); 2405 2406 return TEST_SUCCESS; 2407 } 2408 2409 static void 2410 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 2411 { 2412 unsigned int iter = 0; 2413 double total_mops = 0, total_mbps = 0; 2414 2415 for (iter = 0; iter < used_cores; iter++) { 2416 printf( 2417 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 2418 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2419 t_params[iter].mbps); 2420 total_mops += t_params[iter].ops_per_sec; 2421 total_mbps += t_params[iter].mbps; 2422 } 2423 printf( 2424 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 2425 used_cores, total_mops, total_mbps); 2426 } 2427 2428 static void 2429 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 2430 { 2431 unsigned int iter = 0; 2432 double total_mops = 0, total_mbps = 0; 2433 uint8_t iter_count = 0; 2434 2435 for (iter = 0; iter < used_cores; iter++) { 2436 printf( 2437 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 2438 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2439 t_params[iter].mbps, t_params[iter].iter_count); 2440 total_mops += t_params[iter].ops_per_sec; 2441 total_mbps += t_params[iter].mbps; 2442 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count); 2443 } 2444 printf( 2445 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 2446 used_cores, total_mops, total_mbps, iter_count); 2447 } 2448 2449 /* 2450 * Test function that determines how long an enqueue + dequeue of a burst 2451 * takes on available lcores. 2452 */ 2453 static int 2454 throughput_test(struct active_device *ad, 2455 struct test_op_params *op_params) 2456 { 2457 int ret; 2458 unsigned int lcore_id, used_cores = 0; 2459 struct thread_params *t_params, *tp; 2460 struct rte_bbdev_info info; 2461 lcore_function_t *throughput_function; 2462 uint16_t num_lcores; 2463 const char *op_type_str; 2464 2465 rte_bbdev_info_get(ad->dev_id, &info); 2466 2467 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 2468 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 2469 test_vector.op_type); 2470 2471 printf("+ ------------------------------------------------------- +\n"); 2472 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 2473 info.dev_name, ad->nb_queues, op_params->burst_sz, 2474 op_params->num_to_process, op_params->num_lcores, 2475 op_type_str, 2476 intr_enabled ? "Interrupt mode" : "PMD mode", 2477 (double)rte_get_tsc_hz() / 1000000000.0); 2478 2479 /* Set number of lcores */ 2480 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 2481 ? ad->nb_queues 2482 : op_params->num_lcores; 2483 2484 /* Allocate memory for thread parameters structure */ 2485 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 2486 RTE_CACHE_LINE_SIZE); 2487 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 2488 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 2489 RTE_CACHE_LINE_SIZE)); 2490 2491 if (intr_enabled) { 2492 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2493 throughput_function = throughput_intr_lcore_dec; 2494 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2495 throughput_function = throughput_intr_lcore_dec; 2496 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2497 throughput_function = throughput_intr_lcore_enc; 2498 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2499 throughput_function = throughput_intr_lcore_enc; 2500 else 2501 throughput_function = throughput_intr_lcore_enc; 2502 2503 /* Dequeue interrupt callback registration */ 2504 ret = rte_bbdev_callback_register(ad->dev_id, 2505 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 2506 t_params); 2507 if (ret < 0) { 2508 rte_free(t_params); 2509 return ret; 2510 } 2511 } else { 2512 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2513 throughput_function = throughput_pmd_lcore_dec; 2514 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2515 throughput_function = throughput_pmd_lcore_ldpc_dec; 2516 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2517 throughput_function = throughput_pmd_lcore_enc; 2518 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2519 throughput_function = throughput_pmd_lcore_ldpc_enc; 2520 else 2521 throughput_function = throughput_pmd_lcore_enc; 2522 } 2523 2524 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 2525 2526 /* Master core is set at first entry */ 2527 t_params[0].dev_id = ad->dev_id; 2528 t_params[0].lcore_id = rte_lcore_id(); 2529 t_params[0].op_params = op_params; 2530 t_params[0].queue_id = ad->queue_ids[used_cores++]; 2531 t_params[0].iter_count = 0; 2532 2533 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2534 if (used_cores >= num_lcores) 2535 break; 2536 2537 t_params[used_cores].dev_id = ad->dev_id; 2538 t_params[used_cores].lcore_id = lcore_id; 2539 t_params[used_cores].op_params = op_params; 2540 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 2541 t_params[used_cores].iter_count = 0; 2542 2543 rte_eal_remote_launch(throughput_function, 2544 &t_params[used_cores++], lcore_id); 2545 } 2546 2547 rte_atomic16_set(&op_params->sync, SYNC_START); 2548 ret = throughput_function(&t_params[0]); 2549 2550 /* Master core is always used */ 2551 for (used_cores = 1; used_cores < num_lcores; used_cores++) 2552 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 2553 2554 /* Return if test failed */ 2555 if (ret) { 2556 rte_free(t_params); 2557 return ret; 2558 } 2559 2560 /* Print throughput if interrupts are disabled and test passed */ 2561 if (!intr_enabled) { 2562 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2563 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2564 print_dec_throughput(t_params, num_lcores); 2565 else 2566 print_enc_throughput(t_params, num_lcores); 2567 rte_free(t_params); 2568 return ret; 2569 } 2570 2571 /* In interrupt TC we need to wait for the interrupt callback to deqeue 2572 * all pending operations. Skip waiting for queues which reported an 2573 * error using processing_status variable. 2574 * Wait for master lcore operations. 2575 */ 2576 tp = &t_params[0]; 2577 while ((rte_atomic16_read(&tp->nb_dequeued) < 2578 op_params->num_to_process) && 2579 (rte_atomic16_read(&tp->processing_status) != 2580 TEST_FAILED)) 2581 rte_pause(); 2582 2583 tp->ops_per_sec /= TEST_REPETITIONS; 2584 tp->mbps /= TEST_REPETITIONS; 2585 ret |= (int)rte_atomic16_read(&tp->processing_status); 2586 2587 /* Wait for slave lcores operations */ 2588 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 2589 tp = &t_params[used_cores]; 2590 2591 while ((rte_atomic16_read(&tp->nb_dequeued) < 2592 op_params->num_to_process) && 2593 (rte_atomic16_read(&tp->processing_status) != 2594 TEST_FAILED)) 2595 rte_pause(); 2596 2597 tp->ops_per_sec /= TEST_REPETITIONS; 2598 tp->mbps /= TEST_REPETITIONS; 2599 ret |= (int)rte_atomic16_read(&tp->processing_status); 2600 } 2601 2602 /* Print throughput if test passed */ 2603 if (!ret) { 2604 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2605 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2606 print_dec_throughput(t_params, num_lcores); 2607 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 2608 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2609 print_enc_throughput(t_params, num_lcores); 2610 } 2611 2612 rte_free(t_params); 2613 return ret; 2614 } 2615 2616 static int 2617 latency_test_dec(struct rte_mempool *mempool, 2618 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2619 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2620 const uint16_t num_to_process, uint16_t burst_sz, 2621 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2622 { 2623 int ret = TEST_SUCCESS; 2624 uint16_t i, j, dequeued; 2625 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2626 uint64_t start_time = 0, last_time = 0; 2627 2628 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2629 uint16_t enq = 0, deq = 0; 2630 bool first_time = true; 2631 last_time = 0; 2632 2633 if (unlikely(num_to_process - dequeued < burst_sz)) 2634 burst_sz = num_to_process - dequeued; 2635 2636 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2637 TEST_ASSERT_SUCCESS(ret, 2638 "rte_bbdev_dec_op_alloc_bulk() failed"); 2639 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2640 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 2641 bufs->inputs, 2642 bufs->hard_outputs, 2643 bufs->soft_outputs, 2644 ref_op); 2645 2646 /* Set counter to validate the ordering */ 2647 for (j = 0; j < burst_sz; ++j) 2648 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2649 2650 start_time = rte_rdtsc_precise(); 2651 2652 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 2653 burst_sz); 2654 TEST_ASSERT(enq == burst_sz, 2655 "Error enqueueing burst, expected %u, got %u", 2656 burst_sz, enq); 2657 2658 /* Dequeue */ 2659 do { 2660 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2661 &ops_deq[deq], burst_sz - deq); 2662 if (likely(first_time && (deq > 0))) { 2663 last_time = rte_rdtsc_precise() - start_time; 2664 first_time = false; 2665 } 2666 } while (unlikely(burst_sz != deq)); 2667 2668 *max_time = RTE_MAX(*max_time, last_time); 2669 *min_time = RTE_MIN(*min_time, last_time); 2670 *total_time += last_time; 2671 2672 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2673 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 2674 vector_mask); 2675 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2676 } 2677 2678 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2679 dequeued += deq; 2680 } 2681 2682 return i; 2683 } 2684 2685 static int 2686 latency_test_ldpc_dec(struct rte_mempool *mempool, 2687 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2688 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2689 const uint16_t num_to_process, uint16_t burst_sz, 2690 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2691 { 2692 int ret = TEST_SUCCESS; 2693 uint16_t i, j, dequeued; 2694 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2695 uint64_t start_time = 0, last_time = 0; 2696 2697 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2698 uint16_t enq = 0, deq = 0; 2699 bool first_time = true; 2700 last_time = 0; 2701 2702 if (unlikely(num_to_process - dequeued < burst_sz)) 2703 burst_sz = num_to_process - dequeued; 2704 2705 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2706 TEST_ASSERT_SUCCESS(ret, 2707 "rte_bbdev_dec_op_alloc_bulk() failed"); 2708 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2709 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 2710 bufs->inputs, 2711 bufs->hard_outputs, 2712 bufs->soft_outputs, 2713 bufs->harq_inputs, 2714 bufs->harq_outputs, 2715 ref_op); 2716 2717 /* Set counter to validate the ordering */ 2718 for (j = 0; j < burst_sz; ++j) 2719 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2720 2721 start_time = rte_rdtsc_precise(); 2722 2723 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2724 &ops_enq[enq], burst_sz); 2725 TEST_ASSERT(enq == burst_sz, 2726 "Error enqueueing burst, expected %u, got %u", 2727 burst_sz, enq); 2728 2729 /* Dequeue */ 2730 do { 2731 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2732 &ops_deq[deq], burst_sz - deq); 2733 if (likely(first_time && (deq > 0))) { 2734 last_time = rte_rdtsc_precise() - start_time; 2735 first_time = false; 2736 } 2737 } while (unlikely(burst_sz != deq)); 2738 2739 *max_time = RTE_MAX(*max_time, last_time); 2740 *min_time = RTE_MIN(*min_time, last_time); 2741 *total_time += last_time; 2742 2743 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2744 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 2745 vector_mask); 2746 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2747 } 2748 2749 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2750 dequeued += deq; 2751 } 2752 2753 return i; 2754 } 2755 2756 static int 2757 latency_test_enc(struct rte_mempool *mempool, 2758 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2759 uint16_t dev_id, uint16_t queue_id, 2760 const uint16_t num_to_process, uint16_t burst_sz, 2761 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2762 { 2763 int ret = TEST_SUCCESS; 2764 uint16_t i, j, dequeued; 2765 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2766 uint64_t start_time = 0, last_time = 0; 2767 2768 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2769 uint16_t enq = 0, deq = 0; 2770 bool first_time = true; 2771 last_time = 0; 2772 2773 if (unlikely(num_to_process - dequeued < burst_sz)) 2774 burst_sz = num_to_process - dequeued; 2775 2776 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2777 TEST_ASSERT_SUCCESS(ret, 2778 "rte_bbdev_enc_op_alloc_bulk() failed"); 2779 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2780 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 2781 bufs->inputs, 2782 bufs->hard_outputs, 2783 ref_op); 2784 2785 /* Set counter to validate the ordering */ 2786 for (j = 0; j < burst_sz; ++j) 2787 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2788 2789 start_time = rte_rdtsc_precise(); 2790 2791 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 2792 burst_sz); 2793 TEST_ASSERT(enq == burst_sz, 2794 "Error enqueueing burst, expected %u, got %u", 2795 burst_sz, enq); 2796 2797 /* Dequeue */ 2798 do { 2799 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2800 &ops_deq[deq], burst_sz - deq); 2801 if (likely(first_time && (deq > 0))) { 2802 last_time += rte_rdtsc_precise() - start_time; 2803 first_time = false; 2804 } 2805 } while (unlikely(burst_sz != deq)); 2806 2807 *max_time = RTE_MAX(*max_time, last_time); 2808 *min_time = RTE_MIN(*min_time, last_time); 2809 *total_time += last_time; 2810 2811 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2812 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2813 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2814 } 2815 2816 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2817 dequeued += deq; 2818 } 2819 2820 return i; 2821 } 2822 2823 static int 2824 latency_test_ldpc_enc(struct rte_mempool *mempool, 2825 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2826 uint16_t dev_id, uint16_t queue_id, 2827 const uint16_t num_to_process, uint16_t burst_sz, 2828 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2829 { 2830 int ret = TEST_SUCCESS; 2831 uint16_t i, j, dequeued; 2832 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2833 uint64_t start_time = 0, last_time = 0; 2834 2835 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2836 uint16_t enq = 0, deq = 0; 2837 bool first_time = true; 2838 last_time = 0; 2839 2840 if (unlikely(num_to_process - dequeued < burst_sz)) 2841 burst_sz = num_to_process - dequeued; 2842 2843 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2844 2845 TEST_ASSERT_SUCCESS(ret, 2846 "rte_bbdev_enc_op_alloc_bulk() failed"); 2847 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2848 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 2849 bufs->inputs, 2850 bufs->hard_outputs, 2851 ref_op); 2852 2853 /* Set counter to validate the ordering */ 2854 for (j = 0; j < burst_sz; ++j) 2855 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2856 2857 start_time = rte_rdtsc_precise(); 2858 2859 /* 2860 * printf("Latency Debug %d\n", 2861 * ops_enq[0]->ldpc_enc.cb_params.z_c); REMOVEME 2862 */ 2863 2864 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 2865 &ops_enq[enq], burst_sz); 2866 TEST_ASSERT(enq == burst_sz, 2867 "Error enqueueing burst, expected %u, got %u", 2868 burst_sz, enq); 2869 2870 /* Dequeue */ 2871 do { 2872 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2873 &ops_deq[deq], burst_sz - deq); 2874 if (likely(first_time && (deq > 0))) { 2875 last_time += rte_rdtsc_precise() - start_time; 2876 first_time = false; 2877 } 2878 } while (unlikely(burst_sz != deq)); 2879 2880 *max_time = RTE_MAX(*max_time, last_time); 2881 *min_time = RTE_MIN(*min_time, last_time); 2882 *total_time += last_time; 2883 2884 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2885 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2886 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2887 } 2888 2889 /* 2890 * printf("Ready to free - deq %d num_to_process %d\n", FIXME 2891 * deq, num_to_process); 2892 * printf("cache %d\n", ops_enq[0]->mempool->cache_size); 2893 */ 2894 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2895 dequeued += deq; 2896 } 2897 2898 return i; 2899 } 2900 2901 static int 2902 latency_test(struct active_device *ad, 2903 struct test_op_params *op_params) 2904 { 2905 int iter; 2906 uint16_t burst_sz = op_params->burst_sz; 2907 const uint16_t num_to_process = op_params->num_to_process; 2908 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2909 const uint16_t queue_id = ad->queue_ids[0]; 2910 struct test_buffers *bufs = NULL; 2911 struct rte_bbdev_info info; 2912 uint64_t total_time, min_time, max_time; 2913 const char *op_type_str; 2914 2915 total_time = max_time = 0; 2916 min_time = UINT64_MAX; 2917 2918 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2919 "BURST_SIZE should be <= %u", MAX_BURST); 2920 2921 rte_bbdev_info_get(ad->dev_id, &info); 2922 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2923 2924 op_type_str = rte_bbdev_op_type_str(op_type); 2925 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2926 2927 printf("+ ------------------------------------------------------- +\n"); 2928 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 2929 info.dev_name, burst_sz, num_to_process, op_type_str); 2930 2931 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2932 iter = latency_test_dec(op_params->mp, bufs, 2933 op_params->ref_dec_op, op_params->vector_mask, 2934 ad->dev_id, queue_id, num_to_process, 2935 burst_sz, &total_time, &min_time, &max_time); 2936 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 2937 iter = latency_test_enc(op_params->mp, bufs, 2938 op_params->ref_enc_op, ad->dev_id, queue_id, 2939 num_to_process, burst_sz, &total_time, 2940 &min_time, &max_time); 2941 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 2942 iter = latency_test_ldpc_enc(op_params->mp, bufs, 2943 op_params->ref_enc_op, ad->dev_id, queue_id, 2944 num_to_process, burst_sz, &total_time, 2945 &min_time, &max_time); 2946 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 2947 iter = latency_test_ldpc_dec(op_params->mp, bufs, 2948 op_params->ref_dec_op, op_params->vector_mask, 2949 ad->dev_id, queue_id, num_to_process, 2950 burst_sz, &total_time, &min_time, &max_time); 2951 else 2952 iter = latency_test_enc(op_params->mp, bufs, 2953 op_params->ref_enc_op, 2954 ad->dev_id, queue_id, 2955 num_to_process, burst_sz, &total_time, 2956 &min_time, &max_time); 2957 2958 if (iter <= 0) 2959 return TEST_FAILED; 2960 2961 printf("Operation latency:\n" 2962 "\tavg: %lg cycles, %lg us\n" 2963 "\tmin: %lg cycles, %lg us\n" 2964 "\tmax: %lg cycles, %lg us\n", 2965 (double)total_time / (double)iter, 2966 (double)(total_time * 1000000) / (double)iter / 2967 (double)rte_get_tsc_hz(), (double)min_time, 2968 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 2969 (double)max_time, (double)(max_time * 1000000) / 2970 (double)rte_get_tsc_hz()); 2971 2972 return TEST_SUCCESS; 2973 } 2974 2975 #ifdef RTE_BBDEV_OFFLOAD_COST 2976 static int 2977 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 2978 struct rte_bbdev_stats *stats) 2979 { 2980 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 2981 struct rte_bbdev_stats *q_stats; 2982 2983 if (queue_id >= dev->data->num_queues) 2984 return -1; 2985 2986 q_stats = &dev->data->queues[queue_id].queue_stats; 2987 2988 stats->enqueued_count = q_stats->enqueued_count; 2989 stats->dequeued_count = q_stats->dequeued_count; 2990 stats->enqueue_err_count = q_stats->enqueue_err_count; 2991 stats->dequeue_err_count = q_stats->dequeue_err_count; 2992 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 2993 2994 return 0; 2995 } 2996 2997 static int 2998 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 2999 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 3000 uint16_t queue_id, const uint16_t num_to_process, 3001 uint16_t burst_sz, struct test_time_stats *time_st) 3002 { 3003 int i, dequeued, ret; 3004 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3005 uint64_t enq_start_time, deq_start_time; 3006 uint64_t enq_sw_last_time, deq_last_time; 3007 struct rte_bbdev_stats stats; 3008 3009 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3010 uint16_t enq = 0, deq = 0; 3011 3012 if (unlikely(num_to_process - dequeued < burst_sz)) 3013 burst_sz = num_to_process - dequeued; 3014 3015 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3016 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3017 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3018 bufs->inputs, 3019 bufs->hard_outputs, 3020 bufs->soft_outputs, 3021 ref_op); 3022 3023 /* Start time meas for enqueue function offload latency */ 3024 enq_start_time = rte_rdtsc_precise(); 3025 do { 3026 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 3027 &ops_enq[enq], burst_sz - enq); 3028 } while (unlikely(burst_sz != enq)); 3029 3030 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3031 TEST_ASSERT_SUCCESS(ret, 3032 "Failed to get stats for queue (%u) of device (%u)", 3033 queue_id, dev_id); 3034 3035 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3036 stats.acc_offload_cycles; 3037 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3038 enq_sw_last_time); 3039 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3040 enq_sw_last_time); 3041 time_st->enq_sw_total_time += enq_sw_last_time; 3042 3043 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3044 stats.acc_offload_cycles); 3045 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3046 stats.acc_offload_cycles); 3047 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3048 3049 /* give time for device to process ops */ 3050 rte_delay_us(200); 3051 3052 /* Start time meas for dequeue function offload latency */ 3053 deq_start_time = rte_rdtsc_precise(); 3054 /* Dequeue one operation */ 3055 do { 3056 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3057 &ops_deq[deq], 1); 3058 } while (unlikely(deq != 1)); 3059 3060 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3061 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3062 deq_last_time); 3063 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3064 deq_last_time); 3065 time_st->deq_total_time += deq_last_time; 3066 3067 /* Dequeue remaining operations if needed*/ 3068 while (burst_sz != deq) 3069 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3070 &ops_deq[deq], burst_sz - deq); 3071 3072 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3073 dequeued += deq; 3074 } 3075 3076 return i; 3077 } 3078 3079 static int 3080 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 3081 struct test_buffers *bufs, 3082 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 3083 uint16_t queue_id, const uint16_t num_to_process, 3084 uint16_t burst_sz, struct test_time_stats *time_st) 3085 { 3086 int i, dequeued, ret; 3087 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3088 uint64_t enq_start_time, deq_start_time; 3089 uint64_t enq_sw_last_time, deq_last_time; 3090 struct rte_bbdev_stats stats; 3091 3092 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3093 uint16_t enq = 0, deq = 0; 3094 3095 if (unlikely(num_to_process - dequeued < burst_sz)) 3096 burst_sz = num_to_process - dequeued; 3097 3098 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3099 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3100 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 3101 bufs->inputs, 3102 bufs->hard_outputs, 3103 bufs->soft_outputs, 3104 bufs->harq_inputs, 3105 bufs->harq_outputs, 3106 ref_op); 3107 3108 /* Start time meas for enqueue function offload latency */ 3109 enq_start_time = rte_rdtsc_precise(); 3110 do { 3111 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3112 &ops_enq[enq], burst_sz - enq); 3113 } while (unlikely(burst_sz != enq)); 3114 3115 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3116 TEST_ASSERT_SUCCESS(ret, 3117 "Failed to get stats for queue (%u) of device (%u)", 3118 queue_id, dev_id); 3119 3120 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3121 stats.acc_offload_cycles; 3122 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3123 enq_sw_last_time); 3124 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3125 enq_sw_last_time); 3126 time_st->enq_sw_total_time += enq_sw_last_time; 3127 3128 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3129 stats.acc_offload_cycles); 3130 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3131 stats.acc_offload_cycles); 3132 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3133 3134 /* give time for device to process ops */ 3135 rte_delay_us(200); 3136 3137 /* Start time meas for dequeue function offload latency */ 3138 deq_start_time = rte_rdtsc_precise(); 3139 /* Dequeue one operation */ 3140 do { 3141 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3142 &ops_deq[deq], 1); 3143 } while (unlikely(deq != 1)); 3144 3145 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3146 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3147 deq_last_time); 3148 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3149 deq_last_time); 3150 time_st->deq_total_time += deq_last_time; 3151 3152 /* Dequeue remaining operations if needed*/ 3153 while (burst_sz != deq) 3154 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3155 &ops_deq[deq], burst_sz - deq); 3156 3157 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3158 dequeued += deq; 3159 } 3160 3161 return i; 3162 } 3163 3164 static int 3165 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 3166 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3167 uint16_t queue_id, const uint16_t num_to_process, 3168 uint16_t burst_sz, struct test_time_stats *time_st) 3169 { 3170 int i, dequeued, ret; 3171 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3172 uint64_t enq_start_time, deq_start_time; 3173 uint64_t enq_sw_last_time, deq_last_time; 3174 struct rte_bbdev_stats stats; 3175 3176 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3177 uint16_t enq = 0, deq = 0; 3178 3179 if (unlikely(num_to_process - dequeued < burst_sz)) 3180 burst_sz = num_to_process - dequeued; 3181 3182 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3183 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3184 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3185 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 3186 bufs->inputs, 3187 bufs->hard_outputs, 3188 ref_op); 3189 3190 /* Start time meas for enqueue function offload latency */ 3191 enq_start_time = rte_rdtsc_precise(); 3192 do { 3193 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 3194 &ops_enq[enq], burst_sz - enq); 3195 } while (unlikely(burst_sz != enq)); 3196 3197 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3198 TEST_ASSERT_SUCCESS(ret, 3199 "Failed to get stats for queue (%u) of device (%u)", 3200 queue_id, dev_id); 3201 3202 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3203 stats.acc_offload_cycles; 3204 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3205 enq_sw_last_time); 3206 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3207 enq_sw_last_time); 3208 time_st->enq_sw_total_time += enq_sw_last_time; 3209 3210 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3211 stats.acc_offload_cycles); 3212 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3213 stats.acc_offload_cycles); 3214 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3215 3216 /* give time for device to process ops */ 3217 rte_delay_us(200); 3218 3219 /* Start time meas for dequeue function offload latency */ 3220 deq_start_time = rte_rdtsc_precise(); 3221 /* Dequeue one operation */ 3222 do { 3223 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3224 &ops_deq[deq], 1); 3225 } while (unlikely(deq != 1)); 3226 3227 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3228 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3229 deq_last_time); 3230 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3231 deq_last_time); 3232 time_st->deq_total_time += deq_last_time; 3233 3234 while (burst_sz != deq) 3235 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3236 &ops_deq[deq], burst_sz - deq); 3237 3238 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3239 dequeued += deq; 3240 } 3241 3242 return i; 3243 } 3244 3245 static int 3246 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 3247 struct test_buffers *bufs, 3248 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3249 uint16_t queue_id, const uint16_t num_to_process, 3250 uint16_t burst_sz, struct test_time_stats *time_st) 3251 { 3252 int i, dequeued, ret; 3253 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3254 uint64_t enq_start_time, deq_start_time; 3255 uint64_t enq_sw_last_time, deq_last_time; 3256 struct rte_bbdev_stats stats; 3257 3258 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3259 uint16_t enq = 0, deq = 0; 3260 3261 if (unlikely(num_to_process - dequeued < burst_sz)) 3262 burst_sz = num_to_process - dequeued; 3263 3264 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3265 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3266 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3267 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 3268 bufs->inputs, 3269 bufs->hard_outputs, 3270 ref_op); 3271 3272 /* Start time meas for enqueue function offload latency */ 3273 enq_start_time = rte_rdtsc_precise(); 3274 do { 3275 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 3276 &ops_enq[enq], burst_sz - enq); 3277 } while (unlikely(burst_sz != enq)); 3278 3279 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3280 TEST_ASSERT_SUCCESS(ret, 3281 "Failed to get stats for queue (%u) of device (%u)", 3282 queue_id, dev_id); 3283 3284 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3285 stats.acc_offload_cycles; 3286 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3287 enq_sw_last_time); 3288 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3289 enq_sw_last_time); 3290 time_st->enq_sw_total_time += enq_sw_last_time; 3291 3292 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3293 stats.acc_offload_cycles); 3294 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3295 stats.acc_offload_cycles); 3296 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3297 3298 /* give time for device to process ops */ 3299 rte_delay_us(200); 3300 3301 /* Start time meas for dequeue function offload latency */ 3302 deq_start_time = rte_rdtsc_precise(); 3303 /* Dequeue one operation */ 3304 do { 3305 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3306 &ops_deq[deq], 1); 3307 } while (unlikely(deq != 1)); 3308 3309 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3310 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3311 deq_last_time); 3312 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3313 deq_last_time); 3314 time_st->deq_total_time += deq_last_time; 3315 3316 while (burst_sz != deq) 3317 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3318 &ops_deq[deq], burst_sz - deq); 3319 3320 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3321 dequeued += deq; 3322 } 3323 3324 return i; 3325 } 3326 #endif 3327 3328 static int 3329 offload_cost_test(struct active_device *ad, 3330 struct test_op_params *op_params) 3331 { 3332 #ifndef RTE_BBDEV_OFFLOAD_COST 3333 RTE_SET_USED(ad); 3334 RTE_SET_USED(op_params); 3335 printf("Offload latency test is disabled.\n"); 3336 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3337 return TEST_SKIPPED; 3338 #else 3339 int iter; 3340 uint16_t burst_sz = op_params->burst_sz; 3341 const uint16_t num_to_process = op_params->num_to_process; 3342 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3343 const uint16_t queue_id = ad->queue_ids[0]; 3344 struct test_buffers *bufs = NULL; 3345 struct rte_bbdev_info info; 3346 const char *op_type_str; 3347 struct test_time_stats time_st; 3348 3349 memset(&time_st, 0, sizeof(struct test_time_stats)); 3350 time_st.enq_sw_min_time = UINT64_MAX; 3351 time_st.enq_acc_min_time = UINT64_MAX; 3352 time_st.deq_min_time = UINT64_MAX; 3353 3354 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3355 "BURST_SIZE should be <= %u", MAX_BURST); 3356 3357 rte_bbdev_info_get(ad->dev_id, &info); 3358 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3359 3360 op_type_str = rte_bbdev_op_type_str(op_type); 3361 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3362 3363 printf("+ ------------------------------------------------------- +\n"); 3364 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3365 info.dev_name, burst_sz, num_to_process, op_type_str); 3366 3367 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3368 iter = offload_latency_test_dec(op_params->mp, bufs, 3369 op_params->ref_dec_op, ad->dev_id, queue_id, 3370 num_to_process, burst_sz, &time_st); 3371 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 3372 iter = offload_latency_test_enc(op_params->mp, bufs, 3373 op_params->ref_enc_op, ad->dev_id, queue_id, 3374 num_to_process, burst_sz, &time_st); 3375 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 3376 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 3377 op_params->ref_enc_op, ad->dev_id, queue_id, 3378 num_to_process, burst_sz, &time_st); 3379 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 3380 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 3381 op_params->ref_dec_op, ad->dev_id, queue_id, 3382 num_to_process, burst_sz, &time_st); 3383 else 3384 iter = offload_latency_test_enc(op_params->mp, bufs, 3385 op_params->ref_enc_op, ad->dev_id, queue_id, 3386 num_to_process, burst_sz, &time_st); 3387 3388 if (iter <= 0) 3389 return TEST_FAILED; 3390 3391 printf("Enqueue driver offload cost latency:\n" 3392 "\tavg: %lg cycles, %lg us\n" 3393 "\tmin: %lg cycles, %lg us\n" 3394 "\tmax: %lg cycles, %lg us\n" 3395 "Enqueue accelerator offload cost latency:\n" 3396 "\tavg: %lg cycles, %lg us\n" 3397 "\tmin: %lg cycles, %lg us\n" 3398 "\tmax: %lg cycles, %lg us\n", 3399 (double)time_st.enq_sw_total_time / (double)iter, 3400 (double)(time_st.enq_sw_total_time * 1000000) / 3401 (double)iter / (double)rte_get_tsc_hz(), 3402 (double)time_st.enq_sw_min_time, 3403 (double)(time_st.enq_sw_min_time * 1000000) / 3404 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 3405 (double)(time_st.enq_sw_max_time * 1000000) / 3406 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 3407 (double)iter, 3408 (double)(time_st.enq_acc_total_time * 1000000) / 3409 (double)iter / (double)rte_get_tsc_hz(), 3410 (double)time_st.enq_acc_min_time, 3411 (double)(time_st.enq_acc_min_time * 1000000) / 3412 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 3413 (double)(time_st.enq_acc_max_time * 1000000) / 3414 rte_get_tsc_hz()); 3415 3416 printf("Dequeue offload cost latency - one op:\n" 3417 "\tavg: %lg cycles, %lg us\n" 3418 "\tmin: %lg cycles, %lg us\n" 3419 "\tmax: %lg cycles, %lg us\n", 3420 (double)time_st.deq_total_time / (double)iter, 3421 (double)(time_st.deq_total_time * 1000000) / 3422 (double)iter / (double)rte_get_tsc_hz(), 3423 (double)time_st.deq_min_time, 3424 (double)(time_st.deq_min_time * 1000000) / 3425 rte_get_tsc_hz(), (double)time_st.deq_max_time, 3426 (double)(time_st.deq_max_time * 1000000) / 3427 rte_get_tsc_hz()); 3428 3429 return TEST_SUCCESS; 3430 #endif 3431 } 3432 3433 #ifdef RTE_BBDEV_OFFLOAD_COST 3434 static int 3435 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 3436 const uint16_t num_to_process, uint16_t burst_sz, 3437 uint64_t *deq_total_time, uint64_t *deq_min_time, 3438 uint64_t *deq_max_time) 3439 { 3440 int i, deq_total; 3441 struct rte_bbdev_dec_op *ops[MAX_BURST]; 3442 uint64_t deq_start_time, deq_last_time; 3443 3444 /* Test deq offload latency from an empty queue */ 3445 3446 for (i = 0, deq_total = 0; deq_total < num_to_process; 3447 ++i, deq_total += burst_sz) { 3448 deq_start_time = rte_rdtsc_precise(); 3449 3450 if (unlikely(num_to_process - deq_total < burst_sz)) 3451 burst_sz = num_to_process - deq_total; 3452 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 3453 3454 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3455 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3456 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3457 *deq_total_time += deq_last_time; 3458 } 3459 3460 return i; 3461 } 3462 3463 static int 3464 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 3465 const uint16_t num_to_process, uint16_t burst_sz, 3466 uint64_t *deq_total_time, uint64_t *deq_min_time, 3467 uint64_t *deq_max_time) 3468 { 3469 int i, deq_total; 3470 struct rte_bbdev_enc_op *ops[MAX_BURST]; 3471 uint64_t deq_start_time, deq_last_time; 3472 3473 /* Test deq offload latency from an empty queue */ 3474 for (i = 0, deq_total = 0; deq_total < num_to_process; 3475 ++i, deq_total += burst_sz) { 3476 deq_start_time = rte_rdtsc_precise(); 3477 3478 if (unlikely(num_to_process - deq_total < burst_sz)) 3479 burst_sz = num_to_process - deq_total; 3480 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 3481 3482 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3483 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3484 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3485 *deq_total_time += deq_last_time; 3486 } 3487 3488 return i; 3489 } 3490 #endif 3491 3492 static int 3493 offload_latency_empty_q_test(struct active_device *ad, 3494 struct test_op_params *op_params) 3495 { 3496 #ifndef RTE_BBDEV_OFFLOAD_COST 3497 RTE_SET_USED(ad); 3498 RTE_SET_USED(op_params); 3499 printf("Offload latency empty dequeue test is disabled.\n"); 3500 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3501 return TEST_SKIPPED; 3502 #else 3503 int iter; 3504 uint64_t deq_total_time, deq_min_time, deq_max_time; 3505 uint16_t burst_sz = op_params->burst_sz; 3506 const uint16_t num_to_process = op_params->num_to_process; 3507 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3508 const uint16_t queue_id = ad->queue_ids[0]; 3509 struct rte_bbdev_info info; 3510 const char *op_type_str; 3511 3512 deq_total_time = deq_max_time = 0; 3513 deq_min_time = UINT64_MAX; 3514 3515 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3516 "BURST_SIZE should be <= %u", MAX_BURST); 3517 3518 rte_bbdev_info_get(ad->dev_id, &info); 3519 3520 op_type_str = rte_bbdev_op_type_str(op_type); 3521 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3522 3523 printf("+ ------------------------------------------------------- +\n"); 3524 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3525 info.dev_name, burst_sz, num_to_process, op_type_str); 3526 3527 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3528 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 3529 num_to_process, burst_sz, &deq_total_time, 3530 &deq_min_time, &deq_max_time); 3531 else 3532 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 3533 num_to_process, burst_sz, &deq_total_time, 3534 &deq_min_time, &deq_max_time); 3535 3536 if (iter <= 0) 3537 return TEST_FAILED; 3538 3539 printf("Empty dequeue offload:\n" 3540 "\tavg: %lg cycles, %lg us\n" 3541 "\tmin: %lg cycles, %lg us\n" 3542 "\tmax: %lg cycles, %lg us\n", 3543 (double)deq_total_time / (double)iter, 3544 (double)(deq_total_time * 1000000) / (double)iter / 3545 (double)rte_get_tsc_hz(), (double)deq_min_time, 3546 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 3547 (double)deq_max_time, (double)(deq_max_time * 1000000) / 3548 rte_get_tsc_hz()); 3549 3550 return TEST_SUCCESS; 3551 #endif 3552 } 3553 3554 static int 3555 throughput_tc(void) 3556 { 3557 return run_test_case(throughput_test); 3558 } 3559 3560 static int 3561 offload_cost_tc(void) 3562 { 3563 return run_test_case(offload_cost_test); 3564 } 3565 3566 static int 3567 offload_latency_empty_q_tc(void) 3568 { 3569 return run_test_case(offload_latency_empty_q_test); 3570 } 3571 3572 static int 3573 latency_tc(void) 3574 { 3575 return run_test_case(latency_test); 3576 } 3577 3578 static int 3579 interrupt_tc(void) 3580 { 3581 return run_test_case(throughput_test); 3582 } 3583 3584 static struct unit_test_suite bbdev_throughput_testsuite = { 3585 .suite_name = "BBdev Throughput Tests", 3586 .setup = testsuite_setup, 3587 .teardown = testsuite_teardown, 3588 .unit_test_cases = { 3589 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 3590 TEST_CASES_END() /**< NULL terminate unit test array */ 3591 } 3592 }; 3593 3594 static struct unit_test_suite bbdev_validation_testsuite = { 3595 .suite_name = "BBdev Validation Tests", 3596 .setup = testsuite_setup, 3597 .teardown = testsuite_teardown, 3598 .unit_test_cases = { 3599 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3600 TEST_CASES_END() /**< NULL terminate unit test array */ 3601 } 3602 }; 3603 3604 static struct unit_test_suite bbdev_latency_testsuite = { 3605 .suite_name = "BBdev Latency Tests", 3606 .setup = testsuite_setup, 3607 .teardown = testsuite_teardown, 3608 .unit_test_cases = { 3609 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3610 TEST_CASES_END() /**< NULL terminate unit test array */ 3611 } 3612 }; 3613 3614 static struct unit_test_suite bbdev_offload_cost_testsuite = { 3615 .suite_name = "BBdev Offload Cost Tests", 3616 .setup = testsuite_setup, 3617 .teardown = testsuite_teardown, 3618 .unit_test_cases = { 3619 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 3620 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 3621 TEST_CASES_END() /**< NULL terminate unit test array */ 3622 } 3623 }; 3624 3625 static struct unit_test_suite bbdev_interrupt_testsuite = { 3626 .suite_name = "BBdev Interrupt Tests", 3627 .setup = interrupt_testsuite_setup, 3628 .teardown = testsuite_teardown, 3629 .unit_test_cases = { 3630 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 3631 TEST_CASES_END() /**< NULL terminate unit test array */ 3632 } 3633 }; 3634 3635 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 3636 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 3637 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 3638 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 3639 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 3640