1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #include "main.h" 22 #include "test_bbdev_vector.h" 23 24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 25 26 #define MAX_QUEUES RTE_MAX_LCORE 27 #define TEST_REPETITIONS 1000 28 29 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 30 #include <fpga_lte_fec.h> 31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 33 #define VF_UL_4G_QUEUE_VALUE 4 34 #define VF_DL_4G_QUEUE_VALUE 4 35 #define UL_4G_BANDWIDTH 3 36 #define DL_4G_BANDWIDTH 3 37 #define UL_4G_LOAD_BALANCE 128 38 #define DL_4G_LOAD_BALANCE 128 39 #define FLR_4G_TIMEOUT 610 40 #endif 41 42 #define OPS_CACHE_SIZE 256U 43 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 44 45 #define SYNC_WAIT 0 46 #define SYNC_START 1 47 48 #define INVALID_QUEUE_ID -1 49 50 static struct test_bbdev_vector test_vector; 51 52 /* Switch between PMD and Interrupt for throughput TC */ 53 static bool intr_enabled; 54 55 /* Represents tested active devices */ 56 static struct active_device { 57 const char *driver_name; 58 uint8_t dev_id; 59 uint16_t supported_ops; 60 uint16_t queue_ids[MAX_QUEUES]; 61 uint16_t nb_queues; 62 struct rte_mempool *ops_mempool; 63 struct rte_mempool *in_mbuf_pool; 64 struct rte_mempool *hard_out_mbuf_pool; 65 struct rte_mempool *soft_out_mbuf_pool; 66 struct rte_mempool *harq_in_mbuf_pool; 67 struct rte_mempool *harq_out_mbuf_pool; 68 } active_devs[RTE_BBDEV_MAX_DEVS]; 69 70 static uint8_t nb_active_devs; 71 72 /* Data buffers used by BBDEV ops */ 73 struct test_buffers { 74 struct rte_bbdev_op_data *inputs; 75 struct rte_bbdev_op_data *hard_outputs; 76 struct rte_bbdev_op_data *soft_outputs; 77 struct rte_bbdev_op_data *harq_inputs; 78 struct rte_bbdev_op_data *harq_outputs; 79 }; 80 81 /* Operation parameters specific for given test case */ 82 struct test_op_params { 83 struct rte_mempool *mp; 84 struct rte_bbdev_dec_op *ref_dec_op; 85 struct rte_bbdev_enc_op *ref_enc_op; 86 uint16_t burst_sz; 87 uint16_t num_to_process; 88 uint16_t num_lcores; 89 int vector_mask; 90 rte_atomic16_t sync; 91 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 92 }; 93 94 /* Contains per lcore params */ 95 struct thread_params { 96 uint8_t dev_id; 97 uint16_t queue_id; 98 uint32_t lcore_id; 99 uint64_t start_time; 100 double ops_per_sec; 101 double mbps; 102 uint8_t iter_count; 103 rte_atomic16_t nb_dequeued; 104 rte_atomic16_t processing_status; 105 rte_atomic16_t burst_sz; 106 struct test_op_params *op_params; 107 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 108 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 109 }; 110 111 #ifdef RTE_BBDEV_OFFLOAD_COST 112 /* Stores time statistics */ 113 struct test_time_stats { 114 /* Stores software enqueue total working time */ 115 uint64_t enq_sw_total_time; 116 /* Stores minimum value of software enqueue working time */ 117 uint64_t enq_sw_min_time; 118 /* Stores maximum value of software enqueue working time */ 119 uint64_t enq_sw_max_time; 120 /* Stores turbo enqueue total working time */ 121 uint64_t enq_acc_total_time; 122 /* Stores minimum value of accelerator enqueue working time */ 123 uint64_t enq_acc_min_time; 124 /* Stores maximum value of accelerator enqueue working time */ 125 uint64_t enq_acc_max_time; 126 /* Stores dequeue total working time */ 127 uint64_t deq_total_time; 128 /* Stores minimum value of dequeue working time */ 129 uint64_t deq_min_time; 130 /* Stores maximum value of dequeue working time */ 131 uint64_t deq_max_time; 132 }; 133 #endif 134 135 typedef int (test_case_function)(struct active_device *ad, 136 struct test_op_params *op_params); 137 138 static inline void 139 mbuf_reset(struct rte_mbuf *m) 140 { 141 m->pkt_len = 0; 142 143 do { 144 m->data_len = 0; 145 m = m->next; 146 } while (m != NULL); 147 } 148 149 /* Read flag value 0/1 from bitmap */ 150 static inline bool 151 check_bit(uint32_t bitmap, uint32_t bitmask) 152 { 153 return bitmap & bitmask; 154 } 155 156 static inline void 157 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 158 { 159 ad->supported_ops |= (1 << op_type); 160 } 161 162 static inline bool 163 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 164 { 165 return ad->supported_ops & (1 << op_type); 166 } 167 168 static inline bool 169 flags_match(uint32_t flags_req, uint32_t flags_present) 170 { 171 return (flags_req & flags_present) == flags_req; 172 } 173 174 static void 175 clear_soft_out_cap(uint32_t *op_flags) 176 { 177 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 178 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 179 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 180 } 181 182 static int 183 check_dev_cap(const struct rte_bbdev_info *dev_info) 184 { 185 unsigned int i; 186 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 187 nb_harq_inputs, nb_harq_outputs; 188 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 189 190 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 191 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 192 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 193 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 194 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 195 196 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 197 if (op_cap->type != test_vector.op_type) 198 continue; 199 200 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 201 const struct rte_bbdev_op_cap_turbo_dec *cap = 202 &op_cap->cap.turbo_dec; 203 /* Ignore lack of soft output capability, just skip 204 * checking if soft output is valid. 205 */ 206 if ((test_vector.turbo_dec.op_flags & 207 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 208 !(cap->capability_flags & 209 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 210 printf( 211 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 212 dev_info->dev_name); 213 clear_soft_out_cap( 214 &test_vector.turbo_dec.op_flags); 215 } 216 217 if (!flags_match(test_vector.turbo_dec.op_flags, 218 cap->capability_flags)) 219 return TEST_FAILED; 220 if (nb_inputs > cap->num_buffers_src) { 221 printf("Too many inputs defined: %u, max: %u\n", 222 nb_inputs, cap->num_buffers_src); 223 return TEST_FAILED; 224 } 225 if (nb_soft_outputs > cap->num_buffers_soft_out && 226 (test_vector.turbo_dec.op_flags & 227 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 228 printf( 229 "Too many soft outputs defined: %u, max: %u\n", 230 nb_soft_outputs, 231 cap->num_buffers_soft_out); 232 return TEST_FAILED; 233 } 234 if (nb_hard_outputs > cap->num_buffers_hard_out) { 235 printf( 236 "Too many hard outputs defined: %u, max: %u\n", 237 nb_hard_outputs, 238 cap->num_buffers_hard_out); 239 return TEST_FAILED; 240 } 241 if (intr_enabled && !(cap->capability_flags & 242 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 243 printf( 244 "Dequeue interrupts are not supported!\n"); 245 return TEST_FAILED; 246 } 247 248 return TEST_SUCCESS; 249 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 250 const struct rte_bbdev_op_cap_turbo_enc *cap = 251 &op_cap->cap.turbo_enc; 252 253 if (!flags_match(test_vector.turbo_enc.op_flags, 254 cap->capability_flags)) 255 return TEST_FAILED; 256 if (nb_inputs > cap->num_buffers_src) { 257 printf("Too many inputs defined: %u, max: %u\n", 258 nb_inputs, cap->num_buffers_src); 259 return TEST_FAILED; 260 } 261 if (nb_hard_outputs > cap->num_buffers_dst) { 262 printf( 263 "Too many hard outputs defined: %u, max: %u\n", 264 nb_hard_outputs, cap->num_buffers_dst); 265 return TEST_FAILED; 266 } 267 if (intr_enabled && !(cap->capability_flags & 268 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 269 printf( 270 "Dequeue interrupts are not supported!\n"); 271 return TEST_FAILED; 272 } 273 274 return TEST_SUCCESS; 275 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 276 const struct rte_bbdev_op_cap_ldpc_enc *cap = 277 &op_cap->cap.ldpc_enc; 278 279 if (!flags_match(test_vector.ldpc_enc.op_flags, 280 cap->capability_flags)){ 281 printf("Flag Mismatch\n"); 282 return TEST_FAILED; 283 } 284 if (nb_inputs > cap->num_buffers_src) { 285 printf("Too many inputs defined: %u, max: %u\n", 286 nb_inputs, cap->num_buffers_src); 287 return TEST_FAILED; 288 } 289 if (nb_hard_outputs > cap->num_buffers_dst) { 290 printf( 291 "Too many hard outputs defined: %u, max: %u\n", 292 nb_hard_outputs, cap->num_buffers_dst); 293 return TEST_FAILED; 294 } 295 if (intr_enabled && !(cap->capability_flags & 296 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 297 printf( 298 "Dequeue interrupts are not supported!\n"); 299 return TEST_FAILED; 300 } 301 302 return TEST_SUCCESS; 303 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 304 const struct rte_bbdev_op_cap_ldpc_dec *cap = 305 &op_cap->cap.ldpc_dec; 306 307 if (!flags_match(test_vector.ldpc_dec.op_flags, 308 cap->capability_flags)){ 309 printf("Flag Mismatch\n"); 310 return TEST_FAILED; 311 } 312 if (nb_inputs > cap->num_buffers_src) { 313 printf("Too many inputs defined: %u, max: %u\n", 314 nb_inputs, cap->num_buffers_src); 315 return TEST_FAILED; 316 } 317 if (nb_hard_outputs > cap->num_buffers_hard_out) { 318 printf( 319 "Too many hard outputs defined: %u, max: %u\n", 320 nb_hard_outputs, 321 cap->num_buffers_hard_out); 322 return TEST_FAILED; 323 } 324 if (nb_harq_inputs > cap->num_buffers_hard_out) { 325 printf( 326 "Too many HARQ inputs defined: %u, max: %u\n", 327 nb_hard_outputs, 328 cap->num_buffers_hard_out); 329 return TEST_FAILED; 330 } 331 if (nb_harq_outputs > cap->num_buffers_hard_out) { 332 printf( 333 "Too many HARQ outputs defined: %u, max: %u\n", 334 nb_hard_outputs, 335 cap->num_buffers_hard_out); 336 return TEST_FAILED; 337 } 338 if (intr_enabled && !(cap->capability_flags & 339 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 340 printf( 341 "Dequeue interrupts are not supported!\n"); 342 return TEST_FAILED; 343 } 344 345 return TEST_SUCCESS; 346 } 347 } 348 349 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 350 return TEST_SUCCESS; /* Special case for NULL device */ 351 352 return TEST_FAILED; 353 } 354 355 /* calculates optimal mempool size not smaller than the val */ 356 static unsigned int 357 optimal_mempool_size(unsigned int val) 358 { 359 return rte_align32pow2(val + 1) - 1; 360 } 361 362 /* allocates mbuf mempool for inputs and outputs */ 363 static struct rte_mempool * 364 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 365 int socket_id, unsigned int mbuf_pool_size, 366 const char *op_type_str) 367 { 368 unsigned int i; 369 uint32_t max_seg_sz = 0; 370 char pool_name[RTE_MEMPOOL_NAMESIZE]; 371 372 /* find max input segment size */ 373 for (i = 0; i < entries->nb_segments; ++i) 374 if (entries->segments[i].length > max_seg_sz) 375 max_seg_sz = entries->segments[i].length; 376 377 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 378 dev_id); 379 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 380 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, 381 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 382 } 383 384 static int 385 create_mempools(struct active_device *ad, int socket_id, 386 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 387 { 388 struct rte_mempool *mp; 389 unsigned int ops_pool_size, mbuf_pool_size = 0; 390 char pool_name[RTE_MEMPOOL_NAMESIZE]; 391 const char *op_type_str; 392 enum rte_bbdev_op_type op_type = org_op_type; 393 394 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 395 struct op_data_entries *hard_out = 396 &test_vector.entries[DATA_HARD_OUTPUT]; 397 struct op_data_entries *soft_out = 398 &test_vector.entries[DATA_SOFT_OUTPUT]; 399 struct op_data_entries *harq_in = 400 &test_vector.entries[DATA_HARQ_INPUT]; 401 struct op_data_entries *harq_out = 402 &test_vector.entries[DATA_HARQ_OUTPUT]; 403 404 /* allocate ops mempool */ 405 ops_pool_size = optimal_mempool_size(RTE_MAX( 406 /* Ops used plus 1 reference op */ 407 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 408 /* Minimal cache size plus 1 reference op */ 409 (unsigned int)(1.5 * rte_lcore_count() * 410 OPS_CACHE_SIZE + 1)), 411 OPS_POOL_SIZE_MIN)); 412 413 if (org_op_type == RTE_BBDEV_OP_NONE) 414 op_type = RTE_BBDEV_OP_TURBO_ENC; 415 416 op_type_str = rte_bbdev_op_type_str(op_type); 417 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 418 419 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 420 ad->dev_id); 421 mp = rte_bbdev_op_pool_create(pool_name, op_type, 422 ops_pool_size, OPS_CACHE_SIZE, socket_id); 423 TEST_ASSERT_NOT_NULL(mp, 424 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 425 ops_pool_size, 426 ad->dev_id, 427 socket_id); 428 ad->ops_mempool = mp; 429 430 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 431 if (org_op_type == RTE_BBDEV_OP_NONE) 432 return TEST_SUCCESS; 433 434 /* Inputs */ 435 mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); 436 mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); 437 TEST_ASSERT_NOT_NULL(mp, 438 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 439 mbuf_pool_size, 440 ad->dev_id, 441 socket_id); 442 ad->in_mbuf_pool = mp; 443 444 /* Hard outputs */ 445 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 446 hard_out->nb_segments); 447 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, 448 "hard_out"); 449 TEST_ASSERT_NOT_NULL(mp, 450 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 451 mbuf_pool_size, 452 ad->dev_id, 453 socket_id); 454 ad->hard_out_mbuf_pool = mp; 455 456 457 /* Soft outputs */ 458 if (soft_out->nb_segments > 0) { 459 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 460 soft_out->nb_segments); 461 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 462 mbuf_pool_size, 463 "soft_out"); 464 TEST_ASSERT_NOT_NULL(mp, 465 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 466 mbuf_pool_size, 467 ad->dev_id, 468 socket_id); 469 ad->soft_out_mbuf_pool = mp; 470 } 471 472 /* HARQ inputs */ 473 if (harq_in->nb_segments > 0) { 474 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 475 harq_in->nb_segments); 476 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 477 mbuf_pool_size, 478 "harq_in"); 479 TEST_ASSERT_NOT_NULL(mp, 480 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 481 mbuf_pool_size, 482 ad->dev_id, 483 socket_id); 484 ad->harq_in_mbuf_pool = mp; 485 } 486 487 /* HARQ outputs */ 488 if (harq_out->nb_segments > 0) { 489 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 490 harq_out->nb_segments); 491 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 492 mbuf_pool_size, 493 "harq_out"); 494 TEST_ASSERT_NOT_NULL(mp, 495 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 496 mbuf_pool_size, 497 ad->dev_id, 498 socket_id); 499 ad->harq_out_mbuf_pool = mp; 500 } 501 502 return TEST_SUCCESS; 503 } 504 505 static int 506 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 507 struct test_bbdev_vector *vector) 508 { 509 int ret; 510 unsigned int queue_id; 511 struct rte_bbdev_queue_conf qconf; 512 struct active_device *ad = &active_devs[nb_active_devs]; 513 unsigned int nb_queues; 514 enum rte_bbdev_op_type op_type = vector->op_type; 515 516 /* Configure fpga lte fec with PF & VF values 517 * if '-i' flag is set and using fpga device 518 */ 519 #ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC 520 if ((get_init_device() == true) && 521 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 522 struct fpga_lte_fec_conf conf; 523 unsigned int i; 524 525 printf("Configure FPGA LTE FEC Driver %s with default values\n", 526 info->drv.driver_name); 527 528 /* clear default configuration before initialization */ 529 memset(&conf, 0, sizeof(struct fpga_lte_fec_conf)); 530 531 /* Set PF mode : 532 * true if PF is used for data plane 533 * false for VFs 534 */ 535 conf.pf_mode_en = true; 536 537 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 538 /* Number of UL queues per VF (fpga supports 8 VFs) */ 539 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 540 /* Number of DL queues per VF (fpga supports 8 VFs) */ 541 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 542 } 543 544 /* UL bandwidth. Needed for schedule algorithm */ 545 conf.ul_bandwidth = UL_4G_BANDWIDTH; 546 /* DL bandwidth */ 547 conf.dl_bandwidth = DL_4G_BANDWIDTH; 548 549 /* UL & DL load Balance Factor to 64 */ 550 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 551 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 552 553 /**< FLR timeout value */ 554 conf.flr_time_out = FLR_4G_TIMEOUT; 555 556 /* setup FPGA PF with configuration information */ 557 ret = fpga_lte_fec_configure(info->dev_name, &conf); 558 TEST_ASSERT_SUCCESS(ret, 559 "Failed to configure 4G FPGA PF for bbdev %s", 560 info->dev_name); 561 } 562 #endif 563 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 564 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 565 566 /* setup device */ 567 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 568 if (ret < 0) { 569 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 570 dev_id, nb_queues, info->socket_id, ret); 571 return TEST_FAILED; 572 } 573 574 /* configure interrupts if needed */ 575 if (intr_enabled) { 576 ret = rte_bbdev_intr_enable(dev_id); 577 if (ret < 0) { 578 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 579 ret); 580 return TEST_FAILED; 581 } 582 } 583 584 /* setup device queues */ 585 qconf.socket = info->socket_id; 586 qconf.queue_size = info->drv.default_queue_conf.queue_size; 587 qconf.priority = 0; 588 qconf.deferred_start = 0; 589 qconf.op_type = op_type; 590 591 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 592 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 593 if (ret != 0) { 594 printf( 595 "Allocated all queues (id=%u) at prio%u on dev%u\n", 596 queue_id, qconf.priority, dev_id); 597 qconf.priority++; 598 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 599 &qconf); 600 } 601 if (ret != 0) { 602 printf("All queues on dev %u allocated: %u\n", 603 dev_id, queue_id); 604 break; 605 } 606 ad->queue_ids[queue_id] = queue_id; 607 } 608 TEST_ASSERT(queue_id != 0, 609 "ERROR Failed to configure any queues on dev %u", 610 dev_id); 611 ad->nb_queues = queue_id; 612 613 set_avail_op(ad, op_type); 614 615 return TEST_SUCCESS; 616 } 617 618 static int 619 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 620 struct test_bbdev_vector *vector) 621 { 622 int ret; 623 624 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 625 active_devs[nb_active_devs].dev_id = dev_id; 626 627 ret = add_bbdev_dev(dev_id, info, vector); 628 if (ret == TEST_SUCCESS) 629 ++nb_active_devs; 630 return ret; 631 } 632 633 static uint8_t 634 populate_active_devices(void) 635 { 636 int ret; 637 uint8_t dev_id; 638 uint8_t nb_devs_added = 0; 639 struct rte_bbdev_info info; 640 641 RTE_BBDEV_FOREACH(dev_id) { 642 rte_bbdev_info_get(dev_id, &info); 643 644 if (check_dev_cap(&info)) { 645 printf( 646 "Device %d (%s) does not support specified capabilities\n", 647 dev_id, info.dev_name); 648 continue; 649 } 650 651 ret = add_active_device(dev_id, &info, &test_vector); 652 if (ret != 0) { 653 printf("Adding active bbdev %s skipped\n", 654 info.dev_name); 655 continue; 656 } 657 nb_devs_added++; 658 } 659 660 return nb_devs_added; 661 } 662 663 static int 664 read_test_vector(void) 665 { 666 int ret; 667 668 memset(&test_vector, 0, sizeof(test_vector)); 669 printf("Test vector file = %s\n", get_vector_filename()); 670 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 671 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 672 get_vector_filename()); 673 674 return TEST_SUCCESS; 675 } 676 677 static int 678 testsuite_setup(void) 679 { 680 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 681 682 if (populate_active_devices() == 0) { 683 printf("No suitable devices found!\n"); 684 return TEST_SKIPPED; 685 } 686 687 return TEST_SUCCESS; 688 } 689 690 static int 691 interrupt_testsuite_setup(void) 692 { 693 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 694 695 /* Enable interrupts */ 696 intr_enabled = true; 697 698 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 699 if (populate_active_devices() == 0 || 700 test_vector.op_type == RTE_BBDEV_OP_NONE) { 701 intr_enabled = false; 702 printf("No suitable devices found!\n"); 703 return TEST_SKIPPED; 704 } 705 706 return TEST_SUCCESS; 707 } 708 709 static void 710 testsuite_teardown(void) 711 { 712 uint8_t dev_id; 713 714 /* Unconfigure devices */ 715 RTE_BBDEV_FOREACH(dev_id) 716 rte_bbdev_close(dev_id); 717 718 /* Clear active devices structs. */ 719 memset(active_devs, 0, sizeof(active_devs)); 720 nb_active_devs = 0; 721 } 722 723 static int 724 ut_setup(void) 725 { 726 uint8_t i, dev_id; 727 728 for (i = 0; i < nb_active_devs; i++) { 729 dev_id = active_devs[i].dev_id; 730 /* reset bbdev stats */ 731 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 732 "Failed to reset stats of bbdev %u", dev_id); 733 /* start the device */ 734 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 735 "Failed to start bbdev %u", dev_id); 736 } 737 738 return TEST_SUCCESS; 739 } 740 741 static void 742 ut_teardown(void) 743 { 744 uint8_t i, dev_id; 745 struct rte_bbdev_stats stats; 746 747 for (i = 0; i < nb_active_devs; i++) { 748 dev_id = active_devs[i].dev_id; 749 /* read stats and print */ 750 rte_bbdev_stats_get(dev_id, &stats); 751 /* Stop the device */ 752 rte_bbdev_stop(dev_id); 753 } 754 } 755 756 static int 757 init_op_data_objs(struct rte_bbdev_op_data *bufs, 758 struct op_data_entries *ref_entries, 759 struct rte_mempool *mbuf_pool, const uint16_t n, 760 enum op_data_type op_type, uint16_t min_alignment) 761 { 762 int ret; 763 unsigned int i, j; 764 bool large_input = false; 765 766 for (i = 0; i < n; ++i) { 767 char *data; 768 struct op_data_buf *seg = &ref_entries->segments[0]; 769 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 770 TEST_ASSERT_NOT_NULL(m_head, 771 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 772 op_type, n * ref_entries->nb_segments, 773 mbuf_pool->size); 774 775 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) { 776 /* 777 * Special case when DPDK mbuf cannot handle 778 * the required input size 779 */ 780 printf("Warning: Larger input size than DPDK mbuf %d\n", 781 seg->length); 782 large_input = true; 783 } 784 bufs[i].data = m_head; 785 bufs[i].offset = 0; 786 bufs[i].length = 0; 787 788 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 789 if ((op_type == DATA_INPUT) && large_input) { 790 /* Allocate a fake overused mbuf */ 791 data = rte_malloc(NULL, seg->length, 0); 792 memcpy(data, seg->addr, seg->length); 793 m_head->buf_addr = data; 794 m_head->buf_iova = rte_malloc_virt2iova(data); 795 m_head->data_off = 0; 796 m_head->data_len = seg->length; 797 } else { 798 data = rte_pktmbuf_append(m_head, seg->length); 799 TEST_ASSERT_NOT_NULL(data, 800 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 801 seg->length, op_type); 802 803 TEST_ASSERT(data == RTE_PTR_ALIGN( 804 data, min_alignment), 805 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 806 data, min_alignment); 807 rte_memcpy(data, seg->addr, seg->length); 808 } 809 810 bufs[i].length += seg->length; 811 812 for (j = 1; j < ref_entries->nb_segments; ++j) { 813 struct rte_mbuf *m_tail = 814 rte_pktmbuf_alloc(mbuf_pool); 815 TEST_ASSERT_NOT_NULL(m_tail, 816 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 817 op_type, 818 n * ref_entries->nb_segments, 819 mbuf_pool->size); 820 seg += 1; 821 822 data = rte_pktmbuf_append(m_tail, seg->length); 823 TEST_ASSERT_NOT_NULL(data, 824 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 825 seg->length, op_type); 826 827 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 828 min_alignment), 829 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 830 data, min_alignment); 831 rte_memcpy(data, seg->addr, seg->length); 832 bufs[i].length += seg->length; 833 834 ret = rte_pktmbuf_chain(m_head, m_tail); 835 TEST_ASSERT_SUCCESS(ret, 836 "Couldn't chain mbufs from %d data type mbuf pool", 837 op_type); 838 } 839 } else { 840 841 /* allocate chained-mbuf for output buffer */ 842 for (j = 1; j < ref_entries->nb_segments; ++j) { 843 struct rte_mbuf *m_tail = 844 rte_pktmbuf_alloc(mbuf_pool); 845 TEST_ASSERT_NOT_NULL(m_tail, 846 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 847 op_type, 848 n * ref_entries->nb_segments, 849 mbuf_pool->size); 850 851 ret = rte_pktmbuf_chain(m_head, m_tail); 852 TEST_ASSERT_SUCCESS(ret, 853 "Couldn't chain mbufs from %d data type mbuf pool", 854 op_type); 855 } 856 } 857 } 858 859 return 0; 860 } 861 862 static int 863 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 864 const int socket) 865 { 866 int i; 867 868 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 869 if (*buffers == NULL) { 870 printf("WARNING: Failed to allocate op_data on socket %d\n", 871 socket); 872 /* try to allocate memory on other detected sockets */ 873 for (i = 0; i < socket; i++) { 874 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 875 if (*buffers != NULL) 876 break; 877 } 878 } 879 880 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 881 } 882 883 static void 884 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 885 const uint16_t n, const int8_t max_llr_modulus) 886 { 887 uint16_t i, byte_idx; 888 889 for (i = 0; i < n; ++i) { 890 struct rte_mbuf *m = input_ops[i].data; 891 while (m != NULL) { 892 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 893 input_ops[i].offset); 894 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 895 ++byte_idx) 896 llr[byte_idx] = round((double)max_llr_modulus * 897 llr[byte_idx] / INT8_MAX); 898 899 m = m->next; 900 } 901 } 902 } 903 904 static void 905 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 906 const uint16_t n, const int8_t llr_size, 907 const int8_t llr_decimals) 908 { 909 if (input_ops == NULL) 910 return; 911 912 uint16_t i, byte_idx; 913 914 int16_t llr_max, llr_min, llr_tmp; 915 llr_max = (1 << (llr_size - 1)) - 1; 916 llr_min = -llr_max; 917 for (i = 0; i < n; ++i) { 918 struct rte_mbuf *m = input_ops[i].data; 919 while (m != NULL) { 920 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 921 input_ops[i].offset); 922 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 923 ++byte_idx) { 924 925 llr_tmp = llr[byte_idx]; 926 if (llr_decimals == 2) 927 llr_tmp *= 2; 928 else if (llr_decimals == 0) 929 llr_tmp /= 2; 930 llr_tmp = RTE_MIN(llr_max, 931 RTE_MAX(llr_min, llr_tmp)); 932 llr[byte_idx] = (int8_t) llr_tmp; 933 } 934 935 m = m->next; 936 } 937 } 938 } 939 940 941 942 static int 943 fill_queue_buffers(struct test_op_params *op_params, 944 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 945 struct rte_mempool *soft_out_mp, 946 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 947 uint16_t queue_id, 948 const struct rte_bbdev_op_cap *capabilities, 949 uint16_t min_alignment, const int socket_id) 950 { 951 int ret; 952 enum op_data_type type; 953 const uint16_t n = op_params->num_to_process; 954 955 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 956 in_mp, 957 soft_out_mp, 958 hard_out_mp, 959 harq_in_mp, 960 harq_out_mp, 961 }; 962 963 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 964 &op_params->q_bufs[socket_id][queue_id].inputs, 965 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 966 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 967 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 968 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 969 }; 970 971 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 972 struct op_data_entries *ref_entries = 973 &test_vector.entries[type]; 974 if (ref_entries->nb_segments == 0) 975 continue; 976 977 ret = allocate_buffers_on_socket(queue_ops[type], 978 n * sizeof(struct rte_bbdev_op_data), 979 socket_id); 980 TEST_ASSERT_SUCCESS(ret, 981 "Couldn't allocate memory for rte_bbdev_op_data structs"); 982 983 ret = init_op_data_objs(*queue_ops[type], ref_entries, 984 mbuf_pools[type], n, type, min_alignment); 985 TEST_ASSERT_SUCCESS(ret, 986 "Couldn't init rte_bbdev_op_data structs"); 987 } 988 989 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 990 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 991 capabilities->cap.turbo_dec.max_llr_modulus); 992 993 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 994 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 995 capabilities->cap.ldpc_dec.llr_size, 996 capabilities->cap.ldpc_dec.llr_decimals); 997 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 998 capabilities->cap.ldpc_dec.llr_size, 999 capabilities->cap.ldpc_dec.llr_decimals); 1000 } 1001 1002 return 0; 1003 } 1004 1005 static void 1006 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1007 { 1008 unsigned int i, j; 1009 1010 rte_mempool_free(ad->ops_mempool); 1011 rte_mempool_free(ad->in_mbuf_pool); 1012 rte_mempool_free(ad->hard_out_mbuf_pool); 1013 rte_mempool_free(ad->soft_out_mbuf_pool); 1014 rte_mempool_free(ad->harq_in_mbuf_pool); 1015 rte_mempool_free(ad->harq_out_mbuf_pool); 1016 1017 for (i = 0; i < rte_lcore_count(); ++i) { 1018 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1019 rte_free(op_params->q_bufs[j][i].inputs); 1020 rte_free(op_params->q_bufs[j][i].hard_outputs); 1021 rte_free(op_params->q_bufs[j][i].soft_outputs); 1022 rte_free(op_params->q_bufs[j][i].harq_inputs); 1023 rte_free(op_params->q_bufs[j][i].harq_outputs); 1024 } 1025 } 1026 } 1027 1028 static void 1029 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1030 unsigned int start_idx, 1031 struct rte_bbdev_op_data *inputs, 1032 struct rte_bbdev_op_data *hard_outputs, 1033 struct rte_bbdev_op_data *soft_outputs, 1034 struct rte_bbdev_dec_op *ref_op) 1035 { 1036 unsigned int i; 1037 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1038 1039 for (i = 0; i < n; ++i) { 1040 if (turbo_dec->code_block_mode == 0) { 1041 ops[i]->turbo_dec.tb_params.ea = 1042 turbo_dec->tb_params.ea; 1043 ops[i]->turbo_dec.tb_params.eb = 1044 turbo_dec->tb_params.eb; 1045 ops[i]->turbo_dec.tb_params.k_pos = 1046 turbo_dec->tb_params.k_pos; 1047 ops[i]->turbo_dec.tb_params.k_neg = 1048 turbo_dec->tb_params.k_neg; 1049 ops[i]->turbo_dec.tb_params.c = 1050 turbo_dec->tb_params.c; 1051 ops[i]->turbo_dec.tb_params.c_neg = 1052 turbo_dec->tb_params.c_neg; 1053 ops[i]->turbo_dec.tb_params.cab = 1054 turbo_dec->tb_params.cab; 1055 ops[i]->turbo_dec.tb_params.r = 1056 turbo_dec->tb_params.r; 1057 } else { 1058 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1059 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1060 } 1061 1062 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1063 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1064 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1065 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1066 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1067 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1068 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1069 1070 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1071 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1072 if (soft_outputs != NULL) 1073 ops[i]->turbo_dec.soft_output = 1074 soft_outputs[start_idx + i]; 1075 } 1076 } 1077 1078 static void 1079 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1080 unsigned int start_idx, 1081 struct rte_bbdev_op_data *inputs, 1082 struct rte_bbdev_op_data *outputs, 1083 struct rte_bbdev_enc_op *ref_op) 1084 { 1085 unsigned int i; 1086 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1087 for (i = 0; i < n; ++i) { 1088 if (turbo_enc->code_block_mode == 0) { 1089 ops[i]->turbo_enc.tb_params.ea = 1090 turbo_enc->tb_params.ea; 1091 ops[i]->turbo_enc.tb_params.eb = 1092 turbo_enc->tb_params.eb; 1093 ops[i]->turbo_enc.tb_params.k_pos = 1094 turbo_enc->tb_params.k_pos; 1095 ops[i]->turbo_enc.tb_params.k_neg = 1096 turbo_enc->tb_params.k_neg; 1097 ops[i]->turbo_enc.tb_params.c = 1098 turbo_enc->tb_params.c; 1099 ops[i]->turbo_enc.tb_params.c_neg = 1100 turbo_enc->tb_params.c_neg; 1101 ops[i]->turbo_enc.tb_params.cab = 1102 turbo_enc->tb_params.cab; 1103 ops[i]->turbo_enc.tb_params.ncb_pos = 1104 turbo_enc->tb_params.ncb_pos; 1105 ops[i]->turbo_enc.tb_params.ncb_neg = 1106 turbo_enc->tb_params.ncb_neg; 1107 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1108 } else { 1109 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1110 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1111 ops[i]->turbo_enc.cb_params.ncb = 1112 turbo_enc->cb_params.ncb; 1113 } 1114 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1115 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1116 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1117 1118 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1119 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1120 } 1121 } 1122 1123 static void 1124 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1125 unsigned int start_idx, 1126 struct rte_bbdev_op_data *inputs, 1127 struct rte_bbdev_op_data *hard_outputs, 1128 struct rte_bbdev_op_data *soft_outputs, 1129 struct rte_bbdev_op_data *harq_inputs, 1130 struct rte_bbdev_op_data *harq_outputs, 1131 struct rte_bbdev_dec_op *ref_op) 1132 { 1133 unsigned int i; 1134 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1135 1136 for (i = 0; i < n; ++i) { 1137 if (ldpc_dec->code_block_mode == 0) { 1138 ops[i]->ldpc_dec.tb_params.ea = 1139 ldpc_dec->tb_params.ea; 1140 ops[i]->ldpc_dec.tb_params.eb = 1141 ldpc_dec->tb_params.eb; 1142 ops[i]->ldpc_dec.tb_params.c = 1143 ldpc_dec->tb_params.c; 1144 ops[i]->ldpc_dec.tb_params.cab = 1145 ldpc_dec->tb_params.cab; 1146 ops[i]->ldpc_dec.tb_params.r = 1147 ldpc_dec->tb_params.r; 1148 } else { 1149 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1150 } 1151 1152 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1153 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1154 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1155 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1156 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1157 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1158 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1159 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1160 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1161 1162 ops[i]->ldpc_dec.hard_output = hard_outputs[start_idx + i]; 1163 ops[i]->ldpc_dec.input = inputs[start_idx + i]; 1164 if (soft_outputs != NULL) 1165 ops[i]->ldpc_dec.soft_output = 1166 soft_outputs[start_idx + i]; 1167 if (harq_inputs != NULL) 1168 ops[i]->ldpc_dec.harq_combined_input = 1169 harq_inputs[start_idx + i]; 1170 if (harq_outputs != NULL) 1171 ops[i]->ldpc_dec.harq_combined_output = 1172 harq_outputs[start_idx + i]; 1173 } 1174 } 1175 1176 1177 static void 1178 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1179 unsigned int start_idx, 1180 struct rte_bbdev_op_data *inputs, 1181 struct rte_bbdev_op_data *outputs, 1182 struct rte_bbdev_enc_op *ref_op) 1183 { 1184 unsigned int i; 1185 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1186 for (i = 0; i < n; ++i) { 1187 if (ldpc_enc->code_block_mode == 0) { 1188 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1189 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1190 ops[i]->ldpc_enc.tb_params.cab = 1191 ldpc_enc->tb_params.cab; 1192 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1193 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1194 } else { 1195 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1196 } 1197 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1198 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1199 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1200 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1201 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1202 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1203 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1204 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1205 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1206 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1207 } 1208 } 1209 1210 static int 1211 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1212 unsigned int order_idx, const int expected_status) 1213 { 1214 TEST_ASSERT(op->status == expected_status, 1215 "op_status (%d) != expected_status (%d)", 1216 op->status, expected_status); 1217 1218 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1219 "Ordering error, expected %p, got %p", 1220 (void *)(uintptr_t)order_idx, op->opaque_data); 1221 1222 return TEST_SUCCESS; 1223 } 1224 1225 static int 1226 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1227 unsigned int order_idx, const int expected_status) 1228 { 1229 TEST_ASSERT(op->status == expected_status, 1230 "op_status (%d) != expected_status (%d)", 1231 op->status, expected_status); 1232 1233 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1234 "Ordering error, expected %p, got %p", 1235 (void *)(uintptr_t)order_idx, op->opaque_data); 1236 1237 return TEST_SUCCESS; 1238 } 1239 1240 static inline int 1241 validate_op_chain(struct rte_bbdev_op_data *op, 1242 struct op_data_entries *orig_op) 1243 { 1244 uint8_t i; 1245 struct rte_mbuf *m = op->data; 1246 uint8_t nb_dst_segments = orig_op->nb_segments; 1247 uint32_t total_data_size = 0; 1248 1249 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1250 "Number of segments differ in original (%u) and filled (%u) op", 1251 nb_dst_segments, m->nb_segs); 1252 1253 /* Validate each mbuf segment length */ 1254 for (i = 0; i < nb_dst_segments; ++i) { 1255 /* Apply offset to the first mbuf segment */ 1256 uint16_t offset = (i == 0) ? op->offset : 0; 1257 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1258 total_data_size += orig_op->segments[i].length; 1259 1260 TEST_ASSERT(orig_op->segments[i].length == data_len, 1261 "Length of segment differ in original (%u) and filled (%u) op", 1262 orig_op->segments[i].length, data_len); 1263 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1264 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1265 data_len, 1266 "Output buffers (CB=%u) are not equal", i); 1267 m = m->next; 1268 } 1269 1270 /* Validate total mbuf pkt length */ 1271 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1272 TEST_ASSERT(total_data_size == pkt_len, 1273 "Length of data differ in original (%u) and filled (%u) op", 1274 total_data_size, pkt_len); 1275 1276 return TEST_SUCCESS; 1277 } 1278 1279 static int 1280 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1281 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1282 { 1283 unsigned int i; 1284 int ret; 1285 struct op_data_entries *hard_data_orig = 1286 &test_vector.entries[DATA_HARD_OUTPUT]; 1287 struct op_data_entries *soft_data_orig = 1288 &test_vector.entries[DATA_SOFT_OUTPUT]; 1289 struct rte_bbdev_op_turbo_dec *ops_td; 1290 struct rte_bbdev_op_data *hard_output; 1291 struct rte_bbdev_op_data *soft_output; 1292 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 1293 1294 for (i = 0; i < n; ++i) { 1295 ops_td = &ops[i]->turbo_dec; 1296 hard_output = &ops_td->hard_output; 1297 soft_output = &ops_td->soft_output; 1298 1299 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1300 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1301 "Returned iter_count (%d) > expected iter_count (%d)", 1302 ops_td->iter_count, ref_td->iter_count); 1303 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1304 TEST_ASSERT_SUCCESS(ret, 1305 "Checking status and ordering for decoder failed"); 1306 1307 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1308 hard_data_orig), 1309 "Hard output buffers (CB=%u) are not equal", 1310 i); 1311 1312 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 1313 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1314 soft_data_orig), 1315 "Soft output buffers (CB=%u) are not equal", 1316 i); 1317 } 1318 1319 return TEST_SUCCESS; 1320 } 1321 1322 1323 static int 1324 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1325 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1326 { 1327 unsigned int i; 1328 int ret; 1329 struct op_data_entries *hard_data_orig = 1330 &test_vector.entries[DATA_HARD_OUTPUT]; 1331 struct op_data_entries *soft_data_orig = 1332 &test_vector.entries[DATA_SOFT_OUTPUT]; 1333 struct op_data_entries *harq_data_orig = 1334 &test_vector.entries[DATA_HARQ_OUTPUT]; 1335 struct rte_bbdev_op_ldpc_dec *ops_td; 1336 struct rte_bbdev_op_data *hard_output; 1337 struct rte_bbdev_op_data *harq_output; 1338 struct rte_bbdev_op_data *soft_output; 1339 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 1340 1341 for (i = 0; i < n; ++i) { 1342 ops_td = &ops[i]->ldpc_dec; 1343 hard_output = &ops_td->hard_output; 1344 harq_output = &ops_td->harq_combined_output; 1345 soft_output = &ops_td->soft_output; 1346 1347 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 1348 TEST_ASSERT_SUCCESS(ret, 1349 "Checking status and ordering for decoder failed"); 1350 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 1351 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 1352 "Returned iter_count (%d) > expected iter_count (%d)", 1353 ops_td->iter_count, ref_td->iter_count); 1354 /* We can ignore data when the decoding failed to converge */ 1355 if ((ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) == 0) 1356 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 1357 hard_data_orig), 1358 "Hard output buffers (CB=%u) are not equal", 1359 i); 1360 1361 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 1362 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 1363 soft_data_orig), 1364 "Soft output buffers (CB=%u) are not equal", 1365 i); 1366 if (ref_op->ldpc_dec.op_flags & 1367 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 1368 ldpc_input_llr_scaling(harq_output, 1, 8, 0); 1369 TEST_ASSERT_SUCCESS(validate_op_chain(harq_output, 1370 harq_data_orig), 1371 "HARQ output buffers (CB=%u) are not equal", 1372 i); 1373 } 1374 } 1375 1376 return TEST_SUCCESS; 1377 } 1378 1379 1380 static int 1381 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1382 struct rte_bbdev_enc_op *ref_op) 1383 { 1384 unsigned int i; 1385 int ret; 1386 struct op_data_entries *hard_data_orig = 1387 &test_vector.entries[DATA_HARD_OUTPUT]; 1388 1389 for (i = 0; i < n; ++i) { 1390 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1391 TEST_ASSERT_SUCCESS(ret, 1392 "Checking status and ordering for encoder failed"); 1393 TEST_ASSERT_SUCCESS(validate_op_chain( 1394 &ops[i]->turbo_enc.output, 1395 hard_data_orig), 1396 "Output buffers (CB=%u) are not equal", 1397 i); 1398 } 1399 1400 return TEST_SUCCESS; 1401 } 1402 1403 static int 1404 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 1405 struct rte_bbdev_enc_op *ref_op) 1406 { 1407 unsigned int i; 1408 int ret; 1409 struct op_data_entries *hard_data_orig = 1410 &test_vector.entries[DATA_HARD_OUTPUT]; 1411 1412 for (i = 0; i < n; ++i) { 1413 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 1414 TEST_ASSERT_SUCCESS(ret, 1415 "Checking status and ordering for encoder failed"); 1416 TEST_ASSERT_SUCCESS(validate_op_chain( 1417 &ops[i]->ldpc_enc.output, 1418 hard_data_orig), 1419 "Output buffers (CB=%u) are not equal", 1420 i); 1421 } 1422 1423 return TEST_SUCCESS; 1424 } 1425 1426 static void 1427 create_reference_dec_op(struct rte_bbdev_dec_op *op) 1428 { 1429 unsigned int i; 1430 struct op_data_entries *entry; 1431 1432 op->turbo_dec = test_vector.turbo_dec; 1433 entry = &test_vector.entries[DATA_INPUT]; 1434 for (i = 0; i < entry->nb_segments; ++i) 1435 op->turbo_dec.input.length += 1436 entry->segments[i].length; 1437 } 1438 1439 static void 1440 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 1441 { 1442 unsigned int i; 1443 struct op_data_entries *entry; 1444 1445 op->ldpc_dec = test_vector.ldpc_dec; 1446 entry = &test_vector.entries[DATA_INPUT]; 1447 for (i = 0; i < entry->nb_segments; ++i) 1448 op->ldpc_dec.input.length += 1449 entry->segments[i].length; 1450 if (test_vector.ldpc_dec.op_flags & 1451 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 1452 entry = &test_vector.entries[DATA_HARQ_INPUT]; 1453 for (i = 0; i < entry->nb_segments; ++i) 1454 op->ldpc_dec.harq_combined_input.length += 1455 entry->segments[i].length; 1456 } 1457 } 1458 1459 1460 static void 1461 create_reference_enc_op(struct rte_bbdev_enc_op *op) 1462 { 1463 unsigned int i; 1464 struct op_data_entries *entry; 1465 1466 op->turbo_enc = test_vector.turbo_enc; 1467 entry = &test_vector.entries[DATA_INPUT]; 1468 for (i = 0; i < entry->nb_segments; ++i) 1469 op->turbo_enc.input.length += 1470 entry->segments[i].length; 1471 } 1472 1473 static void 1474 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 1475 { 1476 unsigned int i; 1477 struct op_data_entries *entry; 1478 1479 op->ldpc_enc = test_vector.ldpc_enc; 1480 entry = &test_vector.entries[DATA_INPUT]; 1481 for (i = 0; i < entry->nb_segments; ++i) 1482 op->ldpc_enc.input.length += 1483 entry->segments[i].length; 1484 } 1485 1486 static uint32_t 1487 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 1488 { 1489 uint8_t i; 1490 uint32_t c, r, tb_size = 0; 1491 1492 if (op->turbo_dec.code_block_mode) { 1493 tb_size = op->turbo_dec.tb_params.k_neg; 1494 } else { 1495 c = op->turbo_dec.tb_params.c; 1496 r = op->turbo_dec.tb_params.r; 1497 for (i = 0; i < c-r; i++) 1498 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 1499 op->turbo_dec.tb_params.k_neg : 1500 op->turbo_dec.tb_params.k_pos; 1501 } 1502 return tb_size; 1503 } 1504 1505 static uint32_t 1506 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 1507 { 1508 uint8_t i; 1509 uint32_t c, r, tb_size = 0; 1510 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 1511 1512 if (op->ldpc_dec.code_block_mode) { 1513 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 1514 } else { 1515 c = op->ldpc_dec.tb_params.c; 1516 r = op->ldpc_dec.tb_params.r; 1517 for (i = 0; i < c-r; i++) 1518 tb_size += sys_cols * op->ldpc_dec.z_c 1519 - op->ldpc_dec.n_filler; 1520 } 1521 return tb_size; 1522 } 1523 1524 static uint32_t 1525 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 1526 { 1527 uint8_t i; 1528 uint32_t c, r, tb_size = 0; 1529 1530 if (op->turbo_enc.code_block_mode) { 1531 tb_size = op->turbo_enc.tb_params.k_neg; 1532 } else { 1533 c = op->turbo_enc.tb_params.c; 1534 r = op->turbo_enc.tb_params.r; 1535 for (i = 0; i < c-r; i++) 1536 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 1537 op->turbo_enc.tb_params.k_neg : 1538 op->turbo_enc.tb_params.k_pos; 1539 } 1540 return tb_size; 1541 } 1542 1543 static uint32_t 1544 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 1545 { 1546 uint8_t i; 1547 uint32_t c, r, tb_size = 0; 1548 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 1549 1550 if (op->turbo_enc.code_block_mode) { 1551 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 1552 } else { 1553 c = op->turbo_enc.tb_params.c; 1554 r = op->turbo_enc.tb_params.r; 1555 for (i = 0; i < c-r; i++) 1556 tb_size += sys_cols * op->ldpc_enc.z_c 1557 - op->ldpc_enc.n_filler; 1558 } 1559 return tb_size; 1560 } 1561 1562 1563 static int 1564 init_test_op_params(struct test_op_params *op_params, 1565 enum rte_bbdev_op_type op_type, const int expected_status, 1566 const int vector_mask, struct rte_mempool *ops_mp, 1567 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 1568 { 1569 int ret = 0; 1570 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1571 op_type == RTE_BBDEV_OP_LDPC_DEC) 1572 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 1573 &op_params->ref_dec_op, 1); 1574 else 1575 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 1576 &op_params->ref_enc_op, 1); 1577 1578 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 1579 1580 op_params->mp = ops_mp; 1581 op_params->burst_sz = burst_sz; 1582 op_params->num_to_process = num_to_process; 1583 op_params->num_lcores = num_lcores; 1584 op_params->vector_mask = vector_mask; 1585 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 1586 op_type == RTE_BBDEV_OP_LDPC_DEC) 1587 op_params->ref_dec_op->status = expected_status; 1588 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 1589 || op_type == RTE_BBDEV_OP_LDPC_ENC) 1590 op_params->ref_enc_op->status = expected_status; 1591 return 0; 1592 } 1593 1594 static int 1595 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 1596 struct test_op_params *op_params) 1597 { 1598 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 1599 unsigned int i; 1600 struct active_device *ad; 1601 unsigned int burst_sz = get_burst_sz(); 1602 enum rte_bbdev_op_type op_type = test_vector.op_type; 1603 const struct rte_bbdev_op_cap *capabilities = NULL; 1604 1605 ad = &active_devs[dev_id]; 1606 1607 /* Check if device supports op_type */ 1608 if (!is_avail_op(ad, test_vector.op_type)) 1609 return TEST_SUCCESS; 1610 1611 struct rte_bbdev_info info; 1612 rte_bbdev_info_get(ad->dev_id, &info); 1613 socket_id = GET_SOCKET(info.socket_id); 1614 1615 f_ret = create_mempools(ad, socket_id, op_type, 1616 get_num_ops()); 1617 if (f_ret != TEST_SUCCESS) { 1618 printf("Couldn't create mempools"); 1619 goto fail; 1620 } 1621 if (op_type == RTE_BBDEV_OP_NONE) 1622 op_type = RTE_BBDEV_OP_TURBO_ENC; 1623 1624 f_ret = init_test_op_params(op_params, test_vector.op_type, 1625 test_vector.expected_status, 1626 test_vector.mask, 1627 ad->ops_mempool, 1628 burst_sz, 1629 get_num_ops(), 1630 get_num_lcores()); 1631 if (f_ret != TEST_SUCCESS) { 1632 printf("Couldn't init test op params"); 1633 goto fail; 1634 } 1635 1636 1637 /* Find capabilities */ 1638 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 1639 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 1640 if (cap->type == test_vector.op_type) { 1641 capabilities = cap; 1642 break; 1643 } 1644 cap++; 1645 } 1646 TEST_ASSERT_NOT_NULL(capabilities, 1647 "Couldn't find capabilities"); 1648 1649 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1650 create_reference_dec_op(op_params->ref_dec_op); 1651 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 1652 create_reference_enc_op(op_params->ref_enc_op); 1653 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 1654 create_reference_ldpc_enc_op(op_params->ref_enc_op); 1655 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1656 create_reference_ldpc_dec_op(op_params->ref_dec_op); 1657 1658 for (i = 0; i < ad->nb_queues; ++i) { 1659 f_ret = fill_queue_buffers(op_params, 1660 ad->in_mbuf_pool, 1661 ad->hard_out_mbuf_pool, 1662 ad->soft_out_mbuf_pool, 1663 ad->harq_in_mbuf_pool, 1664 ad->harq_out_mbuf_pool, 1665 ad->queue_ids[i], 1666 capabilities, 1667 info.drv.min_alignment, 1668 socket_id); 1669 if (f_ret != TEST_SUCCESS) { 1670 printf("Couldn't init queue buffers"); 1671 goto fail; 1672 } 1673 } 1674 1675 /* Run test case function */ 1676 t_ret = test_case_func(ad, op_params); 1677 1678 /* Free active device resources and return */ 1679 free_buffers(ad, op_params); 1680 return t_ret; 1681 1682 fail: 1683 free_buffers(ad, op_params); 1684 return TEST_FAILED; 1685 } 1686 1687 /* Run given test function per active device per supported op type 1688 * per burst size. 1689 */ 1690 static int 1691 run_test_case(test_case_function *test_case_func) 1692 { 1693 int ret = 0; 1694 uint8_t dev; 1695 1696 /* Alloc op_params */ 1697 struct test_op_params *op_params = rte_zmalloc(NULL, 1698 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 1699 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 1700 RTE_ALIGN(sizeof(struct test_op_params), 1701 RTE_CACHE_LINE_SIZE)); 1702 1703 /* For each device run test case function */ 1704 for (dev = 0; dev < nb_active_devs; ++dev) 1705 ret |= run_test_case_on_device(test_case_func, dev, op_params); 1706 1707 rte_free(op_params); 1708 1709 return ret; 1710 } 1711 1712 static void 1713 dequeue_event_callback(uint16_t dev_id, 1714 enum rte_bbdev_event_type event, void *cb_arg, 1715 void *ret_param) 1716 { 1717 int ret; 1718 uint16_t i; 1719 uint64_t total_time; 1720 uint16_t deq, burst_sz, num_ops; 1721 uint16_t queue_id = *(uint16_t *) ret_param; 1722 struct rte_bbdev_info info; 1723 double tb_len_bits; 1724 struct thread_params *tp = cb_arg; 1725 1726 /* Find matching thread params using queue_id */ 1727 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 1728 if (tp->queue_id == queue_id) 1729 break; 1730 1731 if (i == MAX_QUEUES) { 1732 printf("%s: Queue_id from interrupt details was not found!\n", 1733 __func__); 1734 return; 1735 } 1736 1737 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 1738 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1739 printf( 1740 "Dequeue interrupt handler called for incorrect event!\n"); 1741 return; 1742 } 1743 1744 burst_sz = rte_atomic16_read(&tp->burst_sz); 1745 num_ops = tp->op_params->num_to_process; 1746 1747 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 1748 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 1749 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 1750 &tp->dec_ops[ 1751 rte_atomic16_read(&tp->nb_dequeued)], 1752 burst_sz); 1753 else 1754 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 1755 &tp->enc_ops[ 1756 rte_atomic16_read(&tp->nb_dequeued)], 1757 burst_sz); 1758 1759 if (deq < burst_sz) { 1760 printf( 1761 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 1762 burst_sz, deq); 1763 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1764 return; 1765 } 1766 1767 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) { 1768 rte_atomic16_add(&tp->nb_dequeued, deq); 1769 return; 1770 } 1771 1772 total_time = rte_rdtsc_precise() - tp->start_time; 1773 1774 rte_bbdev_info_get(dev_id, &info); 1775 1776 ret = TEST_SUCCESS; 1777 1778 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 1779 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1780 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 1781 tp->op_params->vector_mask); 1782 /* get the max of iter_count for all dequeued ops */ 1783 for (i = 0; i < num_ops; ++i) 1784 tp->iter_count = RTE_MAX( 1785 tp->dec_ops[i]->turbo_dec.iter_count, 1786 tp->iter_count); 1787 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1788 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 1789 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1790 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 1791 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1792 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 1793 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 1794 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 1795 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 1796 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1797 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 1798 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 1799 tp->op_params->vector_mask); 1800 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 1801 } 1802 1803 if (ret) { 1804 printf("Buffers validation failed\n"); 1805 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1806 } 1807 1808 switch (test_vector.op_type) { 1809 case RTE_BBDEV_OP_TURBO_DEC: 1810 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 1811 break; 1812 case RTE_BBDEV_OP_TURBO_ENC: 1813 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 1814 break; 1815 case RTE_BBDEV_OP_LDPC_DEC: 1816 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 1817 break; 1818 case RTE_BBDEV_OP_LDPC_ENC: 1819 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 1820 break; 1821 case RTE_BBDEV_OP_NONE: 1822 tb_len_bits = 0.0; 1823 break; 1824 default: 1825 printf("Unknown op type: %d\n", test_vector.op_type); 1826 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 1827 return; 1828 } 1829 1830 tp->ops_per_sec += ((double)num_ops) / 1831 ((double)total_time / (double)rte_get_tsc_hz()); 1832 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 1833 ((double)total_time / (double)rte_get_tsc_hz()); 1834 1835 rte_atomic16_add(&tp->nb_dequeued, deq); 1836 } 1837 1838 static int 1839 throughput_intr_lcore_dec(void *arg) 1840 { 1841 struct thread_params *tp = arg; 1842 unsigned int enqueued; 1843 const uint16_t queue_id = tp->queue_id; 1844 const uint16_t burst_sz = tp->op_params->burst_sz; 1845 const uint16_t num_to_process = tp->op_params->num_to_process; 1846 struct rte_bbdev_dec_op *ops[num_to_process]; 1847 struct test_buffers *bufs = NULL; 1848 struct rte_bbdev_info info; 1849 int ret, i, j; 1850 uint16_t num_to_enq, enq; 1851 1852 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1853 "BURST_SIZE should be <= %u", MAX_BURST); 1854 1855 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1856 "Failed to enable interrupts for dev: %u, queue_id: %u", 1857 tp->dev_id, queue_id); 1858 1859 rte_bbdev_info_get(tp->dev_id, &info); 1860 1861 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1862 "NUM_OPS cannot exceed %u for this device", 1863 info.drv.queue_size_lim); 1864 1865 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1866 1867 rte_atomic16_clear(&tp->processing_status); 1868 rte_atomic16_clear(&tp->nb_dequeued); 1869 1870 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1871 rte_pause(); 1872 1873 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 1874 num_to_process); 1875 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1876 num_to_process); 1877 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1878 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 1879 bufs->hard_outputs, bufs->soft_outputs, 1880 tp->op_params->ref_dec_op); 1881 1882 /* Set counter to validate the ordering */ 1883 for (j = 0; j < num_to_process; ++j) 1884 ops[j]->opaque_data = (void *)(uintptr_t)j; 1885 1886 for (j = 0; j < TEST_REPETITIONS; ++j) { 1887 for (i = 0; i < num_to_process; ++i) 1888 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 1889 1890 tp->start_time = rte_rdtsc_precise(); 1891 for (enqueued = 0; enqueued < num_to_process;) { 1892 num_to_enq = burst_sz; 1893 1894 if (unlikely(num_to_process - enqueued < num_to_enq)) 1895 num_to_enq = num_to_process - enqueued; 1896 1897 enq = 0; 1898 do { 1899 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 1900 queue_id, &ops[enqueued], 1901 num_to_enq); 1902 } while (unlikely(num_to_enq != enq)); 1903 enqueued += enq; 1904 1905 /* Write to thread burst_sz current number of enqueued 1906 * descriptors. It ensures that proper number of 1907 * descriptors will be dequeued in callback 1908 * function - needed for last batch in case where 1909 * the number of operations is not a multiple of 1910 * burst size. 1911 */ 1912 rte_atomic16_set(&tp->burst_sz, num_to_enq); 1913 1914 /* Wait until processing of previous batch is 1915 * completed 1916 */ 1917 while (rte_atomic16_read(&tp->nb_dequeued) != 1918 (int16_t) enqueued) 1919 rte_pause(); 1920 } 1921 if (j != TEST_REPETITIONS - 1) 1922 rte_atomic16_clear(&tp->nb_dequeued); 1923 } 1924 1925 return TEST_SUCCESS; 1926 } 1927 1928 static int 1929 throughput_intr_lcore_enc(void *arg) 1930 { 1931 struct thread_params *tp = arg; 1932 unsigned int enqueued; 1933 const uint16_t queue_id = tp->queue_id; 1934 const uint16_t burst_sz = tp->op_params->burst_sz; 1935 const uint16_t num_to_process = tp->op_params->num_to_process; 1936 struct rte_bbdev_enc_op *ops[num_to_process]; 1937 struct test_buffers *bufs = NULL; 1938 struct rte_bbdev_info info; 1939 int ret, i, j; 1940 uint16_t num_to_enq, enq; 1941 1942 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 1943 "BURST_SIZE should be <= %u", MAX_BURST); 1944 1945 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 1946 "Failed to enable interrupts for dev: %u, queue_id: %u", 1947 tp->dev_id, queue_id); 1948 1949 rte_bbdev_info_get(tp->dev_id, &info); 1950 1951 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 1952 "NUM_OPS cannot exceed %u for this device", 1953 info.drv.queue_size_lim); 1954 1955 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 1956 1957 rte_atomic16_clear(&tp->processing_status); 1958 rte_atomic16_clear(&tp->nb_dequeued); 1959 1960 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 1961 rte_pause(); 1962 1963 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 1964 num_to_process); 1965 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 1966 num_to_process); 1967 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 1968 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 1969 bufs->hard_outputs, tp->op_params->ref_enc_op); 1970 1971 /* Set counter to validate the ordering */ 1972 for (j = 0; j < num_to_process; ++j) 1973 ops[j]->opaque_data = (void *)(uintptr_t)j; 1974 1975 for (j = 0; j < TEST_REPETITIONS; ++j) { 1976 for (i = 0; i < num_to_process; ++i) 1977 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 1978 1979 tp->start_time = rte_rdtsc_precise(); 1980 for (enqueued = 0; enqueued < num_to_process;) { 1981 num_to_enq = burst_sz; 1982 1983 if (unlikely(num_to_process - enqueued < num_to_enq)) 1984 num_to_enq = num_to_process - enqueued; 1985 1986 enq = 0; 1987 do { 1988 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 1989 queue_id, &ops[enqueued], 1990 num_to_enq); 1991 } while (unlikely(enq != num_to_enq)); 1992 enqueued += enq; 1993 1994 /* Write to thread burst_sz current number of enqueued 1995 * descriptors. It ensures that proper number of 1996 * descriptors will be dequeued in callback 1997 * function - needed for last batch in case where 1998 * the number of operations is not a multiple of 1999 * burst size. 2000 */ 2001 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2002 2003 /* Wait until processing of previous batch is 2004 * completed 2005 */ 2006 while (rte_atomic16_read(&tp->nb_dequeued) != 2007 (int16_t) enqueued) 2008 rte_pause(); 2009 } 2010 if (j != TEST_REPETITIONS - 1) 2011 rte_atomic16_clear(&tp->nb_dequeued); 2012 } 2013 2014 return TEST_SUCCESS; 2015 } 2016 2017 static int 2018 throughput_pmd_lcore_dec(void *arg) 2019 { 2020 struct thread_params *tp = arg; 2021 uint16_t enq, deq; 2022 uint64_t total_time = 0, start_time; 2023 const uint16_t queue_id = tp->queue_id; 2024 const uint16_t burst_sz = tp->op_params->burst_sz; 2025 const uint16_t num_ops = tp->op_params->num_to_process; 2026 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2027 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2028 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2029 struct test_buffers *bufs = NULL; 2030 int i, j, ret; 2031 struct rte_bbdev_info info; 2032 uint16_t num_to_enq; 2033 2034 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2035 "BURST_SIZE should be <= %u", MAX_BURST); 2036 2037 rte_bbdev_info_get(tp->dev_id, &info); 2038 2039 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2040 "NUM_OPS cannot exceed %u for this device", 2041 info.drv.queue_size_lim); 2042 2043 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2044 2045 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2046 rte_pause(); 2047 2048 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2049 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2050 2051 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2052 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2053 bufs->hard_outputs, bufs->soft_outputs, ref_op); 2054 2055 /* Set counter to validate the ordering */ 2056 for (j = 0; j < num_ops; ++j) 2057 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2058 2059 for (i = 0; i < TEST_REPETITIONS; ++i) { 2060 2061 for (j = 0; j < num_ops; ++j) 2062 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 2063 2064 start_time = rte_rdtsc_precise(); 2065 2066 for (enq = 0, deq = 0; enq < num_ops;) { 2067 num_to_enq = burst_sz; 2068 2069 if (unlikely(num_ops - enq < num_to_enq)) 2070 num_to_enq = num_ops - enq; 2071 2072 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2073 queue_id, &ops_enq[enq], num_to_enq); 2074 2075 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2076 queue_id, &ops_deq[deq], enq - deq); 2077 } 2078 2079 /* dequeue the remaining */ 2080 while (deq < enq) { 2081 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 2082 queue_id, &ops_deq[deq], enq - deq); 2083 } 2084 2085 total_time += rte_rdtsc_precise() - start_time; 2086 } 2087 2088 tp->iter_count = 0; 2089 /* get the max of iter_count for all dequeued ops */ 2090 for (i = 0; i < num_ops; ++i) { 2091 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 2092 tp->iter_count); 2093 } 2094 2095 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2096 ret = validate_dec_op(ops_deq, num_ops, ref_op, 2097 tp->op_params->vector_mask); 2098 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2099 } 2100 2101 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2102 2103 double tb_len_bits = calc_dec_TB_size(ref_op); 2104 2105 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2106 ((double)total_time / (double)rte_get_tsc_hz()); 2107 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2108 1000000.0) / ((double)total_time / 2109 (double)rte_get_tsc_hz()); 2110 2111 return TEST_SUCCESS; 2112 } 2113 2114 static int 2115 throughput_pmd_lcore_ldpc_dec(void *arg) 2116 { 2117 struct thread_params *tp = arg; 2118 uint16_t enq, deq; 2119 uint64_t total_time = 0, start_time; 2120 const uint16_t queue_id = tp->queue_id; 2121 const uint16_t burst_sz = tp->op_params->burst_sz; 2122 const uint16_t num_ops = tp->op_params->num_to_process; 2123 struct rte_bbdev_dec_op *ops_enq[num_ops]; 2124 struct rte_bbdev_dec_op *ops_deq[num_ops]; 2125 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2126 struct test_buffers *bufs = NULL; 2127 int i, j, ret; 2128 struct rte_bbdev_info info; 2129 uint16_t num_to_enq; 2130 2131 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2132 "BURST_SIZE should be <= %u", MAX_BURST); 2133 2134 rte_bbdev_info_get(tp->dev_id, &info); 2135 2136 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2137 "NUM_OPS cannot exceed %u for this device", 2138 info.drv.queue_size_lim); 2139 2140 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2141 2142 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2143 rte_pause(); 2144 2145 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 2146 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 2147 2148 /* For throughput tests we need to disable early termination */ 2149 if (check_bit(ref_op->ldpc_dec.op_flags, 2150 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 2151 ref_op->ldpc_dec.op_flags -= 2152 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 2153 ref_op->ldpc_dec.iter_max = 6; 2154 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 2155 2156 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2157 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 2158 bufs->hard_outputs, bufs->soft_outputs, 2159 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2160 2161 /* Set counter to validate the ordering */ 2162 for (j = 0; j < num_ops; ++j) 2163 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2164 2165 for (i = 0; i < TEST_REPETITIONS; ++i) { 2166 for (j = 0; j < num_ops; ++j) { 2167 mbuf_reset(ops_enq[j]->ldpc_dec.hard_output.data); 2168 if (check_bit(ref_op->ldpc_dec.op_flags, 2169 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 2170 mbuf_reset( 2171 ops_enq[j]->ldpc_dec.harq_combined_output.data); 2172 } 2173 2174 start_time = rte_rdtsc_precise(); 2175 2176 for (enq = 0, deq = 0; enq < num_ops;) { 2177 num_to_enq = burst_sz; 2178 2179 if (unlikely(num_ops - enq < num_to_enq)) 2180 num_to_enq = num_ops - enq; 2181 2182 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 2183 queue_id, &ops_enq[enq], num_to_enq); 2184 2185 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2186 queue_id, &ops_deq[deq], enq - deq); 2187 } 2188 2189 /* dequeue the remaining */ 2190 while (deq < enq) { 2191 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 2192 queue_id, &ops_deq[deq], enq - deq); 2193 } 2194 2195 total_time += rte_rdtsc_precise() - start_time; 2196 } 2197 2198 tp->iter_count = 0; 2199 /* get the max of iter_count for all dequeued ops */ 2200 for (i = 0; i < num_ops; ++i) { 2201 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 2202 tp->iter_count); 2203 } 2204 2205 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2206 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 2207 tp->op_params->vector_mask); 2208 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2209 } 2210 2211 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 2212 2213 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 2214 2215 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2216 ((double)total_time / (double)rte_get_tsc_hz()); 2217 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 2218 1000000.0) / ((double)total_time / 2219 (double)rte_get_tsc_hz()); 2220 2221 return TEST_SUCCESS; 2222 } 2223 2224 static int 2225 throughput_pmd_lcore_enc(void *arg) 2226 { 2227 struct thread_params *tp = arg; 2228 uint16_t enq, deq; 2229 uint64_t total_time = 0, start_time; 2230 const uint16_t queue_id = tp->queue_id; 2231 const uint16_t burst_sz = tp->op_params->burst_sz; 2232 const uint16_t num_ops = tp->op_params->num_to_process; 2233 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2234 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2235 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2236 struct test_buffers *bufs = NULL; 2237 int i, j, ret; 2238 struct rte_bbdev_info info; 2239 uint16_t num_to_enq; 2240 2241 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2242 "BURST_SIZE should be <= %u", MAX_BURST); 2243 2244 rte_bbdev_info_get(tp->dev_id, &info); 2245 2246 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2247 "NUM_OPS cannot exceed %u for this device", 2248 info.drv.queue_size_lim); 2249 2250 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2251 2252 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2253 rte_pause(); 2254 2255 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2256 num_ops); 2257 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2258 num_ops); 2259 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2260 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2261 bufs->hard_outputs, ref_op); 2262 2263 /* Set counter to validate the ordering */ 2264 for (j = 0; j < num_ops; ++j) 2265 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2266 2267 for (i = 0; i < TEST_REPETITIONS; ++i) { 2268 2269 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2270 for (j = 0; j < num_ops; ++j) 2271 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2272 2273 start_time = rte_rdtsc_precise(); 2274 2275 for (enq = 0, deq = 0; enq < num_ops;) { 2276 num_to_enq = burst_sz; 2277 2278 if (unlikely(num_ops - enq < num_to_enq)) 2279 num_to_enq = num_ops - enq; 2280 2281 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2282 queue_id, &ops_enq[enq], num_to_enq); 2283 2284 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2285 queue_id, &ops_deq[deq], enq - deq); 2286 } 2287 2288 /* dequeue the remaining */ 2289 while (deq < enq) { 2290 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 2291 queue_id, &ops_deq[deq], enq - deq); 2292 } 2293 2294 total_time += rte_rdtsc_precise() - start_time; 2295 } 2296 2297 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2298 ret = validate_enc_op(ops_deq, num_ops, ref_op); 2299 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2300 } 2301 2302 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2303 2304 double tb_len_bits = calc_enc_TB_size(ref_op); 2305 2306 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2307 ((double)total_time / (double)rte_get_tsc_hz()); 2308 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2309 / 1000000.0) / ((double)total_time / 2310 (double)rte_get_tsc_hz()); 2311 2312 return TEST_SUCCESS; 2313 } 2314 2315 static int 2316 throughput_pmd_lcore_ldpc_enc(void *arg) 2317 { 2318 struct thread_params *tp = arg; 2319 uint16_t enq, deq; 2320 uint64_t total_time = 0, start_time; 2321 const uint16_t queue_id = tp->queue_id; 2322 const uint16_t burst_sz = tp->op_params->burst_sz; 2323 const uint16_t num_ops = tp->op_params->num_to_process; 2324 struct rte_bbdev_enc_op *ops_enq[num_ops]; 2325 struct rte_bbdev_enc_op *ops_deq[num_ops]; 2326 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2327 struct test_buffers *bufs = NULL; 2328 int i, j, ret; 2329 struct rte_bbdev_info info; 2330 uint16_t num_to_enq; 2331 2332 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2333 "BURST_SIZE should be <= %u", MAX_BURST); 2334 2335 rte_bbdev_info_get(tp->dev_id, &info); 2336 2337 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 2338 "NUM_OPS cannot exceed %u for this device", 2339 info.drv.queue_size_lim); 2340 2341 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2342 2343 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2344 rte_pause(); 2345 2346 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 2347 num_ops); 2348 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2349 num_ops); 2350 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2351 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 2352 bufs->hard_outputs, ref_op); 2353 2354 /* Set counter to validate the ordering */ 2355 for (j = 0; j < num_ops; ++j) 2356 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2357 2358 for (i = 0; i < TEST_REPETITIONS; ++i) { 2359 2360 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2361 for (j = 0; j < num_ops; ++j) 2362 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 2363 2364 start_time = rte_rdtsc_precise(); 2365 2366 for (enq = 0, deq = 0; enq < num_ops;) { 2367 num_to_enq = burst_sz; 2368 2369 if (unlikely(num_ops - enq < num_to_enq)) 2370 num_to_enq = num_ops - enq; 2371 2372 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 2373 queue_id, &ops_enq[enq], num_to_enq); 2374 2375 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2376 queue_id, &ops_deq[deq], enq - deq); 2377 } 2378 2379 /* dequeue the remaining */ 2380 while (deq < enq) { 2381 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 2382 queue_id, &ops_deq[deq], enq - deq); 2383 } 2384 2385 total_time += rte_rdtsc_precise() - start_time; 2386 } 2387 2388 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2389 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 2390 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2391 } 2392 2393 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 2394 2395 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 2396 2397 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 2398 ((double)total_time / (double)rte_get_tsc_hz()); 2399 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 2400 / 1000000.0) / ((double)total_time / 2401 (double)rte_get_tsc_hz()); 2402 2403 return TEST_SUCCESS; 2404 } 2405 2406 static void 2407 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 2408 { 2409 unsigned int iter = 0; 2410 double total_mops = 0, total_mbps = 0; 2411 2412 for (iter = 0; iter < used_cores; iter++) { 2413 printf( 2414 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 2415 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2416 t_params[iter].mbps); 2417 total_mops += t_params[iter].ops_per_sec; 2418 total_mbps += t_params[iter].mbps; 2419 } 2420 printf( 2421 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 2422 used_cores, total_mops, total_mbps); 2423 } 2424 2425 static void 2426 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 2427 { 2428 unsigned int iter = 0; 2429 double total_mops = 0, total_mbps = 0; 2430 uint8_t iter_count = 0; 2431 2432 for (iter = 0; iter < used_cores; iter++) { 2433 printf( 2434 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 2435 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 2436 t_params[iter].mbps, t_params[iter].iter_count); 2437 total_mops += t_params[iter].ops_per_sec; 2438 total_mbps += t_params[iter].mbps; 2439 iter_count = RTE_MAX(iter_count, t_params[iter].iter_count); 2440 } 2441 printf( 2442 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 2443 used_cores, total_mops, total_mbps, iter_count); 2444 } 2445 2446 /* 2447 * Test function that determines how long an enqueue + dequeue of a burst 2448 * takes on available lcores. 2449 */ 2450 static int 2451 throughput_test(struct active_device *ad, 2452 struct test_op_params *op_params) 2453 { 2454 int ret; 2455 unsigned int lcore_id, used_cores = 0; 2456 struct thread_params *t_params, *tp; 2457 struct rte_bbdev_info info; 2458 lcore_function_t *throughput_function; 2459 uint16_t num_lcores; 2460 const char *op_type_str; 2461 2462 rte_bbdev_info_get(ad->dev_id, &info); 2463 2464 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 2465 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 2466 test_vector.op_type); 2467 2468 printf("+ ------------------------------------------------------- +\n"); 2469 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 2470 info.dev_name, ad->nb_queues, op_params->burst_sz, 2471 op_params->num_to_process, op_params->num_lcores, 2472 op_type_str, 2473 intr_enabled ? "Interrupt mode" : "PMD mode", 2474 (double)rte_get_tsc_hz() / 1000000000.0); 2475 2476 /* Set number of lcores */ 2477 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 2478 ? ad->nb_queues 2479 : op_params->num_lcores; 2480 2481 /* Allocate memory for thread parameters structure */ 2482 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 2483 RTE_CACHE_LINE_SIZE); 2484 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 2485 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 2486 RTE_CACHE_LINE_SIZE)); 2487 2488 if (intr_enabled) { 2489 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2490 throughput_function = throughput_intr_lcore_dec; 2491 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2492 throughput_function = throughput_intr_lcore_dec; 2493 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2494 throughput_function = throughput_intr_lcore_enc; 2495 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2496 throughput_function = throughput_intr_lcore_enc; 2497 else 2498 throughput_function = throughput_intr_lcore_enc; 2499 2500 /* Dequeue interrupt callback registration */ 2501 ret = rte_bbdev_callback_register(ad->dev_id, 2502 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 2503 t_params); 2504 if (ret < 0) { 2505 rte_free(t_params); 2506 return ret; 2507 } 2508 } else { 2509 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2510 throughput_function = throughput_pmd_lcore_dec; 2511 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2512 throughput_function = throughput_pmd_lcore_ldpc_dec; 2513 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2514 throughput_function = throughput_pmd_lcore_enc; 2515 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2516 throughput_function = throughput_pmd_lcore_ldpc_enc; 2517 else 2518 throughput_function = throughput_pmd_lcore_enc; 2519 } 2520 2521 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 2522 2523 /* Master core is set at first entry */ 2524 t_params[0].dev_id = ad->dev_id; 2525 t_params[0].lcore_id = rte_lcore_id(); 2526 t_params[0].op_params = op_params; 2527 t_params[0].queue_id = ad->queue_ids[used_cores++]; 2528 t_params[0].iter_count = 0; 2529 2530 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 2531 if (used_cores >= num_lcores) 2532 break; 2533 2534 t_params[used_cores].dev_id = ad->dev_id; 2535 t_params[used_cores].lcore_id = lcore_id; 2536 t_params[used_cores].op_params = op_params; 2537 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 2538 t_params[used_cores].iter_count = 0; 2539 2540 rte_eal_remote_launch(throughput_function, 2541 &t_params[used_cores++], lcore_id); 2542 } 2543 2544 rte_atomic16_set(&op_params->sync, SYNC_START); 2545 ret = throughput_function(&t_params[0]); 2546 2547 /* Master core is always used */ 2548 for (used_cores = 1; used_cores < num_lcores; used_cores++) 2549 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 2550 2551 /* Return if test failed */ 2552 if (ret) { 2553 rte_free(t_params); 2554 return ret; 2555 } 2556 2557 /* Print throughput if interrupts are disabled and test passed */ 2558 if (!intr_enabled) { 2559 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2560 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2561 print_dec_throughput(t_params, num_lcores); 2562 else 2563 print_enc_throughput(t_params, num_lcores); 2564 rte_free(t_params); 2565 return ret; 2566 } 2567 2568 /* In interrupt TC we need to wait for the interrupt callback to deqeue 2569 * all pending operations. Skip waiting for queues which reported an 2570 * error using processing_status variable. 2571 * Wait for master lcore operations. 2572 */ 2573 tp = &t_params[0]; 2574 while ((rte_atomic16_read(&tp->nb_dequeued) < 2575 op_params->num_to_process) && 2576 (rte_atomic16_read(&tp->processing_status) != 2577 TEST_FAILED)) 2578 rte_pause(); 2579 2580 tp->ops_per_sec /= TEST_REPETITIONS; 2581 tp->mbps /= TEST_REPETITIONS; 2582 ret |= (int)rte_atomic16_read(&tp->processing_status); 2583 2584 /* Wait for slave lcores operations */ 2585 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 2586 tp = &t_params[used_cores]; 2587 2588 while ((rte_atomic16_read(&tp->nb_dequeued) < 2589 op_params->num_to_process) && 2590 (rte_atomic16_read(&tp->processing_status) != 2591 TEST_FAILED)) 2592 rte_pause(); 2593 2594 tp->ops_per_sec /= TEST_REPETITIONS; 2595 tp->mbps /= TEST_REPETITIONS; 2596 ret |= (int)rte_atomic16_read(&tp->processing_status); 2597 } 2598 2599 /* Print throughput if test passed */ 2600 if (!ret) { 2601 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 2602 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2603 print_dec_throughput(t_params, num_lcores); 2604 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 2605 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2606 print_enc_throughput(t_params, num_lcores); 2607 } 2608 2609 rte_free(t_params); 2610 return ret; 2611 } 2612 2613 static int 2614 latency_test_dec(struct rte_mempool *mempool, 2615 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2616 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2617 const uint16_t num_to_process, uint16_t burst_sz, 2618 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2619 { 2620 int ret = TEST_SUCCESS; 2621 uint16_t i, j, dequeued; 2622 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2623 uint64_t start_time = 0, last_time = 0; 2624 2625 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2626 uint16_t enq = 0, deq = 0; 2627 bool first_time = true; 2628 last_time = 0; 2629 2630 if (unlikely(num_to_process - dequeued < burst_sz)) 2631 burst_sz = num_to_process - dequeued; 2632 2633 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2634 TEST_ASSERT_SUCCESS(ret, 2635 "rte_bbdev_dec_op_alloc_bulk() failed"); 2636 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2637 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 2638 bufs->inputs, 2639 bufs->hard_outputs, 2640 bufs->soft_outputs, 2641 ref_op); 2642 2643 /* Set counter to validate the ordering */ 2644 for (j = 0; j < burst_sz; ++j) 2645 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2646 2647 start_time = rte_rdtsc_precise(); 2648 2649 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 2650 burst_sz); 2651 TEST_ASSERT(enq == burst_sz, 2652 "Error enqueueing burst, expected %u, got %u", 2653 burst_sz, enq); 2654 2655 /* Dequeue */ 2656 do { 2657 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2658 &ops_deq[deq], burst_sz - deq); 2659 if (likely(first_time && (deq > 0))) { 2660 last_time = rte_rdtsc_precise() - start_time; 2661 first_time = false; 2662 } 2663 } while (unlikely(burst_sz != deq)); 2664 2665 *max_time = RTE_MAX(*max_time, last_time); 2666 *min_time = RTE_MIN(*min_time, last_time); 2667 *total_time += last_time; 2668 2669 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2670 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 2671 vector_mask); 2672 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2673 } 2674 2675 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2676 dequeued += deq; 2677 } 2678 2679 return i; 2680 } 2681 2682 static int 2683 latency_test_ldpc_dec(struct rte_mempool *mempool, 2684 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 2685 int vector_mask, uint16_t dev_id, uint16_t queue_id, 2686 const uint16_t num_to_process, uint16_t burst_sz, 2687 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2688 { 2689 int ret = TEST_SUCCESS; 2690 uint16_t i, j, dequeued; 2691 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2692 uint64_t start_time = 0, last_time = 0; 2693 2694 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2695 uint16_t enq = 0, deq = 0; 2696 bool first_time = true; 2697 last_time = 0; 2698 2699 if (unlikely(num_to_process - dequeued < burst_sz)) 2700 burst_sz = num_to_process - dequeued; 2701 2702 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 2703 TEST_ASSERT_SUCCESS(ret, 2704 "rte_bbdev_dec_op_alloc_bulk() failed"); 2705 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2706 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 2707 bufs->inputs, 2708 bufs->hard_outputs, 2709 bufs->soft_outputs, 2710 bufs->harq_inputs, 2711 bufs->harq_outputs, 2712 ref_op); 2713 2714 /* Set counter to validate the ordering */ 2715 for (j = 0; j < burst_sz; ++j) 2716 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2717 2718 start_time = rte_rdtsc_precise(); 2719 2720 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2721 &ops_enq[enq], burst_sz); 2722 TEST_ASSERT(enq == burst_sz, 2723 "Error enqueueing burst, expected %u, got %u", 2724 burst_sz, enq); 2725 2726 /* Dequeue */ 2727 do { 2728 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2729 &ops_deq[deq], burst_sz - deq); 2730 if (likely(first_time && (deq > 0))) { 2731 last_time = rte_rdtsc_precise() - start_time; 2732 first_time = false; 2733 } 2734 } while (unlikely(burst_sz != deq)); 2735 2736 *max_time = RTE_MAX(*max_time, last_time); 2737 *min_time = RTE_MIN(*min_time, last_time); 2738 *total_time += last_time; 2739 2740 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2741 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 2742 vector_mask); 2743 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2744 } 2745 2746 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 2747 dequeued += deq; 2748 } 2749 2750 return i; 2751 } 2752 2753 static int 2754 latency_test_enc(struct rte_mempool *mempool, 2755 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2756 uint16_t dev_id, uint16_t queue_id, 2757 const uint16_t num_to_process, uint16_t burst_sz, 2758 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2759 { 2760 int ret = TEST_SUCCESS; 2761 uint16_t i, j, dequeued; 2762 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2763 uint64_t start_time = 0, last_time = 0; 2764 2765 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2766 uint16_t enq = 0, deq = 0; 2767 bool first_time = true; 2768 last_time = 0; 2769 2770 if (unlikely(num_to_process - dequeued < burst_sz)) 2771 burst_sz = num_to_process - dequeued; 2772 2773 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2774 TEST_ASSERT_SUCCESS(ret, 2775 "rte_bbdev_enc_op_alloc_bulk() failed"); 2776 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2777 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 2778 bufs->inputs, 2779 bufs->hard_outputs, 2780 ref_op); 2781 2782 /* Set counter to validate the ordering */ 2783 for (j = 0; j < burst_sz; ++j) 2784 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2785 2786 start_time = rte_rdtsc_precise(); 2787 2788 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 2789 burst_sz); 2790 TEST_ASSERT(enq == burst_sz, 2791 "Error enqueueing burst, expected %u, got %u", 2792 burst_sz, enq); 2793 2794 /* Dequeue */ 2795 do { 2796 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2797 &ops_deq[deq], burst_sz - deq); 2798 if (likely(first_time && (deq > 0))) { 2799 last_time += rte_rdtsc_precise() - start_time; 2800 first_time = false; 2801 } 2802 } while (unlikely(burst_sz != deq)); 2803 2804 *max_time = RTE_MAX(*max_time, last_time); 2805 *min_time = RTE_MIN(*min_time, last_time); 2806 *total_time += last_time; 2807 2808 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2809 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2810 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2811 } 2812 2813 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2814 dequeued += deq; 2815 } 2816 2817 return i; 2818 } 2819 2820 static int 2821 latency_test_ldpc_enc(struct rte_mempool *mempool, 2822 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 2823 uint16_t dev_id, uint16_t queue_id, 2824 const uint16_t num_to_process, uint16_t burst_sz, 2825 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 2826 { 2827 int ret = TEST_SUCCESS; 2828 uint16_t i, j, dequeued; 2829 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2830 uint64_t start_time = 0, last_time = 0; 2831 2832 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2833 uint16_t enq = 0, deq = 0; 2834 bool first_time = true; 2835 last_time = 0; 2836 2837 if (unlikely(num_to_process - dequeued < burst_sz)) 2838 burst_sz = num_to_process - dequeued; 2839 2840 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 2841 2842 TEST_ASSERT_SUCCESS(ret, 2843 "rte_bbdev_enc_op_alloc_bulk() failed"); 2844 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2845 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 2846 bufs->inputs, 2847 bufs->hard_outputs, 2848 ref_op); 2849 2850 /* Set counter to validate the ordering */ 2851 for (j = 0; j < burst_sz; ++j) 2852 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 2853 2854 start_time = rte_rdtsc_precise(); 2855 2856 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 2857 &ops_enq[enq], burst_sz); 2858 TEST_ASSERT(enq == burst_sz, 2859 "Error enqueueing burst, expected %u, got %u", 2860 burst_sz, enq); 2861 2862 /* Dequeue */ 2863 do { 2864 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2865 &ops_deq[deq], burst_sz - deq); 2866 if (likely(first_time && (deq > 0))) { 2867 last_time += rte_rdtsc_precise() - start_time; 2868 first_time = false; 2869 } 2870 } while (unlikely(burst_sz != deq)); 2871 2872 *max_time = RTE_MAX(*max_time, last_time); 2873 *min_time = RTE_MIN(*min_time, last_time); 2874 *total_time += last_time; 2875 2876 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 2877 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 2878 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 2879 } 2880 2881 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 2882 dequeued += deq; 2883 } 2884 2885 return i; 2886 } 2887 2888 static int 2889 latency_test(struct active_device *ad, 2890 struct test_op_params *op_params) 2891 { 2892 int iter; 2893 uint16_t burst_sz = op_params->burst_sz; 2894 const uint16_t num_to_process = op_params->num_to_process; 2895 const enum rte_bbdev_op_type op_type = test_vector.op_type; 2896 const uint16_t queue_id = ad->queue_ids[0]; 2897 struct test_buffers *bufs = NULL; 2898 struct rte_bbdev_info info; 2899 uint64_t total_time, min_time, max_time; 2900 const char *op_type_str; 2901 2902 total_time = max_time = 0; 2903 min_time = UINT64_MAX; 2904 2905 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2906 "BURST_SIZE should be <= %u", MAX_BURST); 2907 2908 rte_bbdev_info_get(ad->dev_id, &info); 2909 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2910 2911 op_type_str = rte_bbdev_op_type_str(op_type); 2912 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 2913 2914 printf("+ ------------------------------------------------------- +\n"); 2915 printf("== test: validation/latency\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 2916 info.dev_name, burst_sz, num_to_process, op_type_str); 2917 2918 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 2919 iter = latency_test_dec(op_params->mp, bufs, 2920 op_params->ref_dec_op, op_params->vector_mask, 2921 ad->dev_id, queue_id, num_to_process, 2922 burst_sz, &total_time, &min_time, &max_time); 2923 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 2924 iter = latency_test_enc(op_params->mp, bufs, 2925 op_params->ref_enc_op, ad->dev_id, queue_id, 2926 num_to_process, burst_sz, &total_time, 2927 &min_time, &max_time); 2928 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 2929 iter = latency_test_ldpc_enc(op_params->mp, bufs, 2930 op_params->ref_enc_op, ad->dev_id, queue_id, 2931 num_to_process, burst_sz, &total_time, 2932 &min_time, &max_time); 2933 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 2934 iter = latency_test_ldpc_dec(op_params->mp, bufs, 2935 op_params->ref_dec_op, op_params->vector_mask, 2936 ad->dev_id, queue_id, num_to_process, 2937 burst_sz, &total_time, &min_time, &max_time); 2938 else 2939 iter = latency_test_enc(op_params->mp, bufs, 2940 op_params->ref_enc_op, 2941 ad->dev_id, queue_id, 2942 num_to_process, burst_sz, &total_time, 2943 &min_time, &max_time); 2944 2945 if (iter <= 0) 2946 return TEST_FAILED; 2947 2948 printf("Operation latency:\n" 2949 "\tavg: %lg cycles, %lg us\n" 2950 "\tmin: %lg cycles, %lg us\n" 2951 "\tmax: %lg cycles, %lg us\n", 2952 (double)total_time / (double)iter, 2953 (double)(total_time * 1000000) / (double)iter / 2954 (double)rte_get_tsc_hz(), (double)min_time, 2955 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 2956 (double)max_time, (double)(max_time * 1000000) / 2957 (double)rte_get_tsc_hz()); 2958 2959 return TEST_SUCCESS; 2960 } 2961 2962 #ifdef RTE_BBDEV_OFFLOAD_COST 2963 static int 2964 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 2965 struct rte_bbdev_stats *stats) 2966 { 2967 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 2968 struct rte_bbdev_stats *q_stats; 2969 2970 if (queue_id >= dev->data->num_queues) 2971 return -1; 2972 2973 q_stats = &dev->data->queues[queue_id].queue_stats; 2974 2975 stats->enqueued_count = q_stats->enqueued_count; 2976 stats->dequeued_count = q_stats->dequeued_count; 2977 stats->enqueue_err_count = q_stats->enqueue_err_count; 2978 stats->dequeue_err_count = q_stats->dequeue_err_count; 2979 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 2980 2981 return 0; 2982 } 2983 2984 static int 2985 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 2986 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 2987 uint16_t queue_id, const uint16_t num_to_process, 2988 uint16_t burst_sz, struct test_time_stats *time_st) 2989 { 2990 int i, dequeued, ret; 2991 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 2992 uint64_t enq_start_time, deq_start_time; 2993 uint64_t enq_sw_last_time, deq_last_time; 2994 struct rte_bbdev_stats stats; 2995 2996 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 2997 uint16_t enq = 0, deq = 0; 2998 2999 if (unlikely(num_to_process - dequeued < burst_sz)) 3000 burst_sz = num_to_process - dequeued; 3001 3002 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3003 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3004 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3005 bufs->inputs, 3006 bufs->hard_outputs, 3007 bufs->soft_outputs, 3008 ref_op); 3009 3010 /* Start time meas for enqueue function offload latency */ 3011 enq_start_time = rte_rdtsc_precise(); 3012 do { 3013 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 3014 &ops_enq[enq], burst_sz - enq); 3015 } while (unlikely(burst_sz != enq)); 3016 3017 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3018 TEST_ASSERT_SUCCESS(ret, 3019 "Failed to get stats for queue (%u) of device (%u)", 3020 queue_id, dev_id); 3021 3022 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3023 stats.acc_offload_cycles; 3024 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3025 enq_sw_last_time); 3026 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3027 enq_sw_last_time); 3028 time_st->enq_sw_total_time += enq_sw_last_time; 3029 3030 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3031 stats.acc_offload_cycles); 3032 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3033 stats.acc_offload_cycles); 3034 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3035 3036 /* give time for device to process ops */ 3037 rte_delay_us(200); 3038 3039 /* Start time meas for dequeue function offload latency */ 3040 deq_start_time = rte_rdtsc_precise(); 3041 /* Dequeue one operation */ 3042 do { 3043 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3044 &ops_deq[deq], 1); 3045 } while (unlikely(deq != 1)); 3046 3047 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3048 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3049 deq_last_time); 3050 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3051 deq_last_time); 3052 time_st->deq_total_time += deq_last_time; 3053 3054 /* Dequeue remaining operations if needed*/ 3055 while (burst_sz != deq) 3056 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3057 &ops_deq[deq], burst_sz - deq); 3058 3059 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3060 dequeued += deq; 3061 } 3062 3063 return i; 3064 } 3065 3066 static int 3067 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 3068 struct test_buffers *bufs, 3069 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 3070 uint16_t queue_id, const uint16_t num_to_process, 3071 uint16_t burst_sz, struct test_time_stats *time_st) 3072 { 3073 int i, dequeued, ret; 3074 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3075 uint64_t enq_start_time, deq_start_time; 3076 uint64_t enq_sw_last_time, deq_last_time; 3077 struct rte_bbdev_stats stats; 3078 3079 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3080 uint16_t enq = 0, deq = 0; 3081 3082 if (unlikely(num_to_process - dequeued < burst_sz)) 3083 burst_sz = num_to_process - dequeued; 3084 3085 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3086 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3087 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 3088 bufs->inputs, 3089 bufs->hard_outputs, 3090 bufs->soft_outputs, 3091 bufs->harq_inputs, 3092 bufs->harq_outputs, 3093 ref_op); 3094 3095 /* Start time meas for enqueue function offload latency */ 3096 enq_start_time = rte_rdtsc_precise(); 3097 do { 3098 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 3099 &ops_enq[enq], burst_sz - enq); 3100 } while (unlikely(burst_sz != enq)); 3101 3102 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3103 TEST_ASSERT_SUCCESS(ret, 3104 "Failed to get stats for queue (%u) of device (%u)", 3105 queue_id, dev_id); 3106 3107 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3108 stats.acc_offload_cycles; 3109 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3110 enq_sw_last_time); 3111 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3112 enq_sw_last_time); 3113 time_st->enq_sw_total_time += enq_sw_last_time; 3114 3115 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3116 stats.acc_offload_cycles); 3117 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3118 stats.acc_offload_cycles); 3119 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3120 3121 /* give time for device to process ops */ 3122 rte_delay_us(200); 3123 3124 /* Start time meas for dequeue function offload latency */ 3125 deq_start_time = rte_rdtsc_precise(); 3126 /* Dequeue one operation */ 3127 do { 3128 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 3129 &ops_deq[deq], 1); 3130 } while (unlikely(deq != 1)); 3131 3132 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3133 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3134 deq_last_time); 3135 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3136 deq_last_time); 3137 time_st->deq_total_time += deq_last_time; 3138 3139 /* Dequeue remaining operations if needed*/ 3140 while (burst_sz != deq) 3141 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3142 &ops_deq[deq], burst_sz - deq); 3143 3144 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3145 dequeued += deq; 3146 } 3147 3148 return i; 3149 } 3150 3151 static int 3152 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 3153 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3154 uint16_t queue_id, const uint16_t num_to_process, 3155 uint16_t burst_sz, struct test_time_stats *time_st) 3156 { 3157 int i, dequeued, ret; 3158 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3159 uint64_t enq_start_time, deq_start_time; 3160 uint64_t enq_sw_last_time, deq_last_time; 3161 struct rte_bbdev_stats stats; 3162 3163 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3164 uint16_t enq = 0, deq = 0; 3165 3166 if (unlikely(num_to_process - dequeued < burst_sz)) 3167 burst_sz = num_to_process - dequeued; 3168 3169 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3170 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3171 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3172 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 3173 bufs->inputs, 3174 bufs->hard_outputs, 3175 ref_op); 3176 3177 /* Start time meas for enqueue function offload latency */ 3178 enq_start_time = rte_rdtsc_precise(); 3179 do { 3180 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 3181 &ops_enq[enq], burst_sz - enq); 3182 } while (unlikely(burst_sz != enq)); 3183 3184 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3185 TEST_ASSERT_SUCCESS(ret, 3186 "Failed to get stats for queue (%u) of device (%u)", 3187 queue_id, dev_id); 3188 3189 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3190 stats.acc_offload_cycles; 3191 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3192 enq_sw_last_time); 3193 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3194 enq_sw_last_time); 3195 time_st->enq_sw_total_time += enq_sw_last_time; 3196 3197 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3198 stats.acc_offload_cycles); 3199 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3200 stats.acc_offload_cycles); 3201 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3202 3203 /* give time for device to process ops */ 3204 rte_delay_us(200); 3205 3206 /* Start time meas for dequeue function offload latency */ 3207 deq_start_time = rte_rdtsc_precise(); 3208 /* Dequeue one operation */ 3209 do { 3210 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3211 &ops_deq[deq], 1); 3212 } while (unlikely(deq != 1)); 3213 3214 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3215 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3216 deq_last_time); 3217 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3218 deq_last_time); 3219 time_st->deq_total_time += deq_last_time; 3220 3221 while (burst_sz != deq) 3222 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 3223 &ops_deq[deq], burst_sz - deq); 3224 3225 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3226 dequeued += deq; 3227 } 3228 3229 return i; 3230 } 3231 3232 static int 3233 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 3234 struct test_buffers *bufs, 3235 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 3236 uint16_t queue_id, const uint16_t num_to_process, 3237 uint16_t burst_sz, struct test_time_stats *time_st) 3238 { 3239 int i, dequeued, ret; 3240 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3241 uint64_t enq_start_time, deq_start_time; 3242 uint64_t enq_sw_last_time, deq_last_time; 3243 struct rte_bbdev_stats stats; 3244 3245 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3246 uint16_t enq = 0, deq = 0; 3247 3248 if (unlikely(num_to_process - dequeued < burst_sz)) 3249 burst_sz = num_to_process - dequeued; 3250 3251 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 3252 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 3253 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3254 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 3255 bufs->inputs, 3256 bufs->hard_outputs, 3257 ref_op); 3258 3259 /* Start time meas for enqueue function offload latency */ 3260 enq_start_time = rte_rdtsc_precise(); 3261 do { 3262 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 3263 &ops_enq[enq], burst_sz - enq); 3264 } while (unlikely(burst_sz != enq)); 3265 3266 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 3267 TEST_ASSERT_SUCCESS(ret, 3268 "Failed to get stats for queue (%u) of device (%u)", 3269 queue_id, dev_id); 3270 3271 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 3272 stats.acc_offload_cycles; 3273 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 3274 enq_sw_last_time); 3275 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 3276 enq_sw_last_time); 3277 time_st->enq_sw_total_time += enq_sw_last_time; 3278 3279 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 3280 stats.acc_offload_cycles); 3281 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 3282 stats.acc_offload_cycles); 3283 time_st->enq_acc_total_time += stats.acc_offload_cycles; 3284 3285 /* give time for device to process ops */ 3286 rte_delay_us(200); 3287 3288 /* Start time meas for dequeue function offload latency */ 3289 deq_start_time = rte_rdtsc_precise(); 3290 /* Dequeue one operation */ 3291 do { 3292 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3293 &ops_deq[deq], 1); 3294 } while (unlikely(deq != 1)); 3295 3296 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3297 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 3298 deq_last_time); 3299 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 3300 deq_last_time); 3301 time_st->deq_total_time += deq_last_time; 3302 3303 while (burst_sz != deq) 3304 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 3305 &ops_deq[deq], burst_sz - deq); 3306 3307 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 3308 dequeued += deq; 3309 } 3310 3311 return i; 3312 } 3313 #endif 3314 3315 static int 3316 offload_cost_test(struct active_device *ad, 3317 struct test_op_params *op_params) 3318 { 3319 #ifndef RTE_BBDEV_OFFLOAD_COST 3320 RTE_SET_USED(ad); 3321 RTE_SET_USED(op_params); 3322 printf("Offload latency test is disabled.\n"); 3323 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3324 return TEST_SKIPPED; 3325 #else 3326 int iter; 3327 uint16_t burst_sz = op_params->burst_sz; 3328 const uint16_t num_to_process = op_params->num_to_process; 3329 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3330 const uint16_t queue_id = ad->queue_ids[0]; 3331 struct test_buffers *bufs = NULL; 3332 struct rte_bbdev_info info; 3333 const char *op_type_str; 3334 struct test_time_stats time_st; 3335 3336 memset(&time_st, 0, sizeof(struct test_time_stats)); 3337 time_st.enq_sw_min_time = UINT64_MAX; 3338 time_st.enq_acc_min_time = UINT64_MAX; 3339 time_st.deq_min_time = UINT64_MAX; 3340 3341 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3342 "BURST_SIZE should be <= %u", MAX_BURST); 3343 3344 rte_bbdev_info_get(ad->dev_id, &info); 3345 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3346 3347 op_type_str = rte_bbdev_op_type_str(op_type); 3348 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3349 3350 printf("+ ------------------------------------------------------- +\n"); 3351 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3352 info.dev_name, burst_sz, num_to_process, op_type_str); 3353 3354 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3355 iter = offload_latency_test_dec(op_params->mp, bufs, 3356 op_params->ref_dec_op, ad->dev_id, queue_id, 3357 num_to_process, burst_sz, &time_st); 3358 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 3359 iter = offload_latency_test_enc(op_params->mp, bufs, 3360 op_params->ref_enc_op, ad->dev_id, queue_id, 3361 num_to_process, burst_sz, &time_st); 3362 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 3363 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 3364 op_params->ref_enc_op, ad->dev_id, queue_id, 3365 num_to_process, burst_sz, &time_st); 3366 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 3367 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 3368 op_params->ref_dec_op, ad->dev_id, queue_id, 3369 num_to_process, burst_sz, &time_st); 3370 else 3371 iter = offload_latency_test_enc(op_params->mp, bufs, 3372 op_params->ref_enc_op, ad->dev_id, queue_id, 3373 num_to_process, burst_sz, &time_st); 3374 3375 if (iter <= 0) 3376 return TEST_FAILED; 3377 3378 printf("Enqueue driver offload cost latency:\n" 3379 "\tavg: %lg cycles, %lg us\n" 3380 "\tmin: %lg cycles, %lg us\n" 3381 "\tmax: %lg cycles, %lg us\n" 3382 "Enqueue accelerator offload cost latency:\n" 3383 "\tavg: %lg cycles, %lg us\n" 3384 "\tmin: %lg cycles, %lg us\n" 3385 "\tmax: %lg cycles, %lg us\n", 3386 (double)time_st.enq_sw_total_time / (double)iter, 3387 (double)(time_st.enq_sw_total_time * 1000000) / 3388 (double)iter / (double)rte_get_tsc_hz(), 3389 (double)time_st.enq_sw_min_time, 3390 (double)(time_st.enq_sw_min_time * 1000000) / 3391 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 3392 (double)(time_st.enq_sw_max_time * 1000000) / 3393 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 3394 (double)iter, 3395 (double)(time_st.enq_acc_total_time * 1000000) / 3396 (double)iter / (double)rte_get_tsc_hz(), 3397 (double)time_st.enq_acc_min_time, 3398 (double)(time_st.enq_acc_min_time * 1000000) / 3399 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 3400 (double)(time_st.enq_acc_max_time * 1000000) / 3401 rte_get_tsc_hz()); 3402 3403 printf("Dequeue offload cost latency - one op:\n" 3404 "\tavg: %lg cycles, %lg us\n" 3405 "\tmin: %lg cycles, %lg us\n" 3406 "\tmax: %lg cycles, %lg us\n", 3407 (double)time_st.deq_total_time / (double)iter, 3408 (double)(time_st.deq_total_time * 1000000) / 3409 (double)iter / (double)rte_get_tsc_hz(), 3410 (double)time_st.deq_min_time, 3411 (double)(time_st.deq_min_time * 1000000) / 3412 rte_get_tsc_hz(), (double)time_st.deq_max_time, 3413 (double)(time_st.deq_max_time * 1000000) / 3414 rte_get_tsc_hz()); 3415 3416 return TEST_SUCCESS; 3417 #endif 3418 } 3419 3420 #ifdef RTE_BBDEV_OFFLOAD_COST 3421 static int 3422 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 3423 const uint16_t num_to_process, uint16_t burst_sz, 3424 uint64_t *deq_total_time, uint64_t *deq_min_time, 3425 uint64_t *deq_max_time) 3426 { 3427 int i, deq_total; 3428 struct rte_bbdev_dec_op *ops[MAX_BURST]; 3429 uint64_t deq_start_time, deq_last_time; 3430 3431 /* Test deq offload latency from an empty queue */ 3432 3433 for (i = 0, deq_total = 0; deq_total < num_to_process; 3434 ++i, deq_total += burst_sz) { 3435 deq_start_time = rte_rdtsc_precise(); 3436 3437 if (unlikely(num_to_process - deq_total < burst_sz)) 3438 burst_sz = num_to_process - deq_total; 3439 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); 3440 3441 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3442 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3443 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3444 *deq_total_time += deq_last_time; 3445 } 3446 3447 return i; 3448 } 3449 3450 static int 3451 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 3452 const uint16_t num_to_process, uint16_t burst_sz, 3453 uint64_t *deq_total_time, uint64_t *deq_min_time, 3454 uint64_t *deq_max_time) 3455 { 3456 int i, deq_total; 3457 struct rte_bbdev_enc_op *ops[MAX_BURST]; 3458 uint64_t deq_start_time, deq_last_time; 3459 3460 /* Test deq offload latency from an empty queue */ 3461 for (i = 0, deq_total = 0; deq_total < num_to_process; 3462 ++i, deq_total += burst_sz) { 3463 deq_start_time = rte_rdtsc_precise(); 3464 3465 if (unlikely(num_to_process - deq_total < burst_sz)) 3466 burst_sz = num_to_process - deq_total; 3467 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); 3468 3469 deq_last_time = rte_rdtsc_precise() - deq_start_time; 3470 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 3471 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 3472 *deq_total_time += deq_last_time; 3473 } 3474 3475 return i; 3476 } 3477 #endif 3478 3479 static int 3480 offload_latency_empty_q_test(struct active_device *ad, 3481 struct test_op_params *op_params) 3482 { 3483 #ifndef RTE_BBDEV_OFFLOAD_COST 3484 RTE_SET_USED(ad); 3485 RTE_SET_USED(op_params); 3486 printf("Offload latency empty dequeue test is disabled.\n"); 3487 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 3488 return TEST_SKIPPED; 3489 #else 3490 int iter; 3491 uint64_t deq_total_time, deq_min_time, deq_max_time; 3492 uint16_t burst_sz = op_params->burst_sz; 3493 const uint16_t num_to_process = op_params->num_to_process; 3494 const enum rte_bbdev_op_type op_type = test_vector.op_type; 3495 const uint16_t queue_id = ad->queue_ids[0]; 3496 struct rte_bbdev_info info; 3497 const char *op_type_str; 3498 3499 deq_total_time = deq_max_time = 0; 3500 deq_min_time = UINT64_MAX; 3501 3502 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3503 "BURST_SIZE should be <= %u", MAX_BURST); 3504 3505 rte_bbdev_info_get(ad->dev_id, &info); 3506 3507 op_type_str = rte_bbdev_op_type_str(op_type); 3508 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 3509 3510 printf("+ ------------------------------------------------------- +\n"); 3511 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 3512 info.dev_name, burst_sz, num_to_process, op_type_str); 3513 3514 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 3515 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 3516 num_to_process, burst_sz, &deq_total_time, 3517 &deq_min_time, &deq_max_time); 3518 else 3519 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 3520 num_to_process, burst_sz, &deq_total_time, 3521 &deq_min_time, &deq_max_time); 3522 3523 if (iter <= 0) 3524 return TEST_FAILED; 3525 3526 printf("Empty dequeue offload:\n" 3527 "\tavg: %lg cycles, %lg us\n" 3528 "\tmin: %lg cycles, %lg us\n" 3529 "\tmax: %lg cycles, %lg us\n", 3530 (double)deq_total_time / (double)iter, 3531 (double)(deq_total_time * 1000000) / (double)iter / 3532 (double)rte_get_tsc_hz(), (double)deq_min_time, 3533 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 3534 (double)deq_max_time, (double)(deq_max_time * 1000000) / 3535 rte_get_tsc_hz()); 3536 3537 return TEST_SUCCESS; 3538 #endif 3539 } 3540 3541 static int 3542 throughput_tc(void) 3543 { 3544 return run_test_case(throughput_test); 3545 } 3546 3547 static int 3548 offload_cost_tc(void) 3549 { 3550 return run_test_case(offload_cost_test); 3551 } 3552 3553 static int 3554 offload_latency_empty_q_tc(void) 3555 { 3556 return run_test_case(offload_latency_empty_q_test); 3557 } 3558 3559 static int 3560 latency_tc(void) 3561 { 3562 return run_test_case(latency_test); 3563 } 3564 3565 static int 3566 interrupt_tc(void) 3567 { 3568 return run_test_case(throughput_test); 3569 } 3570 3571 static struct unit_test_suite bbdev_throughput_testsuite = { 3572 .suite_name = "BBdev Throughput Tests", 3573 .setup = testsuite_setup, 3574 .teardown = testsuite_teardown, 3575 .unit_test_cases = { 3576 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 3577 TEST_CASES_END() /**< NULL terminate unit test array */ 3578 } 3579 }; 3580 3581 static struct unit_test_suite bbdev_validation_testsuite = { 3582 .suite_name = "BBdev Validation Tests", 3583 .setup = testsuite_setup, 3584 .teardown = testsuite_teardown, 3585 .unit_test_cases = { 3586 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3587 TEST_CASES_END() /**< NULL terminate unit test array */ 3588 } 3589 }; 3590 3591 static struct unit_test_suite bbdev_latency_testsuite = { 3592 .suite_name = "BBdev Latency Tests", 3593 .setup = testsuite_setup, 3594 .teardown = testsuite_teardown, 3595 .unit_test_cases = { 3596 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 3597 TEST_CASES_END() /**< NULL terminate unit test array */ 3598 } 3599 }; 3600 3601 static struct unit_test_suite bbdev_offload_cost_testsuite = { 3602 .suite_name = "BBdev Offload Cost Tests", 3603 .setup = testsuite_setup, 3604 .teardown = testsuite_teardown, 3605 .unit_test_cases = { 3606 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 3607 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 3608 TEST_CASES_END() /**< NULL terminate unit test array */ 3609 } 3610 }; 3611 3612 static struct unit_test_suite bbdev_interrupt_testsuite = { 3613 .suite_name = "BBdev Interrupt Tests", 3614 .setup = interrupt_testsuite_setup, 3615 .teardown = testsuite_teardown, 3616 .unit_test_cases = { 3617 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 3618 TEST_CASES_END() /**< NULL terminate unit test array */ 3619 } 3620 }; 3621 3622 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 3623 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 3624 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 3625 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 3626 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 3627