1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <stdio.h> 6 #include <inttypes.h> 7 #include <math.h> 8 9 #include <rte_eal.h> 10 #include <rte_common.h> 11 #include <rte_dev.h> 12 #include <rte_launch.h> 13 #include <rte_bbdev.h> 14 #include <rte_cycles.h> 15 #include <rte_lcore.h> 16 #include <rte_malloc.h> 17 #include <rte_random.h> 18 #include <rte_hexdump.h> 19 #include <rte_interrupts.h> 20 21 #include "main.h" 22 #include "test_bbdev_vector.h" 23 24 #define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) 25 26 #define MAX_QUEUES RTE_MAX_LCORE 27 #define TEST_REPETITIONS 1000 28 29 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 30 #include <fpga_lte_fec.h> 31 #define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf") 32 #define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf") 33 #define VF_UL_4G_QUEUE_VALUE 4 34 #define VF_DL_4G_QUEUE_VALUE 4 35 #define UL_4G_BANDWIDTH 3 36 #define DL_4G_BANDWIDTH 3 37 #define UL_4G_LOAD_BALANCE 128 38 #define DL_4G_LOAD_BALANCE 128 39 #define FLR_4G_TIMEOUT 610 40 #endif 41 42 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 43 #include <rte_pmd_fpga_5gnr_fec.h> 44 #define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf") 45 #define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf") 46 #define VF_UL_5G_QUEUE_VALUE 4 47 #define VF_DL_5G_QUEUE_VALUE 4 48 #define UL_5G_BANDWIDTH 3 49 #define DL_5G_BANDWIDTH 3 50 #define UL_5G_LOAD_BALANCE 128 51 #define DL_5G_LOAD_BALANCE 128 52 #define FLR_5G_TIMEOUT 610 53 #endif 54 55 #ifdef RTE_BASEBAND_ACC100 56 #include <rte_acc100_cfg.h> 57 #define ACC100PF_DRIVER_NAME ("intel_acc100_pf") 58 #define ACC100VF_DRIVER_NAME ("intel_acc100_vf") 59 #define ACC100_QMGR_NUM_AQS 16 60 #define ACC100_QMGR_NUM_QGS 2 61 #define ACC100_QMGR_AQ_DEPTH 5 62 #define ACC100_QMGR_INVALID_IDX -1 63 #define ACC100_QMGR_RR 1 64 #define ACC100_QOS_GBR 0 65 #endif 66 67 #define OPS_CACHE_SIZE 256U 68 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ 69 70 #define SYNC_WAIT 0 71 #define SYNC_START 1 72 #define INVALID_OPAQUE -1 73 74 #define INVALID_QUEUE_ID -1 75 /* Increment for next code block in external HARQ memory */ 76 #define HARQ_INCR 32768 77 /* Headroom for filler LLRs insertion in HARQ buffer */ 78 #define FILLER_HEADROOM 1024 79 /* Constants from K0 computation from 3GPP 38.212 Table 5.4.2.1-2 */ 80 #define N_ZC_1 66 /* N = 66 Zc for BG 1 */ 81 #define N_ZC_2 50 /* N = 50 Zc for BG 2 */ 82 #define K0_1_1 17 /* K0 fraction numerator for rv 1 and BG 1 */ 83 #define K0_1_2 13 /* K0 fraction numerator for rv 1 and BG 2 */ 84 #define K0_2_1 33 /* K0 fraction numerator for rv 2 and BG 1 */ 85 #define K0_2_2 25 /* K0 fraction numerator for rv 2 and BG 2 */ 86 #define K0_3_1 56 /* K0 fraction numerator for rv 3 and BG 1 */ 87 #define K0_3_2 43 /* K0 fraction numerator for rv 3 and BG 2 */ 88 89 static struct test_bbdev_vector test_vector; 90 91 /* Switch between PMD and Interrupt for throughput TC */ 92 static bool intr_enabled; 93 94 /* LLR arithmetic representation for numerical conversion */ 95 static int ldpc_llr_decimals; 96 static int ldpc_llr_size; 97 /* Keep track of the LDPC decoder device capability flag */ 98 static uint32_t ldpc_cap_flags; 99 100 /* Represents tested active devices */ 101 static struct active_device { 102 const char *driver_name; 103 uint8_t dev_id; 104 uint16_t supported_ops; 105 uint16_t queue_ids[MAX_QUEUES]; 106 uint16_t nb_queues; 107 struct rte_mempool *ops_mempool; 108 struct rte_mempool *in_mbuf_pool; 109 struct rte_mempool *hard_out_mbuf_pool; 110 struct rte_mempool *soft_out_mbuf_pool; 111 struct rte_mempool *harq_in_mbuf_pool; 112 struct rte_mempool *harq_out_mbuf_pool; 113 } active_devs[RTE_BBDEV_MAX_DEVS]; 114 115 static uint8_t nb_active_devs; 116 117 /* Data buffers used by BBDEV ops */ 118 struct test_buffers { 119 struct rte_bbdev_op_data *inputs; 120 struct rte_bbdev_op_data *hard_outputs; 121 struct rte_bbdev_op_data *soft_outputs; 122 struct rte_bbdev_op_data *harq_inputs; 123 struct rte_bbdev_op_data *harq_outputs; 124 }; 125 126 /* Operation parameters specific for given test case */ 127 struct test_op_params { 128 struct rte_mempool *mp; 129 struct rte_bbdev_dec_op *ref_dec_op; 130 struct rte_bbdev_enc_op *ref_enc_op; 131 uint16_t burst_sz; 132 uint16_t num_to_process; 133 uint16_t num_lcores; 134 int vector_mask; 135 rte_atomic16_t sync; 136 struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; 137 }; 138 139 /* Contains per lcore params */ 140 struct thread_params { 141 uint8_t dev_id; 142 uint16_t queue_id; 143 uint32_t lcore_id; 144 uint64_t start_time; 145 double ops_per_sec; 146 double mbps; 147 uint8_t iter_count; 148 double iter_average; 149 double bler; 150 rte_atomic16_t nb_dequeued; 151 rte_atomic16_t processing_status; 152 rte_atomic16_t burst_sz; 153 struct test_op_params *op_params; 154 struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; 155 struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; 156 }; 157 158 #ifdef RTE_BBDEV_OFFLOAD_COST 159 /* Stores time statistics */ 160 struct test_time_stats { 161 /* Stores software enqueue total working time */ 162 uint64_t enq_sw_total_time; 163 /* Stores minimum value of software enqueue working time */ 164 uint64_t enq_sw_min_time; 165 /* Stores maximum value of software enqueue working time */ 166 uint64_t enq_sw_max_time; 167 /* Stores turbo enqueue total working time */ 168 uint64_t enq_acc_total_time; 169 /* Stores minimum value of accelerator enqueue working time */ 170 uint64_t enq_acc_min_time; 171 /* Stores maximum value of accelerator enqueue working time */ 172 uint64_t enq_acc_max_time; 173 /* Stores dequeue total working time */ 174 uint64_t deq_total_time; 175 /* Stores minimum value of dequeue working time */ 176 uint64_t deq_min_time; 177 /* Stores maximum value of dequeue working time */ 178 uint64_t deq_max_time; 179 }; 180 #endif 181 182 typedef int (test_case_function)(struct active_device *ad, 183 struct test_op_params *op_params); 184 185 static inline void 186 mbuf_reset(struct rte_mbuf *m) 187 { 188 m->pkt_len = 0; 189 190 do { 191 m->data_len = 0; 192 m = m->next; 193 } while (m != NULL); 194 } 195 196 /* Read flag value 0/1 from bitmap */ 197 static inline bool 198 check_bit(uint32_t bitmap, uint32_t bitmask) 199 { 200 return bitmap & bitmask; 201 } 202 203 static inline void 204 set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 205 { 206 ad->supported_ops |= (1 << op_type); 207 } 208 209 static inline bool 210 is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) 211 { 212 return ad->supported_ops & (1 << op_type); 213 } 214 215 static inline bool 216 flags_match(uint32_t flags_req, uint32_t flags_present) 217 { 218 return (flags_req & flags_present) == flags_req; 219 } 220 221 static void 222 clear_soft_out_cap(uint32_t *op_flags) 223 { 224 *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; 225 *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; 226 *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; 227 } 228 229 static int 230 check_dev_cap(const struct rte_bbdev_info *dev_info) 231 { 232 unsigned int i; 233 unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs, 234 nb_harq_inputs, nb_harq_outputs; 235 const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; 236 237 nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; 238 nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; 239 nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; 240 nb_harq_inputs = test_vector.entries[DATA_HARQ_INPUT].nb_segments; 241 nb_harq_outputs = test_vector.entries[DATA_HARQ_OUTPUT].nb_segments; 242 243 for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { 244 if (op_cap->type != test_vector.op_type) 245 continue; 246 247 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { 248 const struct rte_bbdev_op_cap_turbo_dec *cap = 249 &op_cap->cap.turbo_dec; 250 /* Ignore lack of soft output capability, just skip 251 * checking if soft output is valid. 252 */ 253 if ((test_vector.turbo_dec.op_flags & 254 RTE_BBDEV_TURBO_SOFT_OUTPUT) && 255 !(cap->capability_flags & 256 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 257 printf( 258 "INFO: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", 259 dev_info->dev_name); 260 clear_soft_out_cap( 261 &test_vector.turbo_dec.op_flags); 262 } 263 264 if (!flags_match(test_vector.turbo_dec.op_flags, 265 cap->capability_flags)) 266 return TEST_FAILED; 267 if (nb_inputs > cap->num_buffers_src) { 268 printf("Too many inputs defined: %u, max: %u\n", 269 nb_inputs, cap->num_buffers_src); 270 return TEST_FAILED; 271 } 272 if (nb_soft_outputs > cap->num_buffers_soft_out && 273 (test_vector.turbo_dec.op_flags & 274 RTE_BBDEV_TURBO_SOFT_OUTPUT)) { 275 printf( 276 "Too many soft outputs defined: %u, max: %u\n", 277 nb_soft_outputs, 278 cap->num_buffers_soft_out); 279 return TEST_FAILED; 280 } 281 if (nb_hard_outputs > cap->num_buffers_hard_out) { 282 printf( 283 "Too many hard outputs defined: %u, max: %u\n", 284 nb_hard_outputs, 285 cap->num_buffers_hard_out); 286 return TEST_FAILED; 287 } 288 if (intr_enabled && !(cap->capability_flags & 289 RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { 290 printf( 291 "Dequeue interrupts are not supported!\n"); 292 return TEST_FAILED; 293 } 294 295 return TEST_SUCCESS; 296 } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { 297 const struct rte_bbdev_op_cap_turbo_enc *cap = 298 &op_cap->cap.turbo_enc; 299 300 if (!flags_match(test_vector.turbo_enc.op_flags, 301 cap->capability_flags)) 302 return TEST_FAILED; 303 if (nb_inputs > cap->num_buffers_src) { 304 printf("Too many inputs defined: %u, max: %u\n", 305 nb_inputs, cap->num_buffers_src); 306 return TEST_FAILED; 307 } 308 if (nb_hard_outputs > cap->num_buffers_dst) { 309 printf( 310 "Too many hard outputs defined: %u, max: %u\n", 311 nb_hard_outputs, cap->num_buffers_dst); 312 return TEST_FAILED; 313 } 314 if (intr_enabled && !(cap->capability_flags & 315 RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { 316 printf( 317 "Dequeue interrupts are not supported!\n"); 318 return TEST_FAILED; 319 } 320 321 return TEST_SUCCESS; 322 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_ENC) { 323 const struct rte_bbdev_op_cap_ldpc_enc *cap = 324 &op_cap->cap.ldpc_enc; 325 326 if (!flags_match(test_vector.ldpc_enc.op_flags, 327 cap->capability_flags)){ 328 printf("Flag Mismatch\n"); 329 return TEST_FAILED; 330 } 331 if (nb_inputs > cap->num_buffers_src) { 332 printf("Too many inputs defined: %u, max: %u\n", 333 nb_inputs, cap->num_buffers_src); 334 return TEST_FAILED; 335 } 336 if (nb_hard_outputs > cap->num_buffers_dst) { 337 printf( 338 "Too many hard outputs defined: %u, max: %u\n", 339 nb_hard_outputs, cap->num_buffers_dst); 340 return TEST_FAILED; 341 } 342 if (intr_enabled && !(cap->capability_flags & 343 RTE_BBDEV_LDPC_ENC_INTERRUPTS)) { 344 printf( 345 "Dequeue interrupts are not supported!\n"); 346 return TEST_FAILED; 347 } 348 349 return TEST_SUCCESS; 350 } else if (op_cap->type == RTE_BBDEV_OP_LDPC_DEC) { 351 const struct rte_bbdev_op_cap_ldpc_dec *cap = 352 &op_cap->cap.ldpc_dec; 353 354 if (!flags_match(test_vector.ldpc_dec.op_flags, 355 cap->capability_flags)){ 356 printf("Flag Mismatch\n"); 357 return TEST_FAILED; 358 } 359 if (nb_inputs > cap->num_buffers_src) { 360 printf("Too many inputs defined: %u, max: %u\n", 361 nb_inputs, cap->num_buffers_src); 362 return TEST_FAILED; 363 } 364 if (nb_hard_outputs > cap->num_buffers_hard_out) { 365 printf( 366 "Too many hard outputs defined: %u, max: %u\n", 367 nb_hard_outputs, 368 cap->num_buffers_hard_out); 369 return TEST_FAILED; 370 } 371 if (nb_harq_inputs > cap->num_buffers_hard_out) { 372 printf( 373 "Too many HARQ inputs defined: %u, max: %u\n", 374 nb_hard_outputs, 375 cap->num_buffers_hard_out); 376 return TEST_FAILED; 377 } 378 if (nb_harq_outputs > cap->num_buffers_hard_out) { 379 printf( 380 "Too many HARQ outputs defined: %u, max: %u\n", 381 nb_hard_outputs, 382 cap->num_buffers_hard_out); 383 return TEST_FAILED; 384 } 385 if (intr_enabled && !(cap->capability_flags & 386 RTE_BBDEV_LDPC_DEC_INTERRUPTS)) { 387 printf( 388 "Dequeue interrupts are not supported!\n"); 389 return TEST_FAILED; 390 } 391 if (intr_enabled && (test_vector.ldpc_dec.op_flags & 392 (RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 393 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 394 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 395 ))) { 396 printf("Skip loop-back with interrupt\n"); 397 return TEST_FAILED; 398 } 399 return TEST_SUCCESS; 400 } 401 } 402 403 if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) 404 return TEST_SUCCESS; /* Special case for NULL device */ 405 406 return TEST_FAILED; 407 } 408 409 /* calculates optimal mempool size not smaller than the val */ 410 static unsigned int 411 optimal_mempool_size(unsigned int val) 412 { 413 return rte_align32pow2(val + 1) - 1; 414 } 415 416 /* allocates mbuf mempool for inputs and outputs */ 417 static struct rte_mempool * 418 create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, 419 int socket_id, unsigned int mbuf_pool_size, 420 const char *op_type_str) 421 { 422 unsigned int i; 423 uint32_t max_seg_sz = 0; 424 char pool_name[RTE_MEMPOOL_NAMESIZE]; 425 426 /* find max input segment size */ 427 for (i = 0; i < entries->nb_segments; ++i) 428 if (entries->segments[i].length > max_seg_sz) 429 max_seg_sz = entries->segments[i].length; 430 431 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 432 dev_id); 433 return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, 434 RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM 435 + FILLER_HEADROOM, 436 (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); 437 } 438 439 static int 440 create_mempools(struct active_device *ad, int socket_id, 441 enum rte_bbdev_op_type org_op_type, uint16_t num_ops) 442 { 443 struct rte_mempool *mp; 444 unsigned int ops_pool_size, mbuf_pool_size = 0; 445 char pool_name[RTE_MEMPOOL_NAMESIZE]; 446 const char *op_type_str; 447 enum rte_bbdev_op_type op_type = org_op_type; 448 449 struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; 450 struct op_data_entries *hard_out = 451 &test_vector.entries[DATA_HARD_OUTPUT]; 452 struct op_data_entries *soft_out = 453 &test_vector.entries[DATA_SOFT_OUTPUT]; 454 struct op_data_entries *harq_in = 455 &test_vector.entries[DATA_HARQ_INPUT]; 456 struct op_data_entries *harq_out = 457 &test_vector.entries[DATA_HARQ_OUTPUT]; 458 459 /* allocate ops mempool */ 460 ops_pool_size = optimal_mempool_size(RTE_MAX( 461 /* Ops used plus 1 reference op */ 462 RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), 463 /* Minimal cache size plus 1 reference op */ 464 (unsigned int)(1.5 * rte_lcore_count() * 465 OPS_CACHE_SIZE + 1)), 466 OPS_POOL_SIZE_MIN)); 467 468 if (org_op_type == RTE_BBDEV_OP_NONE) 469 op_type = RTE_BBDEV_OP_TURBO_ENC; 470 471 op_type_str = rte_bbdev_op_type_str(op_type); 472 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 473 474 snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, 475 ad->dev_id); 476 mp = rte_bbdev_op_pool_create(pool_name, op_type, 477 ops_pool_size, OPS_CACHE_SIZE, socket_id); 478 TEST_ASSERT_NOT_NULL(mp, 479 "ERROR Failed to create %u items ops pool for dev %u on socket %u.", 480 ops_pool_size, 481 ad->dev_id, 482 socket_id); 483 ad->ops_mempool = mp; 484 485 /* Do not create inputs and outputs mbufs for BaseBand Null Device */ 486 if (org_op_type == RTE_BBDEV_OP_NONE) 487 return TEST_SUCCESS; 488 489 /* Inputs */ 490 if (in->nb_segments > 0) { 491 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 492 in->nb_segments); 493 mp = create_mbuf_pool(in, ad->dev_id, socket_id, 494 mbuf_pool_size, "in"); 495 TEST_ASSERT_NOT_NULL(mp, 496 "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", 497 mbuf_pool_size, 498 ad->dev_id, 499 socket_id); 500 ad->in_mbuf_pool = mp; 501 } 502 503 /* Hard outputs */ 504 if (hard_out->nb_segments > 0) { 505 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 506 hard_out->nb_segments); 507 mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, 508 mbuf_pool_size, 509 "hard_out"); 510 TEST_ASSERT_NOT_NULL(mp, 511 "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", 512 mbuf_pool_size, 513 ad->dev_id, 514 socket_id); 515 ad->hard_out_mbuf_pool = mp; 516 } 517 518 /* Soft outputs */ 519 if (soft_out->nb_segments > 0) { 520 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 521 soft_out->nb_segments); 522 mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, 523 mbuf_pool_size, 524 "soft_out"); 525 TEST_ASSERT_NOT_NULL(mp, 526 "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", 527 mbuf_pool_size, 528 ad->dev_id, 529 socket_id); 530 ad->soft_out_mbuf_pool = mp; 531 } 532 533 /* HARQ inputs */ 534 if (harq_in->nb_segments > 0) { 535 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 536 harq_in->nb_segments); 537 mp = create_mbuf_pool(harq_in, ad->dev_id, socket_id, 538 mbuf_pool_size, 539 "harq_in"); 540 TEST_ASSERT_NOT_NULL(mp, 541 "ERROR Failed to create %uB harq input pktmbuf pool for dev %u on socket %u.", 542 mbuf_pool_size, 543 ad->dev_id, 544 socket_id); 545 ad->harq_in_mbuf_pool = mp; 546 } 547 548 /* HARQ outputs */ 549 if (harq_out->nb_segments > 0) { 550 mbuf_pool_size = optimal_mempool_size(ops_pool_size * 551 harq_out->nb_segments); 552 mp = create_mbuf_pool(harq_out, ad->dev_id, socket_id, 553 mbuf_pool_size, 554 "harq_out"); 555 TEST_ASSERT_NOT_NULL(mp, 556 "ERROR Failed to create %uB harq output pktmbuf pool for dev %u on socket %u.", 557 mbuf_pool_size, 558 ad->dev_id, 559 socket_id); 560 ad->harq_out_mbuf_pool = mp; 561 } 562 563 return TEST_SUCCESS; 564 } 565 566 static int 567 add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, 568 struct test_bbdev_vector *vector) 569 { 570 int ret; 571 unsigned int queue_id; 572 struct rte_bbdev_queue_conf qconf; 573 struct active_device *ad = &active_devs[nb_active_devs]; 574 unsigned int nb_queues; 575 enum rte_bbdev_op_type op_type = vector->op_type; 576 577 /* Configure fpga lte fec with PF & VF values 578 * if '-i' flag is set and using fpga device 579 */ 580 #ifdef RTE_BASEBAND_FPGA_LTE_FEC 581 if ((get_init_device() == true) && 582 (!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) { 583 struct rte_fpga_lte_fec_conf conf; 584 unsigned int i; 585 586 printf("Configure FPGA LTE FEC Driver %s with default values\n", 587 info->drv.driver_name); 588 589 /* clear default configuration before initialization */ 590 memset(&conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 591 592 /* Set PF mode : 593 * true if PF is used for data plane 594 * false for VFs 595 */ 596 conf.pf_mode_en = true; 597 598 for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) { 599 /* Number of UL queues per VF (fpga supports 8 VFs) */ 600 conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE; 601 /* Number of DL queues per VF (fpga supports 8 VFs) */ 602 conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE; 603 } 604 605 /* UL bandwidth. Needed for schedule algorithm */ 606 conf.ul_bandwidth = UL_4G_BANDWIDTH; 607 /* DL bandwidth */ 608 conf.dl_bandwidth = DL_4G_BANDWIDTH; 609 610 /* UL & DL load Balance Factor to 64 */ 611 conf.ul_load_balance = UL_4G_LOAD_BALANCE; 612 conf.dl_load_balance = DL_4G_LOAD_BALANCE; 613 614 /**< FLR timeout value */ 615 conf.flr_time_out = FLR_4G_TIMEOUT; 616 617 /* setup FPGA PF with configuration information */ 618 ret = rte_fpga_lte_fec_configure(info->dev_name, &conf); 619 TEST_ASSERT_SUCCESS(ret, 620 "Failed to configure 4G FPGA PF for bbdev %s", 621 info->dev_name); 622 } 623 #endif 624 #ifdef RTE_BASEBAND_FPGA_5GNR_FEC 625 if ((get_init_device() == true) && 626 (!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) { 627 struct rte_fpga_5gnr_fec_conf conf; 628 unsigned int i; 629 630 printf("Configure FPGA 5GNR FEC Driver %s with default values\n", 631 info->drv.driver_name); 632 633 /* clear default configuration before initialization */ 634 memset(&conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 635 636 /* Set PF mode : 637 * true if PF is used for data plane 638 * false for VFs 639 */ 640 conf.pf_mode_en = true; 641 642 for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) { 643 /* Number of UL queues per VF (fpga supports 8 VFs) */ 644 conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE; 645 /* Number of DL queues per VF (fpga supports 8 VFs) */ 646 conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE; 647 } 648 649 /* UL bandwidth. Needed for schedule algorithm */ 650 conf.ul_bandwidth = UL_5G_BANDWIDTH; 651 /* DL bandwidth */ 652 conf.dl_bandwidth = DL_5G_BANDWIDTH; 653 654 /* UL & DL load Balance Factor to 64 */ 655 conf.ul_load_balance = UL_5G_LOAD_BALANCE; 656 conf.dl_load_balance = DL_5G_LOAD_BALANCE; 657 658 /**< FLR timeout value */ 659 conf.flr_time_out = FLR_5G_TIMEOUT; 660 661 /* setup FPGA PF with configuration information */ 662 ret = rte_fpga_5gnr_fec_configure(info->dev_name, &conf); 663 TEST_ASSERT_SUCCESS(ret, 664 "Failed to configure 5G FPGA PF for bbdev %s", 665 info->dev_name); 666 } 667 #endif 668 #ifdef RTE_BASEBAND_ACC100 669 if ((get_init_device() == true) && 670 (!strcmp(info->drv.driver_name, ACC100PF_DRIVER_NAME))) { 671 struct rte_acc100_conf conf; 672 unsigned int i; 673 674 printf("Configure ACC100 FEC Driver %s with default values\n", 675 info->drv.driver_name); 676 677 /* clear default configuration before initialization */ 678 memset(&conf, 0, sizeof(struct rte_acc100_conf)); 679 680 /* Always set in PF mode for built-in configuration */ 681 conf.pf_mode_en = true; 682 for (i = 0; i < RTE_ACC100_NUM_VFS; ++i) { 683 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 684 conf.arb_dl_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 685 conf.arb_dl_4g[i].round_robin_weight = ACC100_QMGR_RR; 686 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 687 conf.arb_ul_4g[i].gbr_threshold1 = ACC100_QOS_GBR; 688 conf.arb_ul_4g[i].round_robin_weight = ACC100_QMGR_RR; 689 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 690 conf.arb_dl_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 691 conf.arb_dl_5g[i].round_robin_weight = ACC100_QMGR_RR; 692 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 693 conf.arb_ul_5g[i].gbr_threshold1 = ACC100_QOS_GBR; 694 conf.arb_ul_5g[i].round_robin_weight = ACC100_QMGR_RR; 695 } 696 697 conf.input_pos_llr_1_bit = true; 698 conf.output_pos_llr_1_bit = true; 699 conf.num_vf_bundles = 1; /**< Number of VF bundles to setup */ 700 701 conf.q_ul_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 702 conf.q_ul_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 703 conf.q_ul_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 704 conf.q_ul_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 705 conf.q_dl_4g.num_qgroups = ACC100_QMGR_NUM_QGS; 706 conf.q_dl_4g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 707 conf.q_dl_4g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 708 conf.q_dl_4g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 709 conf.q_ul_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 710 conf.q_ul_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 711 conf.q_ul_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 712 conf.q_ul_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 713 conf.q_dl_5g.num_qgroups = ACC100_QMGR_NUM_QGS; 714 conf.q_dl_5g.first_qgroup_index = ACC100_QMGR_INVALID_IDX; 715 conf.q_dl_5g.num_aqs_per_groups = ACC100_QMGR_NUM_AQS; 716 conf.q_dl_5g.aq_depth_log2 = ACC100_QMGR_AQ_DEPTH; 717 718 /* setup PF with configuration information */ 719 ret = rte_acc100_configure(info->dev_name, &conf); 720 TEST_ASSERT_SUCCESS(ret, 721 "Failed to configure ACC100 PF for bbdev %s", 722 info->dev_name); 723 } 724 #endif 725 /* Let's refresh this now this is configured */ 726 rte_bbdev_info_get(dev_id, info); 727 nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); 728 nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES); 729 730 /* setup device */ 731 ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); 732 if (ret < 0) { 733 printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", 734 dev_id, nb_queues, info->socket_id, ret); 735 return TEST_FAILED; 736 } 737 738 /* configure interrupts if needed */ 739 if (intr_enabled) { 740 ret = rte_bbdev_intr_enable(dev_id); 741 if (ret < 0) { 742 printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, 743 ret); 744 return TEST_FAILED; 745 } 746 } 747 748 /* setup device queues */ 749 qconf.socket = info->socket_id; 750 qconf.queue_size = info->drv.default_queue_conf.queue_size; 751 qconf.priority = 0; 752 qconf.deferred_start = 0; 753 qconf.op_type = op_type; 754 755 for (queue_id = 0; queue_id < nb_queues; ++queue_id) { 756 ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); 757 if (ret != 0) { 758 printf( 759 "Allocated all queues (id=%u) at prio%u on dev%u\n", 760 queue_id, qconf.priority, dev_id); 761 qconf.priority++; 762 ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, 763 &qconf); 764 } 765 if (ret != 0) { 766 printf("All queues on dev %u allocated: %u\n", 767 dev_id, queue_id); 768 break; 769 } 770 ad->queue_ids[queue_id] = queue_id; 771 } 772 TEST_ASSERT(queue_id != 0, 773 "ERROR Failed to configure any queues on dev %u", 774 dev_id); 775 ad->nb_queues = queue_id; 776 777 set_avail_op(ad, op_type); 778 779 return TEST_SUCCESS; 780 } 781 782 static int 783 add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, 784 struct test_bbdev_vector *vector) 785 { 786 int ret; 787 788 active_devs[nb_active_devs].driver_name = info->drv.driver_name; 789 active_devs[nb_active_devs].dev_id = dev_id; 790 791 ret = add_bbdev_dev(dev_id, info, vector); 792 if (ret == TEST_SUCCESS) 793 ++nb_active_devs; 794 return ret; 795 } 796 797 static uint8_t 798 populate_active_devices(void) 799 { 800 int ret; 801 uint8_t dev_id; 802 uint8_t nb_devs_added = 0; 803 struct rte_bbdev_info info; 804 805 RTE_BBDEV_FOREACH(dev_id) { 806 rte_bbdev_info_get(dev_id, &info); 807 808 if (check_dev_cap(&info)) { 809 printf( 810 "Device %d (%s) does not support specified capabilities\n", 811 dev_id, info.dev_name); 812 continue; 813 } 814 815 ret = add_active_device(dev_id, &info, &test_vector); 816 if (ret != 0) { 817 printf("Adding active bbdev %s skipped\n", 818 info.dev_name); 819 continue; 820 } 821 nb_devs_added++; 822 } 823 824 return nb_devs_added; 825 } 826 827 static int 828 read_test_vector(void) 829 { 830 int ret; 831 832 memset(&test_vector, 0, sizeof(test_vector)); 833 printf("Test vector file = %s\n", get_vector_filename()); 834 ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); 835 TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", 836 get_vector_filename()); 837 838 return TEST_SUCCESS; 839 } 840 841 static int 842 testsuite_setup(void) 843 { 844 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 845 846 if (populate_active_devices() == 0) { 847 printf("No suitable devices found!\n"); 848 return TEST_SKIPPED; 849 } 850 851 return TEST_SUCCESS; 852 } 853 854 static int 855 interrupt_testsuite_setup(void) 856 { 857 TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); 858 859 /* Enable interrupts */ 860 intr_enabled = true; 861 862 /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ 863 if (populate_active_devices() == 0 || 864 test_vector.op_type == RTE_BBDEV_OP_NONE) { 865 intr_enabled = false; 866 printf("No suitable devices found!\n"); 867 return TEST_SKIPPED; 868 } 869 870 return TEST_SUCCESS; 871 } 872 873 static void 874 testsuite_teardown(void) 875 { 876 uint8_t dev_id; 877 878 /* Unconfigure devices */ 879 RTE_BBDEV_FOREACH(dev_id) 880 rte_bbdev_close(dev_id); 881 882 /* Clear active devices structs. */ 883 memset(active_devs, 0, sizeof(active_devs)); 884 nb_active_devs = 0; 885 886 /* Disable interrupts */ 887 intr_enabled = false; 888 } 889 890 static int 891 ut_setup(void) 892 { 893 uint8_t i, dev_id; 894 895 for (i = 0; i < nb_active_devs; i++) { 896 dev_id = active_devs[i].dev_id; 897 /* reset bbdev stats */ 898 TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), 899 "Failed to reset stats of bbdev %u", dev_id); 900 /* start the device */ 901 TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), 902 "Failed to start bbdev %u", dev_id); 903 } 904 905 return TEST_SUCCESS; 906 } 907 908 static void 909 ut_teardown(void) 910 { 911 uint8_t i, dev_id; 912 struct rte_bbdev_stats stats; 913 914 for (i = 0; i < nb_active_devs; i++) { 915 dev_id = active_devs[i].dev_id; 916 /* read stats and print */ 917 rte_bbdev_stats_get(dev_id, &stats); 918 /* Stop the device */ 919 rte_bbdev_stop(dev_id); 920 } 921 } 922 923 static int 924 init_op_data_objs(struct rte_bbdev_op_data *bufs, 925 struct op_data_entries *ref_entries, 926 struct rte_mempool *mbuf_pool, const uint16_t n, 927 enum op_data_type op_type, uint16_t min_alignment) 928 { 929 int ret; 930 unsigned int i, j; 931 bool large_input = false; 932 933 for (i = 0; i < n; ++i) { 934 char *data; 935 struct op_data_buf *seg = &ref_entries->segments[0]; 936 struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); 937 TEST_ASSERT_NOT_NULL(m_head, 938 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 939 op_type, n * ref_entries->nb_segments, 940 mbuf_pool->size); 941 942 if (seg->length > RTE_BBDEV_LDPC_E_MAX_MBUF) { 943 /* 944 * Special case when DPDK mbuf cannot handle 945 * the required input size 946 */ 947 printf("Warning: Larger input size than DPDK mbuf %d\n", 948 seg->length); 949 large_input = true; 950 } 951 bufs[i].data = m_head; 952 bufs[i].offset = 0; 953 bufs[i].length = 0; 954 955 if ((op_type == DATA_INPUT) || (op_type == DATA_HARQ_INPUT)) { 956 if ((op_type == DATA_INPUT) && large_input) { 957 /* Allocate a fake overused mbuf */ 958 data = rte_malloc(NULL, seg->length, 0); 959 memcpy(data, seg->addr, seg->length); 960 m_head->buf_addr = data; 961 m_head->buf_iova = rte_malloc_virt2iova(data); 962 m_head->data_off = 0; 963 m_head->data_len = seg->length; 964 } else { 965 data = rte_pktmbuf_append(m_head, seg->length); 966 TEST_ASSERT_NOT_NULL(data, 967 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 968 seg->length, op_type); 969 970 TEST_ASSERT(data == RTE_PTR_ALIGN( 971 data, min_alignment), 972 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 973 data, min_alignment); 974 rte_memcpy(data, seg->addr, seg->length); 975 } 976 977 bufs[i].length += seg->length; 978 979 for (j = 1; j < ref_entries->nb_segments; ++j) { 980 struct rte_mbuf *m_tail = 981 rte_pktmbuf_alloc(mbuf_pool); 982 TEST_ASSERT_NOT_NULL(m_tail, 983 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 984 op_type, 985 n * ref_entries->nb_segments, 986 mbuf_pool->size); 987 seg += 1; 988 989 data = rte_pktmbuf_append(m_tail, seg->length); 990 TEST_ASSERT_NOT_NULL(data, 991 "Couldn't append %u bytes to mbuf from %d data type mbuf pool", 992 seg->length, op_type); 993 994 TEST_ASSERT(data == RTE_PTR_ALIGN(data, 995 min_alignment), 996 "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", 997 data, min_alignment); 998 rte_memcpy(data, seg->addr, seg->length); 999 bufs[i].length += seg->length; 1000 1001 ret = rte_pktmbuf_chain(m_head, m_tail); 1002 TEST_ASSERT_SUCCESS(ret, 1003 "Couldn't chain mbufs from %d data type mbuf pool", 1004 op_type); 1005 } 1006 } else { 1007 1008 /* allocate chained-mbuf for output buffer */ 1009 for (j = 1; j < ref_entries->nb_segments; ++j) { 1010 struct rte_mbuf *m_tail = 1011 rte_pktmbuf_alloc(mbuf_pool); 1012 TEST_ASSERT_NOT_NULL(m_tail, 1013 "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", 1014 op_type, 1015 n * ref_entries->nb_segments, 1016 mbuf_pool->size); 1017 1018 ret = rte_pktmbuf_chain(m_head, m_tail); 1019 TEST_ASSERT_SUCCESS(ret, 1020 "Couldn't chain mbufs from %d data type mbuf pool", 1021 op_type); 1022 } 1023 } 1024 } 1025 1026 return 0; 1027 } 1028 1029 static int 1030 allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, 1031 const int socket) 1032 { 1033 int i; 1034 1035 *buffers = rte_zmalloc_socket(NULL, len, 0, socket); 1036 if (*buffers == NULL) { 1037 printf("WARNING: Failed to allocate op_data on socket %d\n", 1038 socket); 1039 /* try to allocate memory on other detected sockets */ 1040 for (i = 0; i < socket; i++) { 1041 *buffers = rte_zmalloc_socket(NULL, len, 0, i); 1042 if (*buffers != NULL) 1043 break; 1044 } 1045 } 1046 1047 return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; 1048 } 1049 1050 static void 1051 limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, 1052 const uint16_t n, const int8_t max_llr_modulus) 1053 { 1054 uint16_t i, byte_idx; 1055 1056 for (i = 0; i < n; ++i) { 1057 struct rte_mbuf *m = input_ops[i].data; 1058 while (m != NULL) { 1059 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1060 input_ops[i].offset); 1061 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1062 ++byte_idx) 1063 llr[byte_idx] = round((double)max_llr_modulus * 1064 llr[byte_idx] / INT8_MAX); 1065 1066 m = m->next; 1067 } 1068 } 1069 } 1070 1071 /* 1072 * We may have to insert filler bits 1073 * when they are required by the HARQ assumption 1074 */ 1075 static void 1076 ldpc_add_filler(struct rte_bbdev_op_data *input_ops, 1077 const uint16_t n, struct test_op_params *op_params) 1078 { 1079 struct rte_bbdev_op_ldpc_dec dec = op_params->ref_dec_op->ldpc_dec; 1080 1081 if (input_ops == NULL) 1082 return; 1083 /* No need to add filler if not required by device */ 1084 if (!(ldpc_cap_flags & 1085 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS)) 1086 return; 1087 /* No need to add filler for loopback operation */ 1088 if (dec.op_flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 1089 return; 1090 1091 uint16_t i, j, parity_offset; 1092 for (i = 0; i < n; ++i) { 1093 struct rte_mbuf *m = input_ops[i].data; 1094 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1095 input_ops[i].offset); 1096 parity_offset = (dec.basegraph == 1 ? 20 : 8) 1097 * dec.z_c - dec.n_filler; 1098 uint16_t new_hin_size = input_ops[i].length + dec.n_filler; 1099 m->data_len = new_hin_size; 1100 input_ops[i].length = new_hin_size; 1101 for (j = new_hin_size - 1; j >= parity_offset + dec.n_filler; 1102 j--) 1103 llr[j] = llr[j - dec.n_filler]; 1104 uint16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1105 for (j = 0; j < dec.n_filler; j++) 1106 llr[parity_offset + j] = llr_max_pre_scaling; 1107 } 1108 } 1109 1110 static void 1111 ldpc_input_llr_scaling(struct rte_bbdev_op_data *input_ops, 1112 const uint16_t n, const int8_t llr_size, 1113 const int8_t llr_decimals) 1114 { 1115 if (input_ops == NULL) 1116 return; 1117 1118 uint16_t i, byte_idx; 1119 1120 int16_t llr_max, llr_min, llr_tmp; 1121 llr_max = (1 << (llr_size - 1)) - 1; 1122 llr_min = -llr_max; 1123 for (i = 0; i < n; ++i) { 1124 struct rte_mbuf *m = input_ops[i].data; 1125 while (m != NULL) { 1126 int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, 1127 input_ops[i].offset); 1128 for (byte_idx = 0; byte_idx < rte_pktmbuf_data_len(m); 1129 ++byte_idx) { 1130 1131 llr_tmp = llr[byte_idx]; 1132 if (llr_decimals == 4) 1133 llr_tmp *= 8; 1134 else if (llr_decimals == 2) 1135 llr_tmp *= 2; 1136 else if (llr_decimals == 0) 1137 llr_tmp /= 2; 1138 llr_tmp = RTE_MIN(llr_max, 1139 RTE_MAX(llr_min, llr_tmp)); 1140 llr[byte_idx] = (int8_t) llr_tmp; 1141 } 1142 1143 m = m->next; 1144 } 1145 } 1146 } 1147 1148 1149 1150 static int 1151 fill_queue_buffers(struct test_op_params *op_params, 1152 struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, 1153 struct rte_mempool *soft_out_mp, 1154 struct rte_mempool *harq_in_mp, struct rte_mempool *harq_out_mp, 1155 uint16_t queue_id, 1156 const struct rte_bbdev_op_cap *capabilities, 1157 uint16_t min_alignment, const int socket_id) 1158 { 1159 int ret; 1160 enum op_data_type type; 1161 const uint16_t n = op_params->num_to_process; 1162 1163 struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { 1164 in_mp, 1165 soft_out_mp, 1166 hard_out_mp, 1167 harq_in_mp, 1168 harq_out_mp, 1169 }; 1170 1171 struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { 1172 &op_params->q_bufs[socket_id][queue_id].inputs, 1173 &op_params->q_bufs[socket_id][queue_id].soft_outputs, 1174 &op_params->q_bufs[socket_id][queue_id].hard_outputs, 1175 &op_params->q_bufs[socket_id][queue_id].harq_inputs, 1176 &op_params->q_bufs[socket_id][queue_id].harq_outputs, 1177 }; 1178 1179 for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { 1180 struct op_data_entries *ref_entries = 1181 &test_vector.entries[type]; 1182 if (ref_entries->nb_segments == 0) 1183 continue; 1184 1185 ret = allocate_buffers_on_socket(queue_ops[type], 1186 n * sizeof(struct rte_bbdev_op_data), 1187 socket_id); 1188 TEST_ASSERT_SUCCESS(ret, 1189 "Couldn't allocate memory for rte_bbdev_op_data structs"); 1190 1191 ret = init_op_data_objs(*queue_ops[type], ref_entries, 1192 mbuf_pools[type], n, type, min_alignment); 1193 TEST_ASSERT_SUCCESS(ret, 1194 "Couldn't init rte_bbdev_op_data structs"); 1195 } 1196 1197 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 1198 limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, 1199 capabilities->cap.turbo_dec.max_llr_modulus); 1200 1201 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 1202 bool loopback = op_params->ref_dec_op->ldpc_dec.op_flags & 1203 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 1204 bool llr_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1205 RTE_BBDEV_LDPC_LLR_COMPRESSION; 1206 bool harq_comp = op_params->ref_dec_op->ldpc_dec.op_flags & 1207 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 1208 ldpc_llr_decimals = capabilities->cap.ldpc_dec.llr_decimals; 1209 ldpc_llr_size = capabilities->cap.ldpc_dec.llr_size; 1210 ldpc_cap_flags = capabilities->cap.ldpc_dec.capability_flags; 1211 if (!loopback && !llr_comp) 1212 ldpc_input_llr_scaling(*queue_ops[DATA_INPUT], n, 1213 ldpc_llr_size, ldpc_llr_decimals); 1214 if (!loopback && !harq_comp) 1215 ldpc_input_llr_scaling(*queue_ops[DATA_HARQ_INPUT], n, 1216 ldpc_llr_size, ldpc_llr_decimals); 1217 if (!loopback) 1218 ldpc_add_filler(*queue_ops[DATA_HARQ_INPUT], n, 1219 op_params); 1220 } 1221 1222 return 0; 1223 } 1224 1225 static void 1226 free_buffers(struct active_device *ad, struct test_op_params *op_params) 1227 { 1228 unsigned int i, j; 1229 1230 rte_mempool_free(ad->ops_mempool); 1231 rte_mempool_free(ad->in_mbuf_pool); 1232 rte_mempool_free(ad->hard_out_mbuf_pool); 1233 rte_mempool_free(ad->soft_out_mbuf_pool); 1234 rte_mempool_free(ad->harq_in_mbuf_pool); 1235 rte_mempool_free(ad->harq_out_mbuf_pool); 1236 1237 for (i = 0; i < rte_lcore_count(); ++i) { 1238 for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { 1239 rte_free(op_params->q_bufs[j][i].inputs); 1240 rte_free(op_params->q_bufs[j][i].hard_outputs); 1241 rte_free(op_params->q_bufs[j][i].soft_outputs); 1242 rte_free(op_params->q_bufs[j][i].harq_inputs); 1243 rte_free(op_params->q_bufs[j][i].harq_outputs); 1244 } 1245 } 1246 } 1247 1248 static void 1249 copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1250 unsigned int start_idx, 1251 struct rte_bbdev_op_data *inputs, 1252 struct rte_bbdev_op_data *hard_outputs, 1253 struct rte_bbdev_op_data *soft_outputs, 1254 struct rte_bbdev_dec_op *ref_op) 1255 { 1256 unsigned int i; 1257 struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; 1258 1259 for (i = 0; i < n; ++i) { 1260 if (turbo_dec->code_block_mode == 0) { 1261 ops[i]->turbo_dec.tb_params.ea = 1262 turbo_dec->tb_params.ea; 1263 ops[i]->turbo_dec.tb_params.eb = 1264 turbo_dec->tb_params.eb; 1265 ops[i]->turbo_dec.tb_params.k_pos = 1266 turbo_dec->tb_params.k_pos; 1267 ops[i]->turbo_dec.tb_params.k_neg = 1268 turbo_dec->tb_params.k_neg; 1269 ops[i]->turbo_dec.tb_params.c = 1270 turbo_dec->tb_params.c; 1271 ops[i]->turbo_dec.tb_params.c_neg = 1272 turbo_dec->tb_params.c_neg; 1273 ops[i]->turbo_dec.tb_params.cab = 1274 turbo_dec->tb_params.cab; 1275 ops[i]->turbo_dec.tb_params.r = 1276 turbo_dec->tb_params.r; 1277 } else { 1278 ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; 1279 ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; 1280 } 1281 1282 ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; 1283 ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; 1284 ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; 1285 ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; 1286 ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; 1287 ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; 1288 ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; 1289 1290 ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; 1291 ops[i]->turbo_dec.input = inputs[start_idx + i]; 1292 if (soft_outputs != NULL) 1293 ops[i]->turbo_dec.soft_output = 1294 soft_outputs[start_idx + i]; 1295 } 1296 } 1297 1298 static void 1299 copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1300 unsigned int start_idx, 1301 struct rte_bbdev_op_data *inputs, 1302 struct rte_bbdev_op_data *outputs, 1303 struct rte_bbdev_enc_op *ref_op) 1304 { 1305 unsigned int i; 1306 struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; 1307 for (i = 0; i < n; ++i) { 1308 if (turbo_enc->code_block_mode == 0) { 1309 ops[i]->turbo_enc.tb_params.ea = 1310 turbo_enc->tb_params.ea; 1311 ops[i]->turbo_enc.tb_params.eb = 1312 turbo_enc->tb_params.eb; 1313 ops[i]->turbo_enc.tb_params.k_pos = 1314 turbo_enc->tb_params.k_pos; 1315 ops[i]->turbo_enc.tb_params.k_neg = 1316 turbo_enc->tb_params.k_neg; 1317 ops[i]->turbo_enc.tb_params.c = 1318 turbo_enc->tb_params.c; 1319 ops[i]->turbo_enc.tb_params.c_neg = 1320 turbo_enc->tb_params.c_neg; 1321 ops[i]->turbo_enc.tb_params.cab = 1322 turbo_enc->tb_params.cab; 1323 ops[i]->turbo_enc.tb_params.ncb_pos = 1324 turbo_enc->tb_params.ncb_pos; 1325 ops[i]->turbo_enc.tb_params.ncb_neg = 1326 turbo_enc->tb_params.ncb_neg; 1327 ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; 1328 } else { 1329 ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; 1330 ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; 1331 ops[i]->turbo_enc.cb_params.ncb = 1332 turbo_enc->cb_params.ncb; 1333 } 1334 ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; 1335 ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; 1336 ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; 1337 1338 ops[i]->turbo_enc.output = outputs[start_idx + i]; 1339 ops[i]->turbo_enc.input = inputs[start_idx + i]; 1340 } 1341 } 1342 1343 1344 /* Returns a random number drawn from a normal distribution 1345 * with mean of 0 and variance of 1 1346 * Marsaglia algorithm 1347 */ 1348 static double 1349 randn(int n) 1350 { 1351 double S, Z, U1, U2, u, v, fac; 1352 1353 do { 1354 U1 = (double)rand() / RAND_MAX; 1355 U2 = (double)rand() / RAND_MAX; 1356 u = 2. * U1 - 1.; 1357 v = 2. * U2 - 1.; 1358 S = u * u + v * v; 1359 } while (S >= 1 || S == 0); 1360 fac = sqrt(-2. * log(S) / S); 1361 Z = (n % 2) ? u * fac : v * fac; 1362 return Z; 1363 } 1364 1365 static inline double 1366 maxstar(double A, double B) 1367 { 1368 if (fabs(A - B) > 5) 1369 return RTE_MAX(A, B); 1370 else 1371 return RTE_MAX(A, B) + log1p(exp(-fabs(A - B))); 1372 } 1373 1374 /* 1375 * Generate Qm LLRS for Qm==8 1376 * Modulation, AWGN and LLR estimation from max log development 1377 */ 1378 static void 1379 gen_qm8_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1380 { 1381 int qm = 8; 1382 int qam = 256; 1383 int m, k; 1384 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1385 /* 5.1.4 of TS38.211 */ 1386 const double symbols_I[256] = { 1387 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 5, 1388 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1, 3, 3, 1, 1, 11, 1389 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 15, 13, 13, 1390 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 15, 1391 15, 13, 13, 15, 15, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1392 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 3, 3, 1, 1393 1, 3, 3, 1, 1, 11, 11, 9, 9, 11, 11, 9, 9, 13, 13, 1394 15, 15, 13, 13, 15, 15, 11, 11, 9, 9, 11, 11, 9, 9, 1395 13, 13, 15, 15, 13, 13, 15, 15, -5, -5, -7, -7, -5, 1396 -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, 1397 -7, -7, -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, 1398 -1, -1, -11, -11, -9, -9, -11, -11, -9, -9, -13, 1399 -13, -15, -15, -13, -13, -15, -15, -11, -11, -9, 1400 -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1401 -13, -15, -15, -5, -5, -7, -7, -5, -5, -7, -7, -3, 1402 -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, -5, -5, 1403 -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, -11, -11, 1404 -9, -9, -11, -11, -9, -9, -13, -13, -15, -15, -13, 1405 -13, -15, -15, -11, -11, -9, -9, -11, -11, -9, -9, 1406 -13, -13, -15, -15, -13, -13, -15, -15}; 1407 const double symbols_Q[256] = { 1408 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1409 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 13, 1410 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1411 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 1412 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, -5, 1413 -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, 1414 -15, -13, -15, -11, -9, -11, -9, -13, -15, -13, 1415 -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, -5, 1416 -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1417 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15, 5, 1418 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 11, 1419 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 13, 15, 1420 13, 15, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 1421 3, 1, 11, 9, 11, 9, 13, 15, 13, 15, 11, 9, 11, 9, 1422 13, 15, 13, 15, -5, -7, -5, -7, -3, -1, -3, -1, 1423 -5, -7, -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, 1424 -13, -15, -13, -15, -11, -9, -11, -9, -13, -15, 1425 -13, -15, -5, -7, -5, -7, -3, -1, -3, -1, -5, -7, 1426 -5, -7, -3, -1, -3, -1, -11, -9, -11, -9, -13, -15, 1427 -13, -15, -11, -9, -11, -9, -13, -15, -13, -15}; 1428 /* Average constellation point energy */ 1429 N0 *= 170.0; 1430 for (k = 0; k < qm; k++) 1431 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1432 /* 5.1.4 of TS38.211 */ 1433 I = (1 - 2 * b[0]) * (8 - (1 - 2 * b[2]) * 1434 (4 - (1 - 2 * b[4]) * (2 - (1 - 2 * b[6])))); 1435 Q = (1 - 2 * b[1]) * (8 - (1 - 2 * b[3]) * 1436 (4 - (1 - 2 * b[5]) * (2 - (1 - 2 * b[7])))); 1437 /* AWGN channel */ 1438 I += sqrt(N0 / 2) * randn(0); 1439 Q += sqrt(N0 / 2) * randn(1); 1440 /* 1441 * Calculate the log of the probability that each of 1442 * the constellation points was transmitted 1443 */ 1444 for (m = 0; m < qam; m++) 1445 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1446 + pow(Q - symbols_Q[m], 2.0)) / N0; 1447 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1448 for (k = 0; k < qm; k++) { 1449 p0 = -999999; 1450 p1 = -999999; 1451 /* For each constellation point */ 1452 for (m = 0; m < qam; m++) { 1453 if ((m >> (qm - k - 1)) & 1) 1454 p1 = maxstar(p1, log_syml_prob[m]); 1455 else 1456 p0 = maxstar(p0, log_syml_prob[m]); 1457 } 1458 /* Calculate the LLR */ 1459 llr_ = p0 - p1; 1460 llr_ *= (1 << ldpc_llr_decimals); 1461 llr_ = round(llr_); 1462 if (llr_ > llr_max) 1463 llr_ = llr_max; 1464 if (llr_ < -llr_max) 1465 llr_ = -llr_max; 1466 llrs[qm * i + k] = (int8_t) llr_; 1467 } 1468 } 1469 1470 1471 /* 1472 * Generate Qm LLRS for Qm==6 1473 * Modulation, AWGN and LLR estimation from max log development 1474 */ 1475 static void 1476 gen_qm6_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1477 { 1478 int qm = 6; 1479 int qam = 64; 1480 int m, k; 1481 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1482 /* 5.1.4 of TS38.211 */ 1483 const double symbols_I[64] = { 1484 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1485 3, 3, 1, 1, 3, 3, 1, 1, 5, 5, 7, 7, 5, 5, 7, 7, 1486 -3, -3, -1, -1, -3, -3, -1, -1, -5, -5, -7, -7, 1487 -5, -5, -7, -7, -3, -3, -1, -1, -3, -3, -1, -1, 1488 -5, -5, -7, -7, -5, -5, -7, -7}; 1489 const double symbols_Q[64] = { 1490 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 5, 7, 5, 7, 1491 -3, -1, -3, -1, -5, -7, -5, -7, -3, -1, -3, -1, 1492 -5, -7, -5, -7, 3, 1, 3, 1, 5, 7, 5, 7, 3, 1, 3, 1, 1493 5, 7, 5, 7, -3, -1, -3, -1, -5, -7, -5, -7, 1494 -3, -1, -3, -1, -5, -7, -5, -7}; 1495 /* Average constellation point energy */ 1496 N0 *= 42.0; 1497 for (k = 0; k < qm; k++) 1498 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1499 /* 5.1.4 of TS38.211 */ 1500 I = (1 - 2 * b[0])*(4 - (1 - 2 * b[2]) * (2 - (1 - 2 * b[4]))); 1501 Q = (1 - 2 * b[1])*(4 - (1 - 2 * b[3]) * (2 - (1 - 2 * b[5]))); 1502 /* AWGN channel */ 1503 I += sqrt(N0 / 2) * randn(0); 1504 Q += sqrt(N0 / 2) * randn(1); 1505 /* 1506 * Calculate the log of the probability that each of 1507 * the constellation points was transmitted 1508 */ 1509 for (m = 0; m < qam; m++) 1510 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1511 + pow(Q - symbols_Q[m], 2.0)) / N0; 1512 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1513 for (k = 0; k < qm; k++) { 1514 p0 = -999999; 1515 p1 = -999999; 1516 /* For each constellation point */ 1517 for (m = 0; m < qam; m++) { 1518 if ((m >> (qm - k - 1)) & 1) 1519 p1 = maxstar(p1, log_syml_prob[m]); 1520 else 1521 p0 = maxstar(p0, log_syml_prob[m]); 1522 } 1523 /* Calculate the LLR */ 1524 llr_ = p0 - p1; 1525 llr_ *= (1 << ldpc_llr_decimals); 1526 llr_ = round(llr_); 1527 if (llr_ > llr_max) 1528 llr_ = llr_max; 1529 if (llr_ < -llr_max) 1530 llr_ = -llr_max; 1531 llrs[qm * i + k] = (int8_t) llr_; 1532 } 1533 } 1534 1535 /* 1536 * Generate Qm LLRS for Qm==4 1537 * Modulation, AWGN and LLR estimation from max log development 1538 */ 1539 static void 1540 gen_qm4_llr(int8_t *llrs, uint32_t i, double N0, double llr_max) 1541 { 1542 int qm = 4; 1543 int qam = 16; 1544 int m, k; 1545 double I, Q, p0, p1, llr_, b[qm], log_syml_prob[qam]; 1546 /* 5.1.4 of TS38.211 */ 1547 const double symbols_I[16] = {1, 1, 3, 3, 1, 1, 3, 3, 1548 -1, -1, -3, -3, -1, -1, -3, -3}; 1549 const double symbols_Q[16] = {1, 3, 1, 3, -1, -3, -1, -3, 1550 1, 3, 1, 3, -1, -3, -1, -3}; 1551 /* Average constellation point energy */ 1552 N0 *= 10.0; 1553 for (k = 0; k < qm; k++) 1554 b[k] = llrs[qm * i + k] < 0 ? 1.0 : 0.0; 1555 /* 5.1.4 of TS38.211 */ 1556 I = (1 - 2 * b[0]) * (2 - (1 - 2 * b[2])); 1557 Q = (1 - 2 * b[1]) * (2 - (1 - 2 * b[3])); 1558 /* AWGN channel */ 1559 I += sqrt(N0 / 2) * randn(0); 1560 Q += sqrt(N0 / 2) * randn(1); 1561 /* 1562 * Calculate the log of the probability that each of 1563 * the constellation points was transmitted 1564 */ 1565 for (m = 0; m < qam; m++) 1566 log_syml_prob[m] = -(pow(I - symbols_I[m], 2.0) 1567 + pow(Q - symbols_Q[m], 2.0)) / N0; 1568 /* Calculate an LLR for each of the k_64QAM bits in the set */ 1569 for (k = 0; k < qm; k++) { 1570 p0 = -999999; 1571 p1 = -999999; 1572 /* For each constellation point */ 1573 for (m = 0; m < qam; m++) { 1574 if ((m >> (qm - k - 1)) & 1) 1575 p1 = maxstar(p1, log_syml_prob[m]); 1576 else 1577 p0 = maxstar(p0, log_syml_prob[m]); 1578 } 1579 /* Calculate the LLR */ 1580 llr_ = p0 - p1; 1581 llr_ *= (1 << ldpc_llr_decimals); 1582 llr_ = round(llr_); 1583 if (llr_ > llr_max) 1584 llr_ = llr_max; 1585 if (llr_ < -llr_max) 1586 llr_ = -llr_max; 1587 llrs[qm * i + k] = (int8_t) llr_; 1588 } 1589 } 1590 1591 static void 1592 gen_qm2_llr(int8_t *llrs, uint32_t j, double N0, double llr_max) 1593 { 1594 double b, b1, n; 1595 double coeff = 2.0 * sqrt(N0); 1596 1597 /* Ignore in vectors rare quasi null LLRs not to be saturated */ 1598 if (llrs[j] < 8 && llrs[j] > -8) 1599 return; 1600 1601 /* Note don't change sign here */ 1602 n = randn(j % 2); 1603 b1 = ((llrs[j] > 0 ? 2.0 : -2.0) 1604 + coeff * n) / N0; 1605 b = b1 * (1 << ldpc_llr_decimals); 1606 b = round(b); 1607 if (b > llr_max) 1608 b = llr_max; 1609 if (b < -llr_max) 1610 b = -llr_max; 1611 llrs[j] = (int8_t) b; 1612 } 1613 1614 /* Generate LLR for a given SNR */ 1615 static void 1616 generate_llr_input(uint16_t n, struct rte_bbdev_op_data *inputs, 1617 struct rte_bbdev_dec_op *ref_op) 1618 { 1619 struct rte_mbuf *m; 1620 uint16_t qm; 1621 uint32_t i, j, e, range; 1622 double N0, llr_max; 1623 1624 e = ref_op->ldpc_dec.cb_params.e; 1625 qm = ref_op->ldpc_dec.q_m; 1626 llr_max = (1 << (ldpc_llr_size - 1)) - 1; 1627 range = e / qm; 1628 N0 = 1.0 / pow(10.0, get_snr() / 10.0); 1629 1630 for (i = 0; i < n; ++i) { 1631 m = inputs[i].data; 1632 int8_t *llrs = rte_pktmbuf_mtod_offset(m, int8_t *, 0); 1633 if (qm == 8) { 1634 for (j = 0; j < range; ++j) 1635 gen_qm8_llr(llrs, j, N0, llr_max); 1636 } else if (qm == 6) { 1637 for (j = 0; j < range; ++j) 1638 gen_qm6_llr(llrs, j, N0, llr_max); 1639 } else if (qm == 4) { 1640 for (j = 0; j < range; ++j) 1641 gen_qm4_llr(llrs, j, N0, llr_max); 1642 } else { 1643 for (j = 0; j < e; ++j) 1644 gen_qm2_llr(llrs, j, N0, llr_max); 1645 } 1646 } 1647 } 1648 1649 static void 1650 copy_reference_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, 1651 unsigned int start_idx, 1652 struct rte_bbdev_op_data *inputs, 1653 struct rte_bbdev_op_data *hard_outputs, 1654 struct rte_bbdev_op_data *soft_outputs, 1655 struct rte_bbdev_op_data *harq_inputs, 1656 struct rte_bbdev_op_data *harq_outputs, 1657 struct rte_bbdev_dec_op *ref_op) 1658 { 1659 unsigned int i; 1660 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &ref_op->ldpc_dec; 1661 1662 for (i = 0; i < n; ++i) { 1663 if (ldpc_dec->code_block_mode == 0) { 1664 ops[i]->ldpc_dec.tb_params.ea = 1665 ldpc_dec->tb_params.ea; 1666 ops[i]->ldpc_dec.tb_params.eb = 1667 ldpc_dec->tb_params.eb; 1668 ops[i]->ldpc_dec.tb_params.c = 1669 ldpc_dec->tb_params.c; 1670 ops[i]->ldpc_dec.tb_params.cab = 1671 ldpc_dec->tb_params.cab; 1672 ops[i]->ldpc_dec.tb_params.r = 1673 ldpc_dec->tb_params.r; 1674 } else { 1675 ops[i]->ldpc_dec.cb_params.e = ldpc_dec->cb_params.e; 1676 } 1677 1678 ops[i]->ldpc_dec.basegraph = ldpc_dec->basegraph; 1679 ops[i]->ldpc_dec.z_c = ldpc_dec->z_c; 1680 ops[i]->ldpc_dec.q_m = ldpc_dec->q_m; 1681 ops[i]->ldpc_dec.n_filler = ldpc_dec->n_filler; 1682 ops[i]->ldpc_dec.n_cb = ldpc_dec->n_cb; 1683 ops[i]->ldpc_dec.iter_max = ldpc_dec->iter_max; 1684 ops[i]->ldpc_dec.rv_index = ldpc_dec->rv_index; 1685 ops[i]->ldpc_dec.op_flags = ldpc_dec->op_flags; 1686 ops[i]->ldpc_dec.code_block_mode = ldpc_dec->code_block_mode; 1687 1688 if (hard_outputs != NULL) 1689 ops[i]->ldpc_dec.hard_output = 1690 hard_outputs[start_idx + i]; 1691 if (inputs != NULL) 1692 ops[i]->ldpc_dec.input = 1693 inputs[start_idx + i]; 1694 if (soft_outputs != NULL) 1695 ops[i]->ldpc_dec.soft_output = 1696 soft_outputs[start_idx + i]; 1697 if (harq_inputs != NULL) 1698 ops[i]->ldpc_dec.harq_combined_input = 1699 harq_inputs[start_idx + i]; 1700 if (harq_outputs != NULL) 1701 ops[i]->ldpc_dec.harq_combined_output = 1702 harq_outputs[start_idx + i]; 1703 } 1704 } 1705 1706 1707 static void 1708 copy_reference_ldpc_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, 1709 unsigned int start_idx, 1710 struct rte_bbdev_op_data *inputs, 1711 struct rte_bbdev_op_data *outputs, 1712 struct rte_bbdev_enc_op *ref_op) 1713 { 1714 unsigned int i; 1715 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &ref_op->ldpc_enc; 1716 for (i = 0; i < n; ++i) { 1717 if (ldpc_enc->code_block_mode == 0) { 1718 ops[i]->ldpc_enc.tb_params.ea = ldpc_enc->tb_params.ea; 1719 ops[i]->ldpc_enc.tb_params.eb = ldpc_enc->tb_params.eb; 1720 ops[i]->ldpc_enc.tb_params.cab = 1721 ldpc_enc->tb_params.cab; 1722 ops[i]->ldpc_enc.tb_params.c = ldpc_enc->tb_params.c; 1723 ops[i]->ldpc_enc.tb_params.r = ldpc_enc->tb_params.r; 1724 } else { 1725 ops[i]->ldpc_enc.cb_params.e = ldpc_enc->cb_params.e; 1726 } 1727 ops[i]->ldpc_enc.basegraph = ldpc_enc->basegraph; 1728 ops[i]->ldpc_enc.z_c = ldpc_enc->z_c; 1729 ops[i]->ldpc_enc.q_m = ldpc_enc->q_m; 1730 ops[i]->ldpc_enc.n_filler = ldpc_enc->n_filler; 1731 ops[i]->ldpc_enc.n_cb = ldpc_enc->n_cb; 1732 ops[i]->ldpc_enc.rv_index = ldpc_enc->rv_index; 1733 ops[i]->ldpc_enc.op_flags = ldpc_enc->op_flags; 1734 ops[i]->ldpc_enc.code_block_mode = ldpc_enc->code_block_mode; 1735 ops[i]->ldpc_enc.output = outputs[start_idx + i]; 1736 ops[i]->ldpc_enc.input = inputs[start_idx + i]; 1737 } 1738 } 1739 1740 static int 1741 check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, 1742 unsigned int order_idx, const int expected_status) 1743 { 1744 int status = op->status; 1745 /* ignore parity mismatch false alarms for long iterations */ 1746 if (get_iter_max() >= 10) { 1747 if (!(expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1748 (status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1749 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1750 status -= (1 << RTE_BBDEV_SYNDROME_ERROR); 1751 } 1752 if ((expected_status & (1 << RTE_BBDEV_SYNDROME_ERROR)) && 1753 !(status & (1 << RTE_BBDEV_SYNDROME_ERROR))) { 1754 printf("WARNING: Ignore Syndrome Check mismatch\n"); 1755 status += (1 << RTE_BBDEV_SYNDROME_ERROR); 1756 } 1757 } 1758 1759 TEST_ASSERT(status == expected_status, 1760 "op_status (%d) != expected_status (%d)", 1761 op->status, expected_status); 1762 1763 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1764 "Ordering error, expected %p, got %p", 1765 (void *)(uintptr_t)order_idx, op->opaque_data); 1766 1767 return TEST_SUCCESS; 1768 } 1769 1770 static int 1771 check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, 1772 unsigned int order_idx, const int expected_status) 1773 { 1774 TEST_ASSERT(op->status == expected_status, 1775 "op_status (%d) != expected_status (%d)", 1776 op->status, expected_status); 1777 1778 if (op->opaque_data != (void *)(uintptr_t)INVALID_OPAQUE) 1779 TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, 1780 "Ordering error, expected %p, got %p", 1781 (void *)(uintptr_t)order_idx, op->opaque_data); 1782 1783 return TEST_SUCCESS; 1784 } 1785 1786 static inline int 1787 validate_op_chain(struct rte_bbdev_op_data *op, 1788 struct op_data_entries *orig_op) 1789 { 1790 uint8_t i; 1791 struct rte_mbuf *m = op->data; 1792 uint8_t nb_dst_segments = orig_op->nb_segments; 1793 uint32_t total_data_size = 0; 1794 1795 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1796 "Number of segments differ in original (%u) and filled (%u) op", 1797 nb_dst_segments, m->nb_segs); 1798 1799 /* Validate each mbuf segment length */ 1800 for (i = 0; i < nb_dst_segments; ++i) { 1801 /* Apply offset to the first mbuf segment */ 1802 uint16_t offset = (i == 0) ? op->offset : 0; 1803 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1804 total_data_size += orig_op->segments[i].length; 1805 1806 TEST_ASSERT(orig_op->segments[i].length == data_len, 1807 "Length of segment differ in original (%u) and filled (%u) op", 1808 orig_op->segments[i].length, data_len); 1809 TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, 1810 rte_pktmbuf_mtod_offset(m, uint32_t *, offset), 1811 data_len, 1812 "Output buffers (CB=%u) are not equal", i); 1813 m = m->next; 1814 } 1815 1816 /* Validate total mbuf pkt length */ 1817 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1818 TEST_ASSERT(total_data_size == pkt_len, 1819 "Length of data differ in original (%u) and filled (%u) op", 1820 total_data_size, pkt_len); 1821 1822 return TEST_SUCCESS; 1823 } 1824 1825 /* 1826 * Compute K0 for a given configuration for HARQ output length computation 1827 * As per definition in 3GPP 38.212 Table 5.4.2.1-2 1828 */ 1829 static inline uint16_t 1830 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 1831 { 1832 if (rv_index == 0) 1833 return 0; 1834 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1835 if (n_cb == n) { 1836 if (rv_index == 1) 1837 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 1838 else if (rv_index == 2) 1839 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 1840 else 1841 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 1842 } 1843 /* LBRM case - includes a division by N */ 1844 if (rv_index == 1) 1845 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 1846 / n) * z_c; 1847 else if (rv_index == 2) 1848 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 1849 / n) * z_c; 1850 else 1851 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 1852 / n) * z_c; 1853 } 1854 1855 /* HARQ output length including the Filler bits */ 1856 static inline uint16_t 1857 compute_harq_len(struct rte_bbdev_op_ldpc_dec *ops_ld) 1858 { 1859 uint16_t k0 = 0; 1860 uint8_t max_rv = (ops_ld->rv_index == 1) ? 3 : ops_ld->rv_index; 1861 k0 = get_k0(ops_ld->n_cb, ops_ld->z_c, ops_ld->basegraph, max_rv); 1862 /* Compute RM out size and number of rows */ 1863 uint16_t parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1864 * ops_ld->z_c - ops_ld->n_filler; 1865 uint16_t deRmOutSize = RTE_MIN( 1866 k0 + ops_ld->cb_params.e + 1867 ((k0 > parity_offset) ? 1868 0 : ops_ld->n_filler), 1869 ops_ld->n_cb); 1870 uint16_t numRows = ((deRmOutSize + ops_ld->z_c - 1) 1871 / ops_ld->z_c); 1872 uint16_t harq_output_len = numRows * ops_ld->z_c; 1873 return harq_output_len; 1874 } 1875 1876 static inline int 1877 validate_op_harq_chain(struct rte_bbdev_op_data *op, 1878 struct op_data_entries *orig_op, 1879 struct rte_bbdev_op_ldpc_dec *ops_ld) 1880 { 1881 uint8_t i; 1882 uint32_t j, jj, k; 1883 struct rte_mbuf *m = op->data; 1884 uint8_t nb_dst_segments = orig_op->nb_segments; 1885 uint32_t total_data_size = 0; 1886 int8_t *harq_orig, *harq_out, abs_harq_origin; 1887 uint32_t byte_error = 0, cum_error = 0, error; 1888 int16_t llr_max = (1 << (ldpc_llr_size - ldpc_llr_decimals)) - 1; 1889 int16_t llr_max_pre_scaling = (1 << (ldpc_llr_size - 1)) - 1; 1890 uint16_t parity_offset; 1891 1892 TEST_ASSERT(nb_dst_segments == m->nb_segs, 1893 "Number of segments differ in original (%u) and filled (%u) op", 1894 nb_dst_segments, m->nb_segs); 1895 1896 /* Validate each mbuf segment length */ 1897 for (i = 0; i < nb_dst_segments; ++i) { 1898 /* Apply offset to the first mbuf segment */ 1899 uint16_t offset = (i == 0) ? op->offset : 0; 1900 uint16_t data_len = rte_pktmbuf_data_len(m) - offset; 1901 total_data_size += orig_op->segments[i].length; 1902 1903 TEST_ASSERT(orig_op->segments[i].length < 1904 (uint32_t)(data_len + 64), 1905 "Length of segment differ in original (%u) and filled (%u) op", 1906 orig_op->segments[i].length, data_len); 1907 harq_orig = (int8_t *) orig_op->segments[i].addr; 1908 harq_out = rte_pktmbuf_mtod_offset(m, int8_t *, offset); 1909 1910 if (!(ldpc_cap_flags & 1911 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS 1912 ) || (ops_ld->op_flags & 1913 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1914 data_len -= ops_ld->z_c; 1915 parity_offset = data_len; 1916 } else { 1917 /* Compute RM out size and number of rows */ 1918 parity_offset = (ops_ld->basegraph == 1 ? 20 : 8) 1919 * ops_ld->z_c - ops_ld->n_filler; 1920 uint16_t deRmOutSize = compute_harq_len(ops_ld) - 1921 ops_ld->n_filler; 1922 if (data_len > deRmOutSize) 1923 data_len = deRmOutSize; 1924 if (data_len > orig_op->segments[i].length) 1925 data_len = orig_op->segments[i].length; 1926 } 1927 /* 1928 * HARQ output can have minor differences 1929 * due to integer representation and related scaling 1930 */ 1931 for (j = 0, jj = 0; j < data_len; j++, jj++) { 1932 if (j == parity_offset) { 1933 /* Special Handling of the filler bits */ 1934 for (k = 0; k < ops_ld->n_filler; k++) { 1935 if (harq_out[jj] != 1936 llr_max_pre_scaling) { 1937 printf("HARQ Filler issue %d: %d %d\n", 1938 jj, harq_out[jj], 1939 llr_max); 1940 byte_error++; 1941 } 1942 jj++; 1943 } 1944 } 1945 if (!(ops_ld->op_flags & 1946 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1947 if (ldpc_llr_decimals > 1) 1948 harq_out[jj] = (harq_out[jj] + 1) 1949 >> (ldpc_llr_decimals - 1); 1950 /* Saturated to S7 */ 1951 if (harq_orig[j] > llr_max) 1952 harq_orig[j] = llr_max; 1953 if (harq_orig[j] < -llr_max) 1954 harq_orig[j] = -llr_max; 1955 } 1956 if (harq_orig[j] != harq_out[jj]) { 1957 error = (harq_orig[j] > harq_out[jj]) ? 1958 harq_orig[j] - harq_out[jj] : 1959 harq_out[jj] - harq_orig[j]; 1960 abs_harq_origin = harq_orig[j] > 0 ? 1961 harq_orig[j] : 1962 -harq_orig[j]; 1963 /* Residual quantization error */ 1964 if ((error > 8 && (abs_harq_origin < 1965 (llr_max - 16))) || 1966 (error > 16)) { 1967 printf("HARQ mismatch %d: exp %d act %d => %d\n", 1968 j, harq_orig[j], 1969 harq_out[jj], error); 1970 byte_error++; 1971 cum_error += error; 1972 } 1973 } 1974 } 1975 m = m->next; 1976 } 1977 1978 if (byte_error) 1979 TEST_ASSERT(byte_error <= 1, 1980 "HARQ output mismatch (%d) %d", 1981 byte_error, cum_error); 1982 1983 /* Validate total mbuf pkt length */ 1984 uint32_t pkt_len = rte_pktmbuf_pkt_len(op->data) - op->offset; 1985 TEST_ASSERT(total_data_size < pkt_len + 64, 1986 "Length of data differ in original (%u) and filled (%u) op", 1987 total_data_size, pkt_len); 1988 1989 return TEST_SUCCESS; 1990 } 1991 1992 static int 1993 validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 1994 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 1995 { 1996 unsigned int i; 1997 int ret; 1998 struct op_data_entries *hard_data_orig = 1999 &test_vector.entries[DATA_HARD_OUTPUT]; 2000 struct op_data_entries *soft_data_orig = 2001 &test_vector.entries[DATA_SOFT_OUTPUT]; 2002 struct rte_bbdev_op_turbo_dec *ops_td; 2003 struct rte_bbdev_op_data *hard_output; 2004 struct rte_bbdev_op_data *soft_output; 2005 struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; 2006 2007 for (i = 0; i < n; ++i) { 2008 ops_td = &ops[i]->turbo_dec; 2009 hard_output = &ops_td->hard_output; 2010 soft_output = &ops_td->soft_output; 2011 2012 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2013 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2014 "Returned iter_count (%d) > expected iter_count (%d)", 2015 ops_td->iter_count, ref_td->iter_count); 2016 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2017 TEST_ASSERT_SUCCESS(ret, 2018 "Checking status and ordering for decoder failed"); 2019 2020 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2021 hard_data_orig), 2022 "Hard output buffers (CB=%u) are not equal", 2023 i); 2024 2025 if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) 2026 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2027 soft_data_orig), 2028 "Soft output buffers (CB=%u) are not equal", 2029 i); 2030 } 2031 2032 return TEST_SUCCESS; 2033 } 2034 2035 /* Check Number of code blocks errors */ 2036 static int 2037 validate_ldpc_bler(struct rte_bbdev_dec_op **ops, const uint16_t n) 2038 { 2039 unsigned int i; 2040 struct op_data_entries *hard_data_orig = 2041 &test_vector.entries[DATA_HARD_OUTPUT]; 2042 struct rte_bbdev_op_ldpc_dec *ops_td; 2043 struct rte_bbdev_op_data *hard_output; 2044 int errors = 0; 2045 struct rte_mbuf *m; 2046 2047 for (i = 0; i < n; ++i) { 2048 ops_td = &ops[i]->ldpc_dec; 2049 hard_output = &ops_td->hard_output; 2050 m = hard_output->data; 2051 if (memcmp(rte_pktmbuf_mtod_offset(m, uint32_t *, 0), 2052 hard_data_orig->segments[0].addr, 2053 hard_data_orig->segments[0].length)) 2054 errors++; 2055 } 2056 return errors; 2057 } 2058 2059 static int 2060 validate_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, 2061 struct rte_bbdev_dec_op *ref_op, const int vector_mask) 2062 { 2063 unsigned int i; 2064 int ret; 2065 struct op_data_entries *hard_data_orig = 2066 &test_vector.entries[DATA_HARD_OUTPUT]; 2067 struct op_data_entries *soft_data_orig = 2068 &test_vector.entries[DATA_SOFT_OUTPUT]; 2069 struct op_data_entries *harq_data_orig = 2070 &test_vector.entries[DATA_HARQ_OUTPUT]; 2071 struct rte_bbdev_op_ldpc_dec *ops_td; 2072 struct rte_bbdev_op_data *hard_output; 2073 struct rte_bbdev_op_data *harq_output; 2074 struct rte_bbdev_op_data *soft_output; 2075 struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec; 2076 2077 for (i = 0; i < n; ++i) { 2078 ops_td = &ops[i]->ldpc_dec; 2079 hard_output = &ops_td->hard_output; 2080 harq_output = &ops_td->harq_combined_output; 2081 soft_output = &ops_td->soft_output; 2082 2083 ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); 2084 TEST_ASSERT_SUCCESS(ret, 2085 "Checking status and ordering for decoder failed"); 2086 if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) 2087 TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, 2088 "Returned iter_count (%d) > expected iter_count (%d)", 2089 ops_td->iter_count, ref_td->iter_count); 2090 /* 2091 * We can ignore output data when the decoding failed to 2092 * converge or for loop-back cases 2093 */ 2094 if (!check_bit(ops[i]->ldpc_dec.op_flags, 2095 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK 2096 ) && ( 2097 ops[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR 2098 )) == 0) 2099 TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, 2100 hard_data_orig), 2101 "Hard output buffers (CB=%u) are not equal", 2102 i); 2103 2104 if (ref_op->ldpc_dec.op_flags & RTE_BBDEV_LDPC_SOFT_OUT_ENABLE) 2105 TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, 2106 soft_data_orig), 2107 "Soft output buffers (CB=%u) are not equal", 2108 i); 2109 if (ref_op->ldpc_dec.op_flags & 2110 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE) { 2111 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2112 harq_data_orig, ops_td), 2113 "HARQ output buffers (CB=%u) are not equal", 2114 i); 2115 } 2116 if (ref_op->ldpc_dec.op_flags & 2117 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK) 2118 TEST_ASSERT_SUCCESS(validate_op_harq_chain(harq_output, 2119 harq_data_orig, ops_td), 2120 "HARQ output buffers (CB=%u) are not equal", 2121 i); 2122 2123 } 2124 2125 return TEST_SUCCESS; 2126 } 2127 2128 2129 static int 2130 validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2131 struct rte_bbdev_enc_op *ref_op) 2132 { 2133 unsigned int i; 2134 int ret; 2135 struct op_data_entries *hard_data_orig = 2136 &test_vector.entries[DATA_HARD_OUTPUT]; 2137 2138 for (i = 0; i < n; ++i) { 2139 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2140 TEST_ASSERT_SUCCESS(ret, 2141 "Checking status and ordering for encoder failed"); 2142 TEST_ASSERT_SUCCESS(validate_op_chain( 2143 &ops[i]->turbo_enc.output, 2144 hard_data_orig), 2145 "Output buffers (CB=%u) are not equal", 2146 i); 2147 } 2148 2149 return TEST_SUCCESS; 2150 } 2151 2152 static int 2153 validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, 2154 struct rte_bbdev_enc_op *ref_op) 2155 { 2156 unsigned int i; 2157 int ret; 2158 struct op_data_entries *hard_data_orig = 2159 &test_vector.entries[DATA_HARD_OUTPUT]; 2160 2161 for (i = 0; i < n; ++i) { 2162 ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); 2163 TEST_ASSERT_SUCCESS(ret, 2164 "Checking status and ordering for encoder failed"); 2165 TEST_ASSERT_SUCCESS(validate_op_chain( 2166 &ops[i]->ldpc_enc.output, 2167 hard_data_orig), 2168 "Output buffers (CB=%u) are not equal", 2169 i); 2170 } 2171 2172 return TEST_SUCCESS; 2173 } 2174 2175 static void 2176 create_reference_dec_op(struct rte_bbdev_dec_op *op) 2177 { 2178 unsigned int i; 2179 struct op_data_entries *entry; 2180 2181 op->turbo_dec = test_vector.turbo_dec; 2182 entry = &test_vector.entries[DATA_INPUT]; 2183 for (i = 0; i < entry->nb_segments; ++i) 2184 op->turbo_dec.input.length += 2185 entry->segments[i].length; 2186 } 2187 2188 static void 2189 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op) 2190 { 2191 unsigned int i; 2192 struct op_data_entries *entry; 2193 2194 op->ldpc_dec = test_vector.ldpc_dec; 2195 entry = &test_vector.entries[DATA_INPUT]; 2196 for (i = 0; i < entry->nb_segments; ++i) 2197 op->ldpc_dec.input.length += 2198 entry->segments[i].length; 2199 if (test_vector.ldpc_dec.op_flags & 2200 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE) { 2201 entry = &test_vector.entries[DATA_HARQ_INPUT]; 2202 for (i = 0; i < entry->nb_segments; ++i) 2203 op->ldpc_dec.harq_combined_input.length += 2204 entry->segments[i].length; 2205 } 2206 } 2207 2208 2209 static void 2210 create_reference_enc_op(struct rte_bbdev_enc_op *op) 2211 { 2212 unsigned int i; 2213 struct op_data_entries *entry; 2214 2215 op->turbo_enc = test_vector.turbo_enc; 2216 entry = &test_vector.entries[DATA_INPUT]; 2217 for (i = 0; i < entry->nb_segments; ++i) 2218 op->turbo_enc.input.length += 2219 entry->segments[i].length; 2220 } 2221 2222 static void 2223 create_reference_ldpc_enc_op(struct rte_bbdev_enc_op *op) 2224 { 2225 unsigned int i; 2226 struct op_data_entries *entry; 2227 2228 op->ldpc_enc = test_vector.ldpc_enc; 2229 entry = &test_vector.entries[DATA_INPUT]; 2230 for (i = 0; i < entry->nb_segments; ++i) 2231 op->ldpc_enc.input.length += 2232 entry->segments[i].length; 2233 } 2234 2235 static uint32_t 2236 calc_dec_TB_size(struct rte_bbdev_dec_op *op) 2237 { 2238 uint8_t i; 2239 uint32_t c, r, tb_size = 0; 2240 2241 if (op->turbo_dec.code_block_mode) { 2242 tb_size = op->turbo_dec.tb_params.k_neg; 2243 } else { 2244 c = op->turbo_dec.tb_params.c; 2245 r = op->turbo_dec.tb_params.r; 2246 for (i = 0; i < c-r; i++) 2247 tb_size += (r < op->turbo_dec.tb_params.c_neg) ? 2248 op->turbo_dec.tb_params.k_neg : 2249 op->turbo_dec.tb_params.k_pos; 2250 } 2251 return tb_size; 2252 } 2253 2254 static uint32_t 2255 calc_ldpc_dec_TB_size(struct rte_bbdev_dec_op *op) 2256 { 2257 uint8_t i; 2258 uint32_t c, r, tb_size = 0; 2259 uint16_t sys_cols = (op->ldpc_dec.basegraph == 1) ? 22 : 10; 2260 2261 if (op->ldpc_dec.code_block_mode) { 2262 tb_size = sys_cols * op->ldpc_dec.z_c - op->ldpc_dec.n_filler; 2263 } else { 2264 c = op->ldpc_dec.tb_params.c; 2265 r = op->ldpc_dec.tb_params.r; 2266 for (i = 0; i < c-r; i++) 2267 tb_size += sys_cols * op->ldpc_dec.z_c 2268 - op->ldpc_dec.n_filler; 2269 } 2270 return tb_size; 2271 } 2272 2273 static uint32_t 2274 calc_enc_TB_size(struct rte_bbdev_enc_op *op) 2275 { 2276 uint8_t i; 2277 uint32_t c, r, tb_size = 0; 2278 2279 if (op->turbo_enc.code_block_mode) { 2280 tb_size = op->turbo_enc.tb_params.k_neg; 2281 } else { 2282 c = op->turbo_enc.tb_params.c; 2283 r = op->turbo_enc.tb_params.r; 2284 for (i = 0; i < c-r; i++) 2285 tb_size += (r < op->turbo_enc.tb_params.c_neg) ? 2286 op->turbo_enc.tb_params.k_neg : 2287 op->turbo_enc.tb_params.k_pos; 2288 } 2289 return tb_size; 2290 } 2291 2292 static uint32_t 2293 calc_ldpc_enc_TB_size(struct rte_bbdev_enc_op *op) 2294 { 2295 uint8_t i; 2296 uint32_t c, r, tb_size = 0; 2297 uint16_t sys_cols = (op->ldpc_enc.basegraph == 1) ? 22 : 10; 2298 2299 if (op->turbo_enc.code_block_mode) { 2300 tb_size = sys_cols * op->ldpc_enc.z_c - op->ldpc_enc.n_filler; 2301 } else { 2302 c = op->turbo_enc.tb_params.c; 2303 r = op->turbo_enc.tb_params.r; 2304 for (i = 0; i < c-r; i++) 2305 tb_size += sys_cols * op->ldpc_enc.z_c 2306 - op->ldpc_enc.n_filler; 2307 } 2308 return tb_size; 2309 } 2310 2311 2312 static int 2313 init_test_op_params(struct test_op_params *op_params, 2314 enum rte_bbdev_op_type op_type, const int expected_status, 2315 const int vector_mask, struct rte_mempool *ops_mp, 2316 uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) 2317 { 2318 int ret = 0; 2319 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2320 op_type == RTE_BBDEV_OP_LDPC_DEC) 2321 ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, 2322 &op_params->ref_dec_op, 1); 2323 else 2324 ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, 2325 &op_params->ref_enc_op, 1); 2326 2327 TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); 2328 2329 op_params->mp = ops_mp; 2330 op_params->burst_sz = burst_sz; 2331 op_params->num_to_process = num_to_process; 2332 op_params->num_lcores = num_lcores; 2333 op_params->vector_mask = vector_mask; 2334 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 2335 op_type == RTE_BBDEV_OP_LDPC_DEC) 2336 op_params->ref_dec_op->status = expected_status; 2337 else if (op_type == RTE_BBDEV_OP_TURBO_ENC 2338 || op_type == RTE_BBDEV_OP_LDPC_ENC) 2339 op_params->ref_enc_op->status = expected_status; 2340 return 0; 2341 } 2342 2343 static int 2344 run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, 2345 struct test_op_params *op_params) 2346 { 2347 int t_ret, f_ret, socket_id = SOCKET_ID_ANY; 2348 unsigned int i; 2349 struct active_device *ad; 2350 unsigned int burst_sz = get_burst_sz(); 2351 enum rte_bbdev_op_type op_type = test_vector.op_type; 2352 const struct rte_bbdev_op_cap *capabilities = NULL; 2353 2354 ad = &active_devs[dev_id]; 2355 2356 /* Check if device supports op_type */ 2357 if (!is_avail_op(ad, test_vector.op_type)) 2358 return TEST_SUCCESS; 2359 2360 struct rte_bbdev_info info; 2361 rte_bbdev_info_get(ad->dev_id, &info); 2362 socket_id = GET_SOCKET(info.socket_id); 2363 2364 f_ret = create_mempools(ad, socket_id, op_type, 2365 get_num_ops()); 2366 if (f_ret != TEST_SUCCESS) { 2367 printf("Couldn't create mempools"); 2368 goto fail; 2369 } 2370 if (op_type == RTE_BBDEV_OP_NONE) 2371 op_type = RTE_BBDEV_OP_TURBO_ENC; 2372 2373 f_ret = init_test_op_params(op_params, test_vector.op_type, 2374 test_vector.expected_status, 2375 test_vector.mask, 2376 ad->ops_mempool, 2377 burst_sz, 2378 get_num_ops(), 2379 get_num_lcores()); 2380 if (f_ret != TEST_SUCCESS) { 2381 printf("Couldn't init test op params"); 2382 goto fail; 2383 } 2384 2385 2386 /* Find capabilities */ 2387 const struct rte_bbdev_op_cap *cap = info.drv.capabilities; 2388 for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) { 2389 if (cap->type == test_vector.op_type) { 2390 capabilities = cap; 2391 break; 2392 } 2393 cap++; 2394 } 2395 TEST_ASSERT_NOT_NULL(capabilities, 2396 "Couldn't find capabilities"); 2397 2398 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2399 create_reference_dec_op(op_params->ref_dec_op); 2400 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 2401 create_reference_enc_op(op_params->ref_enc_op); 2402 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2403 create_reference_ldpc_enc_op(op_params->ref_enc_op); 2404 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2405 create_reference_ldpc_dec_op(op_params->ref_dec_op); 2406 2407 for (i = 0; i < ad->nb_queues; ++i) { 2408 f_ret = fill_queue_buffers(op_params, 2409 ad->in_mbuf_pool, 2410 ad->hard_out_mbuf_pool, 2411 ad->soft_out_mbuf_pool, 2412 ad->harq_in_mbuf_pool, 2413 ad->harq_out_mbuf_pool, 2414 ad->queue_ids[i], 2415 capabilities, 2416 info.drv.min_alignment, 2417 socket_id); 2418 if (f_ret != TEST_SUCCESS) { 2419 printf("Couldn't init queue buffers"); 2420 goto fail; 2421 } 2422 } 2423 2424 /* Run test case function */ 2425 t_ret = test_case_func(ad, op_params); 2426 2427 /* Free active device resources and return */ 2428 free_buffers(ad, op_params); 2429 return t_ret; 2430 2431 fail: 2432 free_buffers(ad, op_params); 2433 return TEST_FAILED; 2434 } 2435 2436 /* Run given test function per active device per supported op type 2437 * per burst size. 2438 */ 2439 static int 2440 run_test_case(test_case_function *test_case_func) 2441 { 2442 int ret = 0; 2443 uint8_t dev; 2444 2445 /* Alloc op_params */ 2446 struct test_op_params *op_params = rte_zmalloc(NULL, 2447 sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); 2448 TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", 2449 RTE_ALIGN(sizeof(struct test_op_params), 2450 RTE_CACHE_LINE_SIZE)); 2451 2452 /* For each device run test case function */ 2453 for (dev = 0; dev < nb_active_devs; ++dev) 2454 ret |= run_test_case_on_device(test_case_func, dev, op_params); 2455 2456 rte_free(op_params); 2457 2458 return ret; 2459 } 2460 2461 2462 /* Push back the HARQ output from DDR to host */ 2463 static void 2464 retrieve_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2465 struct rte_bbdev_dec_op **ops, 2466 const uint16_t n) 2467 { 2468 uint16_t j; 2469 int save_status, ret; 2470 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024; 2471 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 2472 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2473 bool loopback = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK; 2474 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2475 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2476 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2477 for (j = 0; j < n; ++j) { 2478 if ((loopback && mem_out) || hc_out) { 2479 save_status = ops[j]->status; 2480 ops[j]->ldpc_dec.op_flags = 2481 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2482 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2483 if (h_comp) 2484 ops[j]->ldpc_dec.op_flags += 2485 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2486 ops[j]->ldpc_dec.harq_combined_input.offset = 2487 harq_offset; 2488 ops[j]->ldpc_dec.harq_combined_output.offset = 0; 2489 harq_offset += HARQ_INCR; 2490 if (!loopback) 2491 ops[j]->ldpc_dec.harq_combined_input.length = 2492 ops[j]->ldpc_dec.harq_combined_output.length; 2493 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2494 &ops[j], 1); 2495 ret = 0; 2496 while (ret == 0) 2497 ret = rte_bbdev_dequeue_ldpc_dec_ops( 2498 dev_id, queue_id, 2499 &ops_deq[j], 1); 2500 ops[j]->ldpc_dec.op_flags = flags; 2501 ops[j]->status = save_status; 2502 } 2503 } 2504 } 2505 2506 /* 2507 * Push back the HARQ output from HW DDR to Host 2508 * Preload HARQ memory input and adjust HARQ offset 2509 */ 2510 static void 2511 preload_harq_ddr(uint16_t dev_id, uint16_t queue_id, 2512 struct rte_bbdev_dec_op **ops, const uint16_t n, 2513 bool preload) 2514 { 2515 uint16_t j; 2516 int ret; 2517 uint32_t harq_offset = (uint32_t) queue_id * HARQ_INCR * 1024; 2518 struct rte_bbdev_op_data save_hc_in, save_hc_out; 2519 struct rte_bbdev_dec_op *ops_deq[MAX_BURST]; 2520 uint32_t flags = ops[0]->ldpc_dec.op_flags; 2521 bool mem_in = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE; 2522 bool hc_in = flags & RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE; 2523 bool mem_out = flags & RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2524 bool hc_out = flags & RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE; 2525 bool h_comp = flags & RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2526 for (j = 0; j < n; ++j) { 2527 if ((mem_in || hc_in) && preload) { 2528 save_hc_in = ops[j]->ldpc_dec.harq_combined_input; 2529 save_hc_out = ops[j]->ldpc_dec.harq_combined_output; 2530 ops[j]->ldpc_dec.op_flags = 2531 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK + 2532 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 2533 if (h_comp) 2534 ops[j]->ldpc_dec.op_flags += 2535 RTE_BBDEV_LDPC_HARQ_6BIT_COMPRESSION; 2536 ops[j]->ldpc_dec.harq_combined_output.offset = 2537 harq_offset; 2538 ops[j]->ldpc_dec.harq_combined_input.offset = 0; 2539 rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 2540 &ops[j], 1); 2541 ret = 0; 2542 while (ret == 0) 2543 ret = rte_bbdev_dequeue_ldpc_dec_ops( 2544 dev_id, queue_id, &ops_deq[j], 1); 2545 ops[j]->ldpc_dec.op_flags = flags; 2546 ops[j]->ldpc_dec.harq_combined_input = save_hc_in; 2547 ops[j]->ldpc_dec.harq_combined_output = save_hc_out; 2548 } 2549 /* Adjust HARQ offset when we reach external DDR */ 2550 if (mem_in || hc_in) 2551 ops[j]->ldpc_dec.harq_combined_input.offset 2552 = harq_offset; 2553 if (mem_out || hc_out) 2554 ops[j]->ldpc_dec.harq_combined_output.offset 2555 = harq_offset; 2556 harq_offset += HARQ_INCR; 2557 } 2558 } 2559 2560 static void 2561 dequeue_event_callback(uint16_t dev_id, 2562 enum rte_bbdev_event_type event, void *cb_arg, 2563 void *ret_param) 2564 { 2565 int ret; 2566 uint16_t i; 2567 uint64_t total_time; 2568 uint16_t deq, burst_sz, num_ops; 2569 uint16_t queue_id = *(uint16_t *) ret_param; 2570 struct rte_bbdev_info info; 2571 double tb_len_bits; 2572 struct thread_params *tp = cb_arg; 2573 2574 /* Find matching thread params using queue_id */ 2575 for (i = 0; i < MAX_QUEUES; ++i, ++tp) 2576 if (tp->queue_id == queue_id) 2577 break; 2578 2579 if (i == MAX_QUEUES) { 2580 printf("%s: Queue_id from interrupt details was not found!\n", 2581 __func__); 2582 return; 2583 } 2584 2585 if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { 2586 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2587 printf( 2588 "Dequeue interrupt handler called for incorrect event!\n"); 2589 return; 2590 } 2591 2592 burst_sz = rte_atomic16_read(&tp->burst_sz); 2593 num_ops = tp->op_params->num_to_process; 2594 2595 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 2596 deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 2597 &tp->dec_ops[ 2598 rte_atomic16_read(&tp->nb_dequeued)], 2599 burst_sz); 2600 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 2601 deq = rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 2602 &tp->dec_ops[ 2603 rte_atomic16_read(&tp->nb_dequeued)], 2604 burst_sz); 2605 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 2606 deq = rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 2607 &tp->enc_ops[ 2608 rte_atomic16_read(&tp->nb_dequeued)], 2609 burst_sz); 2610 else /*RTE_BBDEV_OP_TURBO_ENC*/ 2611 deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 2612 &tp->enc_ops[ 2613 rte_atomic16_read(&tp->nb_dequeued)], 2614 burst_sz); 2615 2616 if (deq < burst_sz) { 2617 printf( 2618 "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", 2619 burst_sz, deq); 2620 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2621 return; 2622 } 2623 2624 if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_ops) { 2625 rte_atomic16_add(&tp->nb_dequeued, deq); 2626 return; 2627 } 2628 2629 total_time = rte_rdtsc_precise() - tp->start_time; 2630 2631 rte_bbdev_info_get(dev_id, &info); 2632 2633 ret = TEST_SUCCESS; 2634 2635 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { 2636 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2637 ret = validate_dec_op(tp->dec_ops, num_ops, ref_op, 2638 tp->op_params->vector_mask); 2639 /* get the max of iter_count for all dequeued ops */ 2640 for (i = 0; i < num_ops; ++i) 2641 tp->iter_count = RTE_MAX( 2642 tp->dec_ops[i]->turbo_dec.iter_count, 2643 tp->iter_count); 2644 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2645 } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) { 2646 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2647 ret = validate_enc_op(tp->enc_ops, num_ops, ref_op); 2648 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2649 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) { 2650 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 2651 ret = validate_ldpc_enc_op(tp->enc_ops, num_ops, ref_op); 2652 rte_bbdev_enc_op_free_bulk(tp->enc_ops, deq); 2653 } else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) { 2654 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2655 ret = validate_ldpc_dec_op(tp->dec_ops, num_ops, ref_op, 2656 tp->op_params->vector_mask); 2657 rte_bbdev_dec_op_free_bulk(tp->dec_ops, deq); 2658 } 2659 2660 if (ret) { 2661 printf("Buffers validation failed\n"); 2662 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2663 } 2664 2665 switch (test_vector.op_type) { 2666 case RTE_BBDEV_OP_TURBO_DEC: 2667 tb_len_bits = calc_dec_TB_size(tp->op_params->ref_dec_op); 2668 break; 2669 case RTE_BBDEV_OP_TURBO_ENC: 2670 tb_len_bits = calc_enc_TB_size(tp->op_params->ref_enc_op); 2671 break; 2672 case RTE_BBDEV_OP_LDPC_DEC: 2673 tb_len_bits = calc_ldpc_dec_TB_size(tp->op_params->ref_dec_op); 2674 break; 2675 case RTE_BBDEV_OP_LDPC_ENC: 2676 tb_len_bits = calc_ldpc_enc_TB_size(tp->op_params->ref_enc_op); 2677 break; 2678 case RTE_BBDEV_OP_NONE: 2679 tb_len_bits = 0.0; 2680 break; 2681 default: 2682 printf("Unknown op type: %d\n", test_vector.op_type); 2683 rte_atomic16_set(&tp->processing_status, TEST_FAILED); 2684 return; 2685 } 2686 2687 tp->ops_per_sec += ((double)num_ops) / 2688 ((double)total_time / (double)rte_get_tsc_hz()); 2689 tp->mbps += (((double)(num_ops * tb_len_bits)) / 1000000.0) / 2690 ((double)total_time / (double)rte_get_tsc_hz()); 2691 2692 rte_atomic16_add(&tp->nb_dequeued, deq); 2693 } 2694 2695 static int 2696 throughput_intr_lcore_ldpc_dec(void *arg) 2697 { 2698 struct thread_params *tp = arg; 2699 unsigned int enqueued; 2700 const uint16_t queue_id = tp->queue_id; 2701 const uint16_t burst_sz = tp->op_params->burst_sz; 2702 const uint16_t num_to_process = tp->op_params->num_to_process; 2703 struct rte_bbdev_dec_op *ops[num_to_process]; 2704 struct test_buffers *bufs = NULL; 2705 struct rte_bbdev_info info; 2706 int ret, i, j; 2707 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 2708 uint16_t num_to_enq, enq; 2709 2710 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 2711 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 2712 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 2713 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 2714 2715 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2716 "BURST_SIZE should be <= %u", MAX_BURST); 2717 2718 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2719 "Failed to enable interrupts for dev: %u, queue_id: %u", 2720 tp->dev_id, queue_id); 2721 2722 rte_bbdev_info_get(tp->dev_id, &info); 2723 2724 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2725 "NUM_OPS cannot exceed %u for this device", 2726 info.drv.queue_size_lim); 2727 2728 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2729 2730 rte_atomic16_clear(&tp->processing_status); 2731 rte_atomic16_clear(&tp->nb_dequeued); 2732 2733 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2734 rte_pause(); 2735 2736 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2737 num_to_process); 2738 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2739 num_to_process); 2740 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2741 copy_reference_ldpc_dec_op(ops, num_to_process, 0, bufs->inputs, 2742 bufs->hard_outputs, bufs->soft_outputs, 2743 bufs->harq_inputs, bufs->harq_outputs, ref_op); 2744 2745 /* Set counter to validate the ordering */ 2746 for (j = 0; j < num_to_process; ++j) 2747 ops[j]->opaque_data = (void *)(uintptr_t)j; 2748 2749 for (j = 0; j < TEST_REPETITIONS; ++j) { 2750 for (i = 0; i < num_to_process; ++i) { 2751 if (!loopback) 2752 rte_pktmbuf_reset( 2753 ops[i]->ldpc_dec.hard_output.data); 2754 if (hc_out || loopback) 2755 mbuf_reset( 2756 ops[i]->ldpc_dec.harq_combined_output.data); 2757 } 2758 2759 tp->start_time = rte_rdtsc_precise(); 2760 for (enqueued = 0; enqueued < num_to_process;) { 2761 num_to_enq = burst_sz; 2762 2763 if (unlikely(num_to_process - enqueued < num_to_enq)) 2764 num_to_enq = num_to_process - enqueued; 2765 2766 enq = 0; 2767 do { 2768 enq += rte_bbdev_enqueue_ldpc_dec_ops( 2769 tp->dev_id, 2770 queue_id, &ops[enqueued], 2771 num_to_enq); 2772 } while (unlikely(num_to_enq != enq)); 2773 enqueued += enq; 2774 2775 /* Write to thread burst_sz current number of enqueued 2776 * descriptors. It ensures that proper number of 2777 * descriptors will be dequeued in callback 2778 * function - needed for last batch in case where 2779 * the number of operations is not a multiple of 2780 * burst size. 2781 */ 2782 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2783 2784 /* Wait until processing of previous batch is 2785 * completed 2786 */ 2787 while (rte_atomic16_read(&tp->nb_dequeued) != 2788 (int16_t) enqueued) 2789 rte_pause(); 2790 } 2791 if (j != TEST_REPETITIONS - 1) 2792 rte_atomic16_clear(&tp->nb_dequeued); 2793 } 2794 2795 return TEST_SUCCESS; 2796 } 2797 2798 static int 2799 throughput_intr_lcore_dec(void *arg) 2800 { 2801 struct thread_params *tp = arg; 2802 unsigned int enqueued; 2803 const uint16_t queue_id = tp->queue_id; 2804 const uint16_t burst_sz = tp->op_params->burst_sz; 2805 const uint16_t num_to_process = tp->op_params->num_to_process; 2806 struct rte_bbdev_dec_op *ops[num_to_process]; 2807 struct test_buffers *bufs = NULL; 2808 struct rte_bbdev_info info; 2809 int ret, i, j; 2810 uint16_t num_to_enq, enq; 2811 2812 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2813 "BURST_SIZE should be <= %u", MAX_BURST); 2814 2815 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2816 "Failed to enable interrupts for dev: %u, queue_id: %u", 2817 tp->dev_id, queue_id); 2818 2819 rte_bbdev_info_get(tp->dev_id, &info); 2820 2821 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2822 "NUM_OPS cannot exceed %u for this device", 2823 info.drv.queue_size_lim); 2824 2825 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2826 2827 rte_atomic16_clear(&tp->processing_status); 2828 rte_atomic16_clear(&tp->nb_dequeued); 2829 2830 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2831 rte_pause(); 2832 2833 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, 2834 num_to_process); 2835 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2836 num_to_process); 2837 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2838 copy_reference_dec_op(ops, num_to_process, 0, bufs->inputs, 2839 bufs->hard_outputs, bufs->soft_outputs, 2840 tp->op_params->ref_dec_op); 2841 2842 /* Set counter to validate the ordering */ 2843 for (j = 0; j < num_to_process; ++j) 2844 ops[j]->opaque_data = (void *)(uintptr_t)j; 2845 2846 for (j = 0; j < TEST_REPETITIONS; ++j) { 2847 for (i = 0; i < num_to_process; ++i) 2848 rte_pktmbuf_reset(ops[i]->turbo_dec.hard_output.data); 2849 2850 tp->start_time = rte_rdtsc_precise(); 2851 for (enqueued = 0; enqueued < num_to_process;) { 2852 num_to_enq = burst_sz; 2853 2854 if (unlikely(num_to_process - enqueued < num_to_enq)) 2855 num_to_enq = num_to_process - enqueued; 2856 2857 enq = 0; 2858 do { 2859 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 2860 queue_id, &ops[enqueued], 2861 num_to_enq); 2862 } while (unlikely(num_to_enq != enq)); 2863 enqueued += enq; 2864 2865 /* Write to thread burst_sz current number of enqueued 2866 * descriptors. It ensures that proper number of 2867 * descriptors will be dequeued in callback 2868 * function - needed for last batch in case where 2869 * the number of operations is not a multiple of 2870 * burst size. 2871 */ 2872 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2873 2874 /* Wait until processing of previous batch is 2875 * completed 2876 */ 2877 while (rte_atomic16_read(&tp->nb_dequeued) != 2878 (int16_t) enqueued) 2879 rte_pause(); 2880 } 2881 if (j != TEST_REPETITIONS - 1) 2882 rte_atomic16_clear(&tp->nb_dequeued); 2883 } 2884 2885 return TEST_SUCCESS; 2886 } 2887 2888 static int 2889 throughput_intr_lcore_enc(void *arg) 2890 { 2891 struct thread_params *tp = arg; 2892 unsigned int enqueued; 2893 const uint16_t queue_id = tp->queue_id; 2894 const uint16_t burst_sz = tp->op_params->burst_sz; 2895 const uint16_t num_to_process = tp->op_params->num_to_process; 2896 struct rte_bbdev_enc_op *ops[num_to_process]; 2897 struct test_buffers *bufs = NULL; 2898 struct rte_bbdev_info info; 2899 int ret, i, j; 2900 uint16_t num_to_enq, enq; 2901 2902 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2903 "BURST_SIZE should be <= %u", MAX_BURST); 2904 2905 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2906 "Failed to enable interrupts for dev: %u, queue_id: %u", 2907 tp->dev_id, queue_id); 2908 2909 rte_bbdev_info_get(tp->dev_id, &info); 2910 2911 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 2912 "NUM_OPS cannot exceed %u for this device", 2913 info.drv.queue_size_lim); 2914 2915 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 2916 2917 rte_atomic16_clear(&tp->processing_status); 2918 rte_atomic16_clear(&tp->nb_dequeued); 2919 2920 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 2921 rte_pause(); 2922 2923 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 2924 num_to_process); 2925 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 2926 num_to_process); 2927 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 2928 copy_reference_enc_op(ops, num_to_process, 0, bufs->inputs, 2929 bufs->hard_outputs, tp->op_params->ref_enc_op); 2930 2931 /* Set counter to validate the ordering */ 2932 for (j = 0; j < num_to_process; ++j) 2933 ops[j]->opaque_data = (void *)(uintptr_t)j; 2934 2935 for (j = 0; j < TEST_REPETITIONS; ++j) { 2936 for (i = 0; i < num_to_process; ++i) 2937 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 2938 2939 tp->start_time = rte_rdtsc_precise(); 2940 for (enqueued = 0; enqueued < num_to_process;) { 2941 num_to_enq = burst_sz; 2942 2943 if (unlikely(num_to_process - enqueued < num_to_enq)) 2944 num_to_enq = num_to_process - enqueued; 2945 2946 enq = 0; 2947 do { 2948 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 2949 queue_id, &ops[enqueued], 2950 num_to_enq); 2951 } while (unlikely(enq != num_to_enq)); 2952 enqueued += enq; 2953 2954 /* Write to thread burst_sz current number of enqueued 2955 * descriptors. It ensures that proper number of 2956 * descriptors will be dequeued in callback 2957 * function - needed for last batch in case where 2958 * the number of operations is not a multiple of 2959 * burst size. 2960 */ 2961 rte_atomic16_set(&tp->burst_sz, num_to_enq); 2962 2963 /* Wait until processing of previous batch is 2964 * completed 2965 */ 2966 while (rte_atomic16_read(&tp->nb_dequeued) != 2967 (int16_t) enqueued) 2968 rte_pause(); 2969 } 2970 if (j != TEST_REPETITIONS - 1) 2971 rte_atomic16_clear(&tp->nb_dequeued); 2972 } 2973 2974 return TEST_SUCCESS; 2975 } 2976 2977 2978 static int 2979 throughput_intr_lcore_ldpc_enc(void *arg) 2980 { 2981 struct thread_params *tp = arg; 2982 unsigned int enqueued; 2983 const uint16_t queue_id = tp->queue_id; 2984 const uint16_t burst_sz = tp->op_params->burst_sz; 2985 const uint16_t num_to_process = tp->op_params->num_to_process; 2986 struct rte_bbdev_enc_op *ops[num_to_process]; 2987 struct test_buffers *bufs = NULL; 2988 struct rte_bbdev_info info; 2989 int ret, i, j; 2990 uint16_t num_to_enq, enq; 2991 2992 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 2993 "BURST_SIZE should be <= %u", MAX_BURST); 2994 2995 TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), 2996 "Failed to enable interrupts for dev: %u, queue_id: %u", 2997 tp->dev_id, queue_id); 2998 2999 rte_bbdev_info_get(tp->dev_id, &info); 3000 3001 TEST_ASSERT_SUCCESS((num_to_process > info.drv.queue_size_lim), 3002 "NUM_OPS cannot exceed %u for this device", 3003 info.drv.queue_size_lim); 3004 3005 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3006 3007 rte_atomic16_clear(&tp->processing_status); 3008 rte_atomic16_clear(&tp->nb_dequeued); 3009 3010 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3011 rte_pause(); 3012 3013 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, 3014 num_to_process); 3015 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3016 num_to_process); 3017 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3018 copy_reference_ldpc_enc_op(ops, num_to_process, 0, 3019 bufs->inputs, bufs->hard_outputs, 3020 tp->op_params->ref_enc_op); 3021 3022 /* Set counter to validate the ordering */ 3023 for (j = 0; j < num_to_process; ++j) 3024 ops[j]->opaque_data = (void *)(uintptr_t)j; 3025 3026 for (j = 0; j < TEST_REPETITIONS; ++j) { 3027 for (i = 0; i < num_to_process; ++i) 3028 rte_pktmbuf_reset(ops[i]->turbo_enc.output.data); 3029 3030 tp->start_time = rte_rdtsc_precise(); 3031 for (enqueued = 0; enqueued < num_to_process;) { 3032 num_to_enq = burst_sz; 3033 3034 if (unlikely(num_to_process - enqueued < num_to_enq)) 3035 num_to_enq = num_to_process - enqueued; 3036 3037 enq = 0; 3038 do { 3039 enq += rte_bbdev_enqueue_ldpc_enc_ops( 3040 tp->dev_id, 3041 queue_id, &ops[enqueued], 3042 num_to_enq); 3043 } while (unlikely(enq != num_to_enq)); 3044 enqueued += enq; 3045 3046 /* Write to thread burst_sz current number of enqueued 3047 * descriptors. It ensures that proper number of 3048 * descriptors will be dequeued in callback 3049 * function - needed for last batch in case where 3050 * the number of operations is not a multiple of 3051 * burst size. 3052 */ 3053 rte_atomic16_set(&tp->burst_sz, num_to_enq); 3054 3055 /* Wait until processing of previous batch is 3056 * completed 3057 */ 3058 while (rte_atomic16_read(&tp->nb_dequeued) != 3059 (int16_t) enqueued) 3060 rte_pause(); 3061 } 3062 if (j != TEST_REPETITIONS - 1) 3063 rte_atomic16_clear(&tp->nb_dequeued); 3064 } 3065 3066 return TEST_SUCCESS; 3067 } 3068 3069 static int 3070 throughput_pmd_lcore_dec(void *arg) 3071 { 3072 struct thread_params *tp = arg; 3073 uint16_t enq, deq; 3074 uint64_t total_time = 0, start_time; 3075 const uint16_t queue_id = tp->queue_id; 3076 const uint16_t burst_sz = tp->op_params->burst_sz; 3077 const uint16_t num_ops = tp->op_params->num_to_process; 3078 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3079 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3080 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3081 struct test_buffers *bufs = NULL; 3082 int i, j, ret; 3083 struct rte_bbdev_info info; 3084 uint16_t num_to_enq; 3085 3086 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3087 "BURST_SIZE should be <= %u", MAX_BURST); 3088 3089 rte_bbdev_info_get(tp->dev_id, &info); 3090 3091 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3092 "NUM_OPS cannot exceed %u for this device", 3093 info.drv.queue_size_lim); 3094 3095 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3096 3097 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3098 rte_pause(); 3099 3100 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3101 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3102 3103 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3104 copy_reference_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3105 bufs->hard_outputs, bufs->soft_outputs, ref_op); 3106 3107 /* Set counter to validate the ordering */ 3108 for (j = 0; j < num_ops; ++j) 3109 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3110 3111 for (i = 0; i < TEST_REPETITIONS; ++i) { 3112 3113 for (j = 0; j < num_ops; ++j) 3114 mbuf_reset(ops_enq[j]->turbo_dec.hard_output.data); 3115 3116 start_time = rte_rdtsc_precise(); 3117 3118 for (enq = 0, deq = 0; enq < num_ops;) { 3119 num_to_enq = burst_sz; 3120 3121 if (unlikely(num_ops - enq < num_to_enq)) 3122 num_to_enq = num_ops - enq; 3123 3124 enq += rte_bbdev_enqueue_dec_ops(tp->dev_id, 3125 queue_id, &ops_enq[enq], num_to_enq); 3126 3127 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3128 queue_id, &ops_deq[deq], enq - deq); 3129 } 3130 3131 /* dequeue the remaining */ 3132 while (deq < enq) { 3133 deq += rte_bbdev_dequeue_dec_ops(tp->dev_id, 3134 queue_id, &ops_deq[deq], enq - deq); 3135 } 3136 3137 total_time += rte_rdtsc_precise() - start_time; 3138 } 3139 3140 tp->iter_count = 0; 3141 /* get the max of iter_count for all dequeued ops */ 3142 for (i = 0; i < num_ops; ++i) { 3143 tp->iter_count = RTE_MAX(ops_enq[i]->turbo_dec.iter_count, 3144 tp->iter_count); 3145 } 3146 3147 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3148 ret = validate_dec_op(ops_deq, num_ops, ref_op, 3149 tp->op_params->vector_mask); 3150 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3151 } 3152 3153 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3154 3155 double tb_len_bits = calc_dec_TB_size(ref_op); 3156 3157 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3158 ((double)total_time / (double)rte_get_tsc_hz()); 3159 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3160 1000000.0) / ((double)total_time / 3161 (double)rte_get_tsc_hz()); 3162 3163 return TEST_SUCCESS; 3164 } 3165 3166 static int 3167 bler_pmd_lcore_ldpc_dec(void *arg) 3168 { 3169 struct thread_params *tp = arg; 3170 uint16_t enq, deq; 3171 uint64_t total_time = 0, start_time; 3172 const uint16_t queue_id = tp->queue_id; 3173 const uint16_t burst_sz = tp->op_params->burst_sz; 3174 const uint16_t num_ops = tp->op_params->num_to_process; 3175 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3176 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3177 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3178 struct test_buffers *bufs = NULL; 3179 int i, j, ret; 3180 float parity_bler = 0; 3181 struct rte_bbdev_info info; 3182 uint16_t num_to_enq; 3183 bool extDdr = check_bit(ldpc_cap_flags, 3184 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3185 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3186 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3187 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3188 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3189 3190 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3191 "BURST_SIZE should be <= %u", MAX_BURST); 3192 3193 rte_bbdev_info_get(tp->dev_id, &info); 3194 3195 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3196 "NUM_OPS cannot exceed %u for this device", 3197 info.drv.queue_size_lim); 3198 3199 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3200 3201 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3202 rte_pause(); 3203 3204 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3205 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3206 3207 /* For BLER tests we need to enable early termination */ 3208 if (!check_bit(ref_op->ldpc_dec.op_flags, 3209 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3210 ref_op->ldpc_dec.op_flags += 3211 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3212 ref_op->ldpc_dec.iter_max = get_iter_max(); 3213 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3214 3215 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3216 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3217 bufs->hard_outputs, bufs->soft_outputs, 3218 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3219 generate_llr_input(num_ops, bufs->inputs, ref_op); 3220 3221 /* Set counter to validate the ordering */ 3222 for (j = 0; j < num_ops; ++j) 3223 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3224 3225 for (i = 0; i < 1; ++i) { /* Could add more iterations */ 3226 for (j = 0; j < num_ops; ++j) { 3227 if (!loopback) 3228 mbuf_reset( 3229 ops_enq[j]->ldpc_dec.hard_output.data); 3230 if (hc_out || loopback) 3231 mbuf_reset( 3232 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3233 } 3234 if (extDdr) { 3235 bool preload = i == (TEST_REPETITIONS - 1); 3236 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3237 num_ops, preload); 3238 } 3239 start_time = rte_rdtsc_precise(); 3240 3241 for (enq = 0, deq = 0; enq < num_ops;) { 3242 num_to_enq = burst_sz; 3243 3244 if (unlikely(num_ops - enq < num_to_enq)) 3245 num_to_enq = num_ops - enq; 3246 3247 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3248 queue_id, &ops_enq[enq], num_to_enq); 3249 3250 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3251 queue_id, &ops_deq[deq], enq - deq); 3252 } 3253 3254 /* dequeue the remaining */ 3255 while (deq < enq) { 3256 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3257 queue_id, &ops_deq[deq], enq - deq); 3258 } 3259 3260 total_time += rte_rdtsc_precise() - start_time; 3261 } 3262 3263 tp->iter_count = 0; 3264 tp->iter_average = 0; 3265 /* get the max of iter_count for all dequeued ops */ 3266 for (i = 0; i < num_ops; ++i) { 3267 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3268 tp->iter_count); 3269 tp->iter_average += (double) ops_enq[i]->ldpc_dec.iter_count; 3270 if (ops_enq[i]->status & (1 << RTE_BBDEV_SYNDROME_ERROR)) 3271 parity_bler += 1.0; 3272 } 3273 3274 parity_bler /= num_ops; /* This one is based on SYND */ 3275 tp->iter_average /= num_ops; 3276 tp->bler = (double) validate_ldpc_bler(ops_deq, num_ops) / num_ops; 3277 3278 if (test_vector.op_type != RTE_BBDEV_OP_NONE 3279 && tp->bler == 0 3280 && parity_bler == 0 3281 && !hc_out) { 3282 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3283 tp->op_params->vector_mask); 3284 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3285 } 3286 3287 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3288 3289 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3290 tp->ops_per_sec = ((double)num_ops * 1) / 3291 ((double)total_time / (double)rte_get_tsc_hz()); 3292 tp->mbps = (((double)(num_ops * 1 * tb_len_bits)) / 3293 1000000.0) / ((double)total_time / 3294 (double)rte_get_tsc_hz()); 3295 3296 return TEST_SUCCESS; 3297 } 3298 3299 static int 3300 throughput_pmd_lcore_ldpc_dec(void *arg) 3301 { 3302 struct thread_params *tp = arg; 3303 uint16_t enq, deq; 3304 uint64_t total_time = 0, start_time; 3305 const uint16_t queue_id = tp->queue_id; 3306 const uint16_t burst_sz = tp->op_params->burst_sz; 3307 const uint16_t num_ops = tp->op_params->num_to_process; 3308 struct rte_bbdev_dec_op *ops_enq[num_ops]; 3309 struct rte_bbdev_dec_op *ops_deq[num_ops]; 3310 struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; 3311 struct test_buffers *bufs = NULL; 3312 int i, j, ret; 3313 struct rte_bbdev_info info; 3314 uint16_t num_to_enq; 3315 bool extDdr = check_bit(ldpc_cap_flags, 3316 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE); 3317 bool loopback = check_bit(ref_op->ldpc_dec.op_flags, 3318 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK); 3319 bool hc_out = check_bit(ref_op->ldpc_dec.op_flags, 3320 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE); 3321 3322 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3323 "BURST_SIZE should be <= %u", MAX_BURST); 3324 3325 rte_bbdev_info_get(tp->dev_id, &info); 3326 3327 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3328 "NUM_OPS cannot exceed %u for this device", 3329 info.drv.queue_size_lim); 3330 3331 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3332 3333 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3334 rte_pause(); 3335 3336 ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops); 3337 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops); 3338 3339 /* For throughput tests we need to disable early termination */ 3340 if (check_bit(ref_op->ldpc_dec.op_flags, 3341 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 3342 ref_op->ldpc_dec.op_flags -= 3343 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 3344 ref_op->ldpc_dec.iter_max = get_iter_max(); 3345 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 3346 3347 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3348 copy_reference_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs, 3349 bufs->hard_outputs, bufs->soft_outputs, 3350 bufs->harq_inputs, bufs->harq_outputs, ref_op); 3351 3352 /* Set counter to validate the ordering */ 3353 for (j = 0; j < num_ops; ++j) 3354 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3355 3356 for (i = 0; i < TEST_REPETITIONS; ++i) { 3357 for (j = 0; j < num_ops; ++j) { 3358 if (!loopback) 3359 mbuf_reset( 3360 ops_enq[j]->ldpc_dec.hard_output.data); 3361 if (hc_out || loopback) 3362 mbuf_reset( 3363 ops_enq[j]->ldpc_dec.harq_combined_output.data); 3364 } 3365 if (extDdr) { 3366 bool preload = i == (TEST_REPETITIONS - 1); 3367 preload_harq_ddr(tp->dev_id, queue_id, ops_enq, 3368 num_ops, preload); 3369 } 3370 start_time = rte_rdtsc_precise(); 3371 3372 for (enq = 0, deq = 0; enq < num_ops;) { 3373 num_to_enq = burst_sz; 3374 3375 if (unlikely(num_ops - enq < num_to_enq)) 3376 num_to_enq = num_ops - enq; 3377 3378 enq += rte_bbdev_enqueue_ldpc_dec_ops(tp->dev_id, 3379 queue_id, &ops_enq[enq], num_to_enq); 3380 3381 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3382 queue_id, &ops_deq[deq], enq - deq); 3383 } 3384 3385 /* dequeue the remaining */ 3386 while (deq < enq) { 3387 deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id, 3388 queue_id, &ops_deq[deq], enq - deq); 3389 } 3390 3391 total_time += rte_rdtsc_precise() - start_time; 3392 } 3393 3394 tp->iter_count = 0; 3395 /* get the max of iter_count for all dequeued ops */ 3396 for (i = 0; i < num_ops; ++i) { 3397 tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count, 3398 tp->iter_count); 3399 } 3400 if (extDdr) { 3401 /* Read loopback is not thread safe */ 3402 retrieve_harq_ddr(tp->dev_id, queue_id, ops_enq, num_ops); 3403 } 3404 3405 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3406 ret = validate_ldpc_dec_op(ops_deq, num_ops, ref_op, 3407 tp->op_params->vector_mask); 3408 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3409 } 3410 3411 rte_bbdev_dec_op_free_bulk(ops_enq, num_ops); 3412 3413 double tb_len_bits = calc_ldpc_dec_TB_size(ref_op); 3414 3415 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3416 ((double)total_time / (double)rte_get_tsc_hz()); 3417 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) / 3418 1000000.0) / ((double)total_time / 3419 (double)rte_get_tsc_hz()); 3420 3421 return TEST_SUCCESS; 3422 } 3423 3424 static int 3425 throughput_pmd_lcore_enc(void *arg) 3426 { 3427 struct thread_params *tp = arg; 3428 uint16_t enq, deq; 3429 uint64_t total_time = 0, start_time; 3430 const uint16_t queue_id = tp->queue_id; 3431 const uint16_t burst_sz = tp->op_params->burst_sz; 3432 const uint16_t num_ops = tp->op_params->num_to_process; 3433 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3434 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3435 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3436 struct test_buffers *bufs = NULL; 3437 int i, j, ret; 3438 struct rte_bbdev_info info; 3439 uint16_t num_to_enq; 3440 3441 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3442 "BURST_SIZE should be <= %u", MAX_BURST); 3443 3444 rte_bbdev_info_get(tp->dev_id, &info); 3445 3446 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3447 "NUM_OPS cannot exceed %u for this device", 3448 info.drv.queue_size_lim); 3449 3450 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3451 3452 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3453 rte_pause(); 3454 3455 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3456 num_ops); 3457 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3458 num_ops); 3459 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3460 copy_reference_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3461 bufs->hard_outputs, ref_op); 3462 3463 /* Set counter to validate the ordering */ 3464 for (j = 0; j < num_ops; ++j) 3465 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3466 3467 for (i = 0; i < TEST_REPETITIONS; ++i) { 3468 3469 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3470 for (j = 0; j < num_ops; ++j) 3471 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3472 3473 start_time = rte_rdtsc_precise(); 3474 3475 for (enq = 0, deq = 0; enq < num_ops;) { 3476 num_to_enq = burst_sz; 3477 3478 if (unlikely(num_ops - enq < num_to_enq)) 3479 num_to_enq = num_ops - enq; 3480 3481 enq += rte_bbdev_enqueue_enc_ops(tp->dev_id, 3482 queue_id, &ops_enq[enq], num_to_enq); 3483 3484 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3485 queue_id, &ops_deq[deq], enq - deq); 3486 } 3487 3488 /* dequeue the remaining */ 3489 while (deq < enq) { 3490 deq += rte_bbdev_dequeue_enc_ops(tp->dev_id, 3491 queue_id, &ops_deq[deq], enq - deq); 3492 } 3493 3494 total_time += rte_rdtsc_precise() - start_time; 3495 } 3496 3497 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3498 ret = validate_enc_op(ops_deq, num_ops, ref_op); 3499 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3500 } 3501 3502 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3503 3504 double tb_len_bits = calc_enc_TB_size(ref_op); 3505 3506 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3507 ((double)total_time / (double)rte_get_tsc_hz()); 3508 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3509 / 1000000.0) / ((double)total_time / 3510 (double)rte_get_tsc_hz()); 3511 3512 return TEST_SUCCESS; 3513 } 3514 3515 static int 3516 throughput_pmd_lcore_ldpc_enc(void *arg) 3517 { 3518 struct thread_params *tp = arg; 3519 uint16_t enq, deq; 3520 uint64_t total_time = 0, start_time; 3521 const uint16_t queue_id = tp->queue_id; 3522 const uint16_t burst_sz = tp->op_params->burst_sz; 3523 const uint16_t num_ops = tp->op_params->num_to_process; 3524 struct rte_bbdev_enc_op *ops_enq[num_ops]; 3525 struct rte_bbdev_enc_op *ops_deq[num_ops]; 3526 struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; 3527 struct test_buffers *bufs = NULL; 3528 int i, j, ret; 3529 struct rte_bbdev_info info; 3530 uint16_t num_to_enq; 3531 3532 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 3533 "BURST_SIZE should be <= %u", MAX_BURST); 3534 3535 rte_bbdev_info_get(tp->dev_id, &info); 3536 3537 TEST_ASSERT_SUCCESS((num_ops > info.drv.queue_size_lim), 3538 "NUM_OPS cannot exceed %u for this device", 3539 info.drv.queue_size_lim); 3540 3541 bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 3542 3543 while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) 3544 rte_pause(); 3545 3546 ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops_enq, 3547 num_ops); 3548 TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", 3549 num_ops); 3550 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3551 copy_reference_ldpc_enc_op(ops_enq, num_ops, 0, bufs->inputs, 3552 bufs->hard_outputs, ref_op); 3553 3554 /* Set counter to validate the ordering */ 3555 for (j = 0; j < num_ops; ++j) 3556 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3557 3558 for (i = 0; i < TEST_REPETITIONS; ++i) { 3559 3560 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3561 for (j = 0; j < num_ops; ++j) 3562 mbuf_reset(ops_enq[j]->turbo_enc.output.data); 3563 3564 start_time = rte_rdtsc_precise(); 3565 3566 for (enq = 0, deq = 0; enq < num_ops;) { 3567 num_to_enq = burst_sz; 3568 3569 if (unlikely(num_ops - enq < num_to_enq)) 3570 num_to_enq = num_ops - enq; 3571 3572 enq += rte_bbdev_enqueue_ldpc_enc_ops(tp->dev_id, 3573 queue_id, &ops_enq[enq], num_to_enq); 3574 3575 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3576 queue_id, &ops_deq[deq], enq - deq); 3577 } 3578 3579 /* dequeue the remaining */ 3580 while (deq < enq) { 3581 deq += rte_bbdev_dequeue_ldpc_enc_ops(tp->dev_id, 3582 queue_id, &ops_deq[deq], enq - deq); 3583 } 3584 3585 total_time += rte_rdtsc_precise() - start_time; 3586 } 3587 3588 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3589 ret = validate_ldpc_enc_op(ops_deq, num_ops, ref_op); 3590 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3591 } 3592 3593 rte_bbdev_enc_op_free_bulk(ops_enq, num_ops); 3594 3595 double tb_len_bits = calc_ldpc_enc_TB_size(ref_op); 3596 3597 tp->ops_per_sec = ((double)num_ops * TEST_REPETITIONS) / 3598 ((double)total_time / (double)rte_get_tsc_hz()); 3599 tp->mbps = (((double)(num_ops * TEST_REPETITIONS * tb_len_bits)) 3600 / 1000000.0) / ((double)total_time / 3601 (double)rte_get_tsc_hz()); 3602 3603 return TEST_SUCCESS; 3604 } 3605 3606 static void 3607 print_enc_throughput(struct thread_params *t_params, unsigned int used_cores) 3608 { 3609 unsigned int iter = 0; 3610 double total_mops = 0, total_mbps = 0; 3611 3612 for (iter = 0; iter < used_cores; iter++) { 3613 printf( 3614 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps\n", 3615 t_params[iter].lcore_id, t_params[iter].ops_per_sec, 3616 t_params[iter].mbps); 3617 total_mops += t_params[iter].ops_per_sec; 3618 total_mbps += t_params[iter].mbps; 3619 } 3620 printf( 3621 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps\n", 3622 used_cores, total_mops, total_mbps); 3623 } 3624 3625 /* Aggregate the performance results over the number of cores used */ 3626 static void 3627 print_dec_throughput(struct thread_params *t_params, unsigned int used_cores) 3628 { 3629 unsigned int core_idx = 0; 3630 double total_mops = 0, total_mbps = 0; 3631 uint8_t iter_count = 0; 3632 3633 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3634 printf( 3635 "Throughput for core (%u): %.8lg Ops/s, %.8lg Mbps @ max %u iterations\n", 3636 t_params[core_idx].lcore_id, 3637 t_params[core_idx].ops_per_sec, 3638 t_params[core_idx].mbps, 3639 t_params[core_idx].iter_count); 3640 total_mops += t_params[core_idx].ops_per_sec; 3641 total_mbps += t_params[core_idx].mbps; 3642 iter_count = RTE_MAX(iter_count, 3643 t_params[core_idx].iter_count); 3644 } 3645 printf( 3646 "\nTotal throughput for %u cores: %.8lg MOPS, %.8lg Mbps @ max %u iterations\n", 3647 used_cores, total_mops, total_mbps, iter_count); 3648 } 3649 3650 /* Aggregate the performance results over the number of cores used */ 3651 static void 3652 print_dec_bler(struct thread_params *t_params, unsigned int used_cores) 3653 { 3654 unsigned int core_idx = 0; 3655 double total_mbps = 0, total_bler = 0, total_iter = 0; 3656 double snr = get_snr(); 3657 3658 for (core_idx = 0; core_idx < used_cores; core_idx++) { 3659 printf("Core%u BLER %.1f %% - Iters %.1f - Tp %.1f Mbps %s\n", 3660 t_params[core_idx].lcore_id, 3661 t_params[core_idx].bler * 100, 3662 t_params[core_idx].iter_average, 3663 t_params[core_idx].mbps, 3664 get_vector_filename()); 3665 total_mbps += t_params[core_idx].mbps; 3666 total_bler += t_params[core_idx].bler; 3667 total_iter += t_params[core_idx].iter_average; 3668 } 3669 total_bler /= used_cores; 3670 total_iter /= used_cores; 3671 3672 printf("SNR %.2f BLER %.1f %% - Iterations %.1f %d - Tp %.1f Mbps %s\n", 3673 snr, total_bler * 100, total_iter, get_iter_max(), 3674 total_mbps, get_vector_filename()); 3675 } 3676 3677 /* 3678 * Test function that determines BLER wireless performance 3679 */ 3680 static int 3681 bler_test(struct active_device *ad, 3682 struct test_op_params *op_params) 3683 { 3684 int ret; 3685 unsigned int lcore_id, used_cores = 0; 3686 struct thread_params *t_params; 3687 struct rte_bbdev_info info; 3688 lcore_function_t *bler_function; 3689 uint16_t num_lcores; 3690 const char *op_type_str; 3691 3692 rte_bbdev_info_get(ad->dev_id, &info); 3693 3694 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3695 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3696 test_vector.op_type); 3697 3698 printf("+ ------------------------------------------------------- +\n"); 3699 printf("== test: bler\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3700 info.dev_name, ad->nb_queues, op_params->burst_sz, 3701 op_params->num_to_process, op_params->num_lcores, 3702 op_type_str, 3703 intr_enabled ? "Interrupt mode" : "PMD mode", 3704 (double)rte_get_tsc_hz() / 1000000000.0); 3705 3706 /* Set number of lcores */ 3707 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3708 ? ad->nb_queues 3709 : op_params->num_lcores; 3710 3711 /* Allocate memory for thread parameters structure */ 3712 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3713 RTE_CACHE_LINE_SIZE); 3714 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3715 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3716 RTE_CACHE_LINE_SIZE)); 3717 3718 if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3719 bler_function = bler_pmd_lcore_ldpc_dec; 3720 else 3721 return TEST_SKIPPED; 3722 3723 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 3724 3725 /* Main core is set at first entry */ 3726 t_params[0].dev_id = ad->dev_id; 3727 t_params[0].lcore_id = rte_lcore_id(); 3728 t_params[0].op_params = op_params; 3729 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3730 t_params[0].iter_count = 0; 3731 3732 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3733 if (used_cores >= num_lcores) 3734 break; 3735 3736 t_params[used_cores].dev_id = ad->dev_id; 3737 t_params[used_cores].lcore_id = lcore_id; 3738 t_params[used_cores].op_params = op_params; 3739 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3740 t_params[used_cores].iter_count = 0; 3741 3742 rte_eal_remote_launch(bler_function, 3743 &t_params[used_cores++], lcore_id); 3744 } 3745 3746 rte_atomic16_set(&op_params->sync, SYNC_START); 3747 ret = bler_function(&t_params[0]); 3748 3749 /* Main core is always used */ 3750 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3751 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3752 3753 print_dec_bler(t_params, num_lcores); 3754 3755 /* Return if test failed */ 3756 if (ret) { 3757 rte_free(t_params); 3758 return ret; 3759 } 3760 3761 /* Function to print something here*/ 3762 rte_free(t_params); 3763 return ret; 3764 } 3765 3766 /* 3767 * Test function that determines how long an enqueue + dequeue of a burst 3768 * takes on available lcores. 3769 */ 3770 static int 3771 throughput_test(struct active_device *ad, 3772 struct test_op_params *op_params) 3773 { 3774 int ret; 3775 unsigned int lcore_id, used_cores = 0; 3776 struct thread_params *t_params, *tp; 3777 struct rte_bbdev_info info; 3778 lcore_function_t *throughput_function; 3779 uint16_t num_lcores; 3780 const char *op_type_str; 3781 3782 rte_bbdev_info_get(ad->dev_id, &info); 3783 3784 op_type_str = rte_bbdev_op_type_str(test_vector.op_type); 3785 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", 3786 test_vector.op_type); 3787 3788 printf("+ ------------------------------------------------------- +\n"); 3789 printf("== test: throughput\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, itr mode: %s, GHz: %lg\n", 3790 info.dev_name, ad->nb_queues, op_params->burst_sz, 3791 op_params->num_to_process, op_params->num_lcores, 3792 op_type_str, 3793 intr_enabled ? "Interrupt mode" : "PMD mode", 3794 (double)rte_get_tsc_hz() / 1000000000.0); 3795 3796 /* Set number of lcores */ 3797 num_lcores = (ad->nb_queues < (op_params->num_lcores)) 3798 ? ad->nb_queues 3799 : op_params->num_lcores; 3800 3801 /* Allocate memory for thread parameters structure */ 3802 t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params), 3803 RTE_CACHE_LINE_SIZE); 3804 TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", 3805 RTE_ALIGN(sizeof(struct thread_params) * num_lcores, 3806 RTE_CACHE_LINE_SIZE)); 3807 3808 if (intr_enabled) { 3809 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3810 throughput_function = throughput_intr_lcore_dec; 3811 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3812 throughput_function = throughput_intr_lcore_ldpc_dec; 3813 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3814 throughput_function = throughput_intr_lcore_enc; 3815 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3816 throughput_function = throughput_intr_lcore_ldpc_enc; 3817 else 3818 throughput_function = throughput_intr_lcore_enc; 3819 3820 /* Dequeue interrupt callback registration */ 3821 ret = rte_bbdev_callback_register(ad->dev_id, 3822 RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, 3823 t_params); 3824 if (ret < 0) { 3825 rte_free(t_params); 3826 return ret; 3827 } 3828 } else { 3829 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) 3830 throughput_function = throughput_pmd_lcore_dec; 3831 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3832 throughput_function = throughput_pmd_lcore_ldpc_dec; 3833 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) 3834 throughput_function = throughput_pmd_lcore_enc; 3835 else if (test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3836 throughput_function = throughput_pmd_lcore_ldpc_enc; 3837 else 3838 throughput_function = throughput_pmd_lcore_enc; 3839 } 3840 3841 rte_atomic16_set(&op_params->sync, SYNC_WAIT); 3842 3843 /* Main core is set at first entry */ 3844 t_params[0].dev_id = ad->dev_id; 3845 t_params[0].lcore_id = rte_lcore_id(); 3846 t_params[0].op_params = op_params; 3847 t_params[0].queue_id = ad->queue_ids[used_cores++]; 3848 t_params[0].iter_count = 0; 3849 3850 RTE_LCORE_FOREACH_WORKER(lcore_id) { 3851 if (used_cores >= num_lcores) 3852 break; 3853 3854 t_params[used_cores].dev_id = ad->dev_id; 3855 t_params[used_cores].lcore_id = lcore_id; 3856 t_params[used_cores].op_params = op_params; 3857 t_params[used_cores].queue_id = ad->queue_ids[used_cores]; 3858 t_params[used_cores].iter_count = 0; 3859 3860 rte_eal_remote_launch(throughput_function, 3861 &t_params[used_cores++], lcore_id); 3862 } 3863 3864 rte_atomic16_set(&op_params->sync, SYNC_START); 3865 ret = throughput_function(&t_params[0]); 3866 3867 /* Main core is always used */ 3868 for (used_cores = 1; used_cores < num_lcores; used_cores++) 3869 ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id); 3870 3871 /* Return if test failed */ 3872 if (ret) { 3873 rte_free(t_params); 3874 return ret; 3875 } 3876 3877 /* Print throughput if interrupts are disabled and test passed */ 3878 if (!intr_enabled) { 3879 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3880 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3881 print_dec_throughput(t_params, num_lcores); 3882 else 3883 print_enc_throughput(t_params, num_lcores); 3884 rte_free(t_params); 3885 return ret; 3886 } 3887 3888 /* In interrupt TC we need to wait for the interrupt callback to deqeue 3889 * all pending operations. Skip waiting for queues which reported an 3890 * error using processing_status variable. 3891 * Wait for main lcore operations. 3892 */ 3893 tp = &t_params[0]; 3894 while ((rte_atomic16_read(&tp->nb_dequeued) < 3895 op_params->num_to_process) && 3896 (rte_atomic16_read(&tp->processing_status) != 3897 TEST_FAILED)) 3898 rte_pause(); 3899 3900 tp->ops_per_sec /= TEST_REPETITIONS; 3901 tp->mbps /= TEST_REPETITIONS; 3902 ret |= (int)rte_atomic16_read(&tp->processing_status); 3903 3904 /* Wait for worker lcores operations */ 3905 for (used_cores = 1; used_cores < num_lcores; used_cores++) { 3906 tp = &t_params[used_cores]; 3907 3908 while ((rte_atomic16_read(&tp->nb_dequeued) < 3909 op_params->num_to_process) && 3910 (rte_atomic16_read(&tp->processing_status) != 3911 TEST_FAILED)) 3912 rte_pause(); 3913 3914 tp->ops_per_sec /= TEST_REPETITIONS; 3915 tp->mbps /= TEST_REPETITIONS; 3916 ret |= (int)rte_atomic16_read(&tp->processing_status); 3917 } 3918 3919 /* Print throughput if test passed */ 3920 if (!ret) { 3921 if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC || 3922 test_vector.op_type == RTE_BBDEV_OP_LDPC_DEC) 3923 print_dec_throughput(t_params, num_lcores); 3924 else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC || 3925 test_vector.op_type == RTE_BBDEV_OP_LDPC_ENC) 3926 print_enc_throughput(t_params, num_lcores); 3927 } 3928 3929 rte_free(t_params); 3930 return ret; 3931 } 3932 3933 static int 3934 latency_test_dec(struct rte_mempool *mempool, 3935 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 3936 int vector_mask, uint16_t dev_id, uint16_t queue_id, 3937 const uint16_t num_to_process, uint16_t burst_sz, 3938 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 3939 { 3940 int ret = TEST_SUCCESS; 3941 uint16_t i, j, dequeued; 3942 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 3943 uint64_t start_time = 0, last_time = 0; 3944 3945 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 3946 uint16_t enq = 0, deq = 0; 3947 bool first_time = true; 3948 last_time = 0; 3949 3950 if (unlikely(num_to_process - dequeued < burst_sz)) 3951 burst_sz = num_to_process - dequeued; 3952 3953 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 3954 TEST_ASSERT_SUCCESS(ret, 3955 "rte_bbdev_dec_op_alloc_bulk() failed"); 3956 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 3957 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 3958 bufs->inputs, 3959 bufs->hard_outputs, 3960 bufs->soft_outputs, 3961 ref_op); 3962 3963 /* Set counter to validate the ordering */ 3964 for (j = 0; j < burst_sz; ++j) 3965 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 3966 3967 start_time = rte_rdtsc_precise(); 3968 3969 enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], 3970 burst_sz); 3971 TEST_ASSERT(enq == burst_sz, 3972 "Error enqueueing burst, expected %u, got %u", 3973 burst_sz, enq); 3974 3975 /* Dequeue */ 3976 do { 3977 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 3978 &ops_deq[deq], burst_sz - deq); 3979 if (likely(first_time && (deq > 0))) { 3980 last_time = rte_rdtsc_precise() - start_time; 3981 first_time = false; 3982 } 3983 } while (unlikely(burst_sz != deq)); 3984 3985 *max_time = RTE_MAX(*max_time, last_time); 3986 *min_time = RTE_MIN(*min_time, last_time); 3987 *total_time += last_time; 3988 3989 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 3990 ret = validate_dec_op(ops_deq, burst_sz, ref_op, 3991 vector_mask); 3992 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 3993 } 3994 3995 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 3996 dequeued += deq; 3997 } 3998 3999 return i; 4000 } 4001 4002 /* Test case for latency/validation for LDPC Decoder */ 4003 static int 4004 latency_test_ldpc_dec(struct rte_mempool *mempool, 4005 struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, 4006 int vector_mask, uint16_t dev_id, uint16_t queue_id, 4007 const uint16_t num_to_process, uint16_t burst_sz, 4008 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time, 4009 bool disable_et) 4010 { 4011 int ret = TEST_SUCCESS; 4012 uint16_t i, j, dequeued; 4013 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4014 uint64_t start_time = 0, last_time = 0; 4015 bool extDdr = ldpc_cap_flags & 4016 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4017 4018 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4019 uint16_t enq = 0, deq = 0; 4020 bool first_time = true; 4021 last_time = 0; 4022 4023 if (unlikely(num_to_process - dequeued < burst_sz)) 4024 burst_sz = num_to_process - dequeued; 4025 4026 ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4027 TEST_ASSERT_SUCCESS(ret, 4028 "rte_bbdev_dec_op_alloc_bulk() failed"); 4029 4030 /* For latency tests we need to disable early termination */ 4031 if (disable_et && check_bit(ref_op->ldpc_dec.op_flags, 4032 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE)) 4033 ref_op->ldpc_dec.op_flags -= 4034 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE; 4035 ref_op->ldpc_dec.iter_max = get_iter_max(); 4036 ref_op->ldpc_dec.iter_count = ref_op->ldpc_dec.iter_max; 4037 4038 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4039 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4040 bufs->inputs, 4041 bufs->hard_outputs, 4042 bufs->soft_outputs, 4043 bufs->harq_inputs, 4044 bufs->harq_outputs, 4045 ref_op); 4046 4047 if (extDdr) 4048 preload_harq_ddr(dev_id, queue_id, ops_enq, 4049 burst_sz, true); 4050 4051 /* Set counter to validate the ordering */ 4052 for (j = 0; j < burst_sz; ++j) 4053 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4054 4055 start_time = rte_rdtsc_precise(); 4056 4057 enq = rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4058 &ops_enq[enq], burst_sz); 4059 TEST_ASSERT(enq == burst_sz, 4060 "Error enqueueing burst, expected %u, got %u", 4061 burst_sz, enq); 4062 4063 /* Dequeue */ 4064 do { 4065 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4066 &ops_deq[deq], burst_sz - deq); 4067 if (likely(first_time && (deq > 0))) { 4068 last_time = rte_rdtsc_precise() - start_time; 4069 first_time = false; 4070 } 4071 } while (unlikely(burst_sz != deq)); 4072 4073 *max_time = RTE_MAX(*max_time, last_time); 4074 *min_time = RTE_MIN(*min_time, last_time); 4075 *total_time += last_time; 4076 4077 if (extDdr) 4078 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4079 4080 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4081 ret = validate_ldpc_dec_op(ops_deq, burst_sz, ref_op, 4082 vector_mask); 4083 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4084 } 4085 4086 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4087 dequeued += deq; 4088 } 4089 return i; 4090 } 4091 4092 static int 4093 latency_test_enc(struct rte_mempool *mempool, 4094 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4095 uint16_t dev_id, uint16_t queue_id, 4096 const uint16_t num_to_process, uint16_t burst_sz, 4097 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4098 { 4099 int ret = TEST_SUCCESS; 4100 uint16_t i, j, dequeued; 4101 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4102 uint64_t start_time = 0, last_time = 0; 4103 4104 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4105 uint16_t enq = 0, deq = 0; 4106 bool first_time = true; 4107 last_time = 0; 4108 4109 if (unlikely(num_to_process - dequeued < burst_sz)) 4110 burst_sz = num_to_process - dequeued; 4111 4112 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4113 TEST_ASSERT_SUCCESS(ret, 4114 "rte_bbdev_enc_op_alloc_bulk() failed"); 4115 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4116 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4117 bufs->inputs, 4118 bufs->hard_outputs, 4119 ref_op); 4120 4121 /* Set counter to validate the ordering */ 4122 for (j = 0; j < burst_sz; ++j) 4123 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4124 4125 start_time = rte_rdtsc_precise(); 4126 4127 enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], 4128 burst_sz); 4129 TEST_ASSERT(enq == burst_sz, 4130 "Error enqueueing burst, expected %u, got %u", 4131 burst_sz, enq); 4132 4133 /* Dequeue */ 4134 do { 4135 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4136 &ops_deq[deq], burst_sz - deq); 4137 if (likely(first_time && (deq > 0))) { 4138 last_time += rte_rdtsc_precise() - start_time; 4139 first_time = false; 4140 } 4141 } while (unlikely(burst_sz != deq)); 4142 4143 *max_time = RTE_MAX(*max_time, last_time); 4144 *min_time = RTE_MIN(*min_time, last_time); 4145 *total_time += last_time; 4146 4147 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4148 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4149 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4150 } 4151 4152 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4153 dequeued += deq; 4154 } 4155 4156 return i; 4157 } 4158 4159 static int 4160 latency_test_ldpc_enc(struct rte_mempool *mempool, 4161 struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, 4162 uint16_t dev_id, uint16_t queue_id, 4163 const uint16_t num_to_process, uint16_t burst_sz, 4164 uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) 4165 { 4166 int ret = TEST_SUCCESS; 4167 uint16_t i, j, dequeued; 4168 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4169 uint64_t start_time = 0, last_time = 0; 4170 4171 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4172 uint16_t enq = 0, deq = 0; 4173 bool first_time = true; 4174 last_time = 0; 4175 4176 if (unlikely(num_to_process - dequeued < burst_sz)) 4177 burst_sz = num_to_process - dequeued; 4178 4179 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4180 TEST_ASSERT_SUCCESS(ret, 4181 "rte_bbdev_enc_op_alloc_bulk() failed"); 4182 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4183 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4184 bufs->inputs, 4185 bufs->hard_outputs, 4186 ref_op); 4187 4188 /* Set counter to validate the ordering */ 4189 for (j = 0; j < burst_sz; ++j) 4190 ops_enq[j]->opaque_data = (void *)(uintptr_t)j; 4191 4192 start_time = rte_rdtsc_precise(); 4193 4194 enq = rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4195 &ops_enq[enq], burst_sz); 4196 TEST_ASSERT(enq == burst_sz, 4197 "Error enqueueing burst, expected %u, got %u", 4198 burst_sz, enq); 4199 4200 /* Dequeue */ 4201 do { 4202 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4203 &ops_deq[deq], burst_sz - deq); 4204 if (likely(first_time && (deq > 0))) { 4205 last_time += rte_rdtsc_precise() - start_time; 4206 first_time = false; 4207 } 4208 } while (unlikely(burst_sz != deq)); 4209 4210 *max_time = RTE_MAX(*max_time, last_time); 4211 *min_time = RTE_MIN(*min_time, last_time); 4212 *total_time += last_time; 4213 4214 if (test_vector.op_type != RTE_BBDEV_OP_NONE) { 4215 ret = validate_enc_op(ops_deq, burst_sz, ref_op); 4216 TEST_ASSERT_SUCCESS(ret, "Validation failed!"); 4217 } 4218 4219 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4220 dequeued += deq; 4221 } 4222 4223 return i; 4224 } 4225 4226 /* Common function for running validation and latency test cases */ 4227 static int 4228 validation_latency_test(struct active_device *ad, 4229 struct test_op_params *op_params, bool latency_flag) 4230 { 4231 int iter; 4232 uint16_t burst_sz = op_params->burst_sz; 4233 const uint16_t num_to_process = op_params->num_to_process; 4234 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4235 const uint16_t queue_id = ad->queue_ids[0]; 4236 struct test_buffers *bufs = NULL; 4237 struct rte_bbdev_info info; 4238 uint64_t total_time, min_time, max_time; 4239 const char *op_type_str; 4240 4241 total_time = max_time = 0; 4242 min_time = UINT64_MAX; 4243 4244 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4245 "BURST_SIZE should be <= %u", MAX_BURST); 4246 4247 rte_bbdev_info_get(ad->dev_id, &info); 4248 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4249 4250 op_type_str = rte_bbdev_op_type_str(op_type); 4251 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4252 4253 printf("+ ------------------------------------------------------- +\n"); 4254 if (latency_flag) 4255 printf("== test: latency\ndev:"); 4256 else 4257 printf("== test: validation\ndev:"); 4258 printf("%s, burst size: %u, num ops: %u, op type: %s\n", 4259 info.dev_name, burst_sz, num_to_process, op_type_str); 4260 4261 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4262 iter = latency_test_dec(op_params->mp, bufs, 4263 op_params->ref_dec_op, op_params->vector_mask, 4264 ad->dev_id, queue_id, num_to_process, 4265 burst_sz, &total_time, &min_time, &max_time); 4266 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4267 iter = latency_test_ldpc_enc(op_params->mp, bufs, 4268 op_params->ref_enc_op, ad->dev_id, queue_id, 4269 num_to_process, burst_sz, &total_time, 4270 &min_time, &max_time); 4271 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4272 iter = latency_test_ldpc_dec(op_params->mp, bufs, 4273 op_params->ref_dec_op, op_params->vector_mask, 4274 ad->dev_id, queue_id, num_to_process, 4275 burst_sz, &total_time, &min_time, &max_time, 4276 latency_flag); 4277 else /* RTE_BBDEV_OP_TURBO_ENC */ 4278 iter = latency_test_enc(op_params->mp, bufs, 4279 op_params->ref_enc_op, 4280 ad->dev_id, queue_id, 4281 num_to_process, burst_sz, &total_time, 4282 &min_time, &max_time); 4283 4284 if (iter <= 0) 4285 return TEST_FAILED; 4286 4287 printf("Operation latency:\n" 4288 "\tavg: %lg cycles, %lg us\n" 4289 "\tmin: %lg cycles, %lg us\n" 4290 "\tmax: %lg cycles, %lg us\n", 4291 (double)total_time / (double)iter, 4292 (double)(total_time * 1000000) / (double)iter / 4293 (double)rte_get_tsc_hz(), (double)min_time, 4294 (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), 4295 (double)max_time, (double)(max_time * 1000000) / 4296 (double)rte_get_tsc_hz()); 4297 4298 return TEST_SUCCESS; 4299 } 4300 4301 static int 4302 latency_test(struct active_device *ad, struct test_op_params *op_params) 4303 { 4304 return validation_latency_test(ad, op_params, true); 4305 } 4306 4307 static int 4308 validation_test(struct active_device *ad, struct test_op_params *op_params) 4309 { 4310 return validation_latency_test(ad, op_params, false); 4311 } 4312 4313 #ifdef RTE_BBDEV_OFFLOAD_COST 4314 static int 4315 get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, 4316 struct rte_bbdev_stats *stats) 4317 { 4318 struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; 4319 struct rte_bbdev_stats *q_stats; 4320 4321 if (queue_id >= dev->data->num_queues) 4322 return -1; 4323 4324 q_stats = &dev->data->queues[queue_id].queue_stats; 4325 4326 stats->enqueued_count = q_stats->enqueued_count; 4327 stats->dequeued_count = q_stats->dequeued_count; 4328 stats->enqueue_err_count = q_stats->enqueue_err_count; 4329 stats->dequeue_err_count = q_stats->dequeue_err_count; 4330 stats->acc_offload_cycles = q_stats->acc_offload_cycles; 4331 4332 return 0; 4333 } 4334 4335 static int 4336 offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, 4337 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4338 uint16_t queue_id, const uint16_t num_to_process, 4339 uint16_t burst_sz, struct test_time_stats *time_st) 4340 { 4341 int i, dequeued, ret; 4342 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4343 uint64_t enq_start_time, deq_start_time; 4344 uint64_t enq_sw_last_time, deq_last_time; 4345 struct rte_bbdev_stats stats; 4346 4347 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4348 uint16_t enq = 0, deq = 0; 4349 4350 if (unlikely(num_to_process - dequeued < burst_sz)) 4351 burst_sz = num_to_process - dequeued; 4352 4353 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4354 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4355 copy_reference_dec_op(ops_enq, burst_sz, dequeued, 4356 bufs->inputs, 4357 bufs->hard_outputs, 4358 bufs->soft_outputs, 4359 ref_op); 4360 4361 /* Start time meas for enqueue function offload latency */ 4362 enq_start_time = rte_rdtsc_precise(); 4363 do { 4364 enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, 4365 &ops_enq[enq], burst_sz - enq); 4366 } while (unlikely(burst_sz != enq)); 4367 4368 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4369 TEST_ASSERT_SUCCESS(ret, 4370 "Failed to get stats for queue (%u) of device (%u)", 4371 queue_id, dev_id); 4372 4373 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - 4374 stats.acc_offload_cycles; 4375 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4376 enq_sw_last_time); 4377 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4378 enq_sw_last_time); 4379 time_st->enq_sw_total_time += enq_sw_last_time; 4380 4381 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4382 stats.acc_offload_cycles); 4383 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4384 stats.acc_offload_cycles); 4385 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4386 4387 /* give time for device to process ops */ 4388 rte_delay_us(200); 4389 4390 /* Start time meas for dequeue function offload latency */ 4391 deq_start_time = rte_rdtsc_precise(); 4392 /* Dequeue one operation */ 4393 do { 4394 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4395 &ops_deq[deq], 1); 4396 } while (unlikely(deq != 1)); 4397 4398 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4399 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4400 deq_last_time); 4401 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4402 deq_last_time); 4403 time_st->deq_total_time += deq_last_time; 4404 4405 /* Dequeue remaining operations if needed*/ 4406 while (burst_sz != deq) 4407 deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, 4408 &ops_deq[deq], burst_sz - deq); 4409 4410 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4411 dequeued += deq; 4412 } 4413 4414 return i; 4415 } 4416 4417 static int 4418 offload_latency_test_ldpc_dec(struct rte_mempool *mempool, 4419 struct test_buffers *bufs, 4420 struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, 4421 uint16_t queue_id, const uint16_t num_to_process, 4422 uint16_t burst_sz, struct test_time_stats *time_st) 4423 { 4424 int i, dequeued, ret; 4425 struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4426 uint64_t enq_start_time, deq_start_time; 4427 uint64_t enq_sw_last_time, deq_last_time; 4428 struct rte_bbdev_stats stats; 4429 bool extDdr = ldpc_cap_flags & 4430 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; 4431 4432 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4433 uint16_t enq = 0, deq = 0; 4434 4435 if (unlikely(num_to_process - dequeued < burst_sz)) 4436 burst_sz = num_to_process - dequeued; 4437 4438 rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); 4439 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4440 copy_reference_ldpc_dec_op(ops_enq, burst_sz, dequeued, 4441 bufs->inputs, 4442 bufs->hard_outputs, 4443 bufs->soft_outputs, 4444 bufs->harq_inputs, 4445 bufs->harq_outputs, 4446 ref_op); 4447 4448 if (extDdr) 4449 preload_harq_ddr(dev_id, queue_id, ops_enq, 4450 burst_sz, true); 4451 4452 /* Start time meas for enqueue function offload latency */ 4453 enq_start_time = rte_rdtsc_precise(); 4454 do { 4455 enq += rte_bbdev_enqueue_ldpc_dec_ops(dev_id, queue_id, 4456 &ops_enq[enq], burst_sz - enq); 4457 } while (unlikely(burst_sz != enq)); 4458 4459 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4460 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4461 TEST_ASSERT_SUCCESS(ret, 4462 "Failed to get stats for queue (%u) of device (%u)", 4463 queue_id, dev_id); 4464 4465 enq_sw_last_time -= stats.acc_offload_cycles; 4466 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4467 enq_sw_last_time); 4468 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4469 enq_sw_last_time); 4470 time_st->enq_sw_total_time += enq_sw_last_time; 4471 4472 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4473 stats.acc_offload_cycles); 4474 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4475 stats.acc_offload_cycles); 4476 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4477 4478 /* give time for device to process ops */ 4479 rte_delay_us(200); 4480 4481 /* Start time meas for dequeue function offload latency */ 4482 deq_start_time = rte_rdtsc_precise(); 4483 /* Dequeue one operation */ 4484 do { 4485 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4486 &ops_deq[deq], 1); 4487 } while (unlikely(deq != 1)); 4488 4489 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4490 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4491 deq_last_time); 4492 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4493 deq_last_time); 4494 time_st->deq_total_time += deq_last_time; 4495 4496 /* Dequeue remaining operations if needed*/ 4497 while (burst_sz != deq) 4498 deq += rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, 4499 &ops_deq[deq], burst_sz - deq); 4500 4501 if (extDdr) { 4502 /* Read loopback is not thread safe */ 4503 retrieve_harq_ddr(dev_id, queue_id, ops_enq, burst_sz); 4504 } 4505 4506 rte_bbdev_dec_op_free_bulk(ops_enq, deq); 4507 dequeued += deq; 4508 } 4509 4510 return i; 4511 } 4512 4513 static int 4514 offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, 4515 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4516 uint16_t queue_id, const uint16_t num_to_process, 4517 uint16_t burst_sz, struct test_time_stats *time_st) 4518 { 4519 int i, dequeued, ret; 4520 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4521 uint64_t enq_start_time, deq_start_time; 4522 uint64_t enq_sw_last_time, deq_last_time; 4523 struct rte_bbdev_stats stats; 4524 4525 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4526 uint16_t enq = 0, deq = 0; 4527 4528 if (unlikely(num_to_process - dequeued < burst_sz)) 4529 burst_sz = num_to_process - dequeued; 4530 4531 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4532 TEST_ASSERT_SUCCESS(ret, 4533 "rte_bbdev_enc_op_alloc_bulk() failed"); 4534 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4535 copy_reference_enc_op(ops_enq, burst_sz, dequeued, 4536 bufs->inputs, 4537 bufs->hard_outputs, 4538 ref_op); 4539 4540 /* Start time meas for enqueue function offload latency */ 4541 enq_start_time = rte_rdtsc_precise(); 4542 do { 4543 enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, 4544 &ops_enq[enq], burst_sz - enq); 4545 } while (unlikely(burst_sz != enq)); 4546 4547 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4548 4549 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4550 TEST_ASSERT_SUCCESS(ret, 4551 "Failed to get stats for queue (%u) of device (%u)", 4552 queue_id, dev_id); 4553 enq_sw_last_time -= stats.acc_offload_cycles; 4554 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4555 enq_sw_last_time); 4556 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4557 enq_sw_last_time); 4558 time_st->enq_sw_total_time += enq_sw_last_time; 4559 4560 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4561 stats.acc_offload_cycles); 4562 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4563 stats.acc_offload_cycles); 4564 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4565 4566 /* give time for device to process ops */ 4567 rte_delay_us(200); 4568 4569 /* Start time meas for dequeue function offload latency */ 4570 deq_start_time = rte_rdtsc_precise(); 4571 /* Dequeue one operation */ 4572 do { 4573 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4574 &ops_deq[deq], 1); 4575 } while (unlikely(deq != 1)); 4576 4577 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4578 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4579 deq_last_time); 4580 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4581 deq_last_time); 4582 time_st->deq_total_time += deq_last_time; 4583 4584 while (burst_sz != deq) 4585 deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, 4586 &ops_deq[deq], burst_sz - deq); 4587 4588 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4589 dequeued += deq; 4590 } 4591 4592 return i; 4593 } 4594 4595 static int 4596 offload_latency_test_ldpc_enc(struct rte_mempool *mempool, 4597 struct test_buffers *bufs, 4598 struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, 4599 uint16_t queue_id, const uint16_t num_to_process, 4600 uint16_t burst_sz, struct test_time_stats *time_st) 4601 { 4602 int i, dequeued, ret; 4603 struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; 4604 uint64_t enq_start_time, deq_start_time; 4605 uint64_t enq_sw_last_time, deq_last_time; 4606 struct rte_bbdev_stats stats; 4607 4608 for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { 4609 uint16_t enq = 0, deq = 0; 4610 4611 if (unlikely(num_to_process - dequeued < burst_sz)) 4612 burst_sz = num_to_process - dequeued; 4613 4614 ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); 4615 TEST_ASSERT_SUCCESS(ret, 4616 "rte_bbdev_enc_op_alloc_bulk() failed"); 4617 if (test_vector.op_type != RTE_BBDEV_OP_NONE) 4618 copy_reference_ldpc_enc_op(ops_enq, burst_sz, dequeued, 4619 bufs->inputs, 4620 bufs->hard_outputs, 4621 ref_op); 4622 4623 /* Start time meas for enqueue function offload latency */ 4624 enq_start_time = rte_rdtsc_precise(); 4625 do { 4626 enq += rte_bbdev_enqueue_ldpc_enc_ops(dev_id, queue_id, 4627 &ops_enq[enq], burst_sz - enq); 4628 } while (unlikely(burst_sz != enq)); 4629 4630 enq_sw_last_time = rte_rdtsc_precise() - enq_start_time; 4631 ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); 4632 TEST_ASSERT_SUCCESS(ret, 4633 "Failed to get stats for queue (%u) of device (%u)", 4634 queue_id, dev_id); 4635 4636 enq_sw_last_time -= stats.acc_offload_cycles; 4637 time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, 4638 enq_sw_last_time); 4639 time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, 4640 enq_sw_last_time); 4641 time_st->enq_sw_total_time += enq_sw_last_time; 4642 4643 time_st->enq_acc_max_time = RTE_MAX(time_st->enq_acc_max_time, 4644 stats.acc_offload_cycles); 4645 time_st->enq_acc_min_time = RTE_MIN(time_st->enq_acc_min_time, 4646 stats.acc_offload_cycles); 4647 time_st->enq_acc_total_time += stats.acc_offload_cycles; 4648 4649 /* give time for device to process ops */ 4650 rte_delay_us(200); 4651 4652 /* Start time meas for dequeue function offload latency */ 4653 deq_start_time = rte_rdtsc_precise(); 4654 /* Dequeue one operation */ 4655 do { 4656 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4657 &ops_deq[deq], 1); 4658 } while (unlikely(deq != 1)); 4659 4660 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4661 time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, 4662 deq_last_time); 4663 time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, 4664 deq_last_time); 4665 time_st->deq_total_time += deq_last_time; 4666 4667 while (burst_sz != deq) 4668 deq += rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, 4669 &ops_deq[deq], burst_sz - deq); 4670 4671 rte_bbdev_enc_op_free_bulk(ops_enq, deq); 4672 dequeued += deq; 4673 } 4674 4675 return i; 4676 } 4677 #endif 4678 4679 static int 4680 offload_cost_test(struct active_device *ad, 4681 struct test_op_params *op_params) 4682 { 4683 #ifndef RTE_BBDEV_OFFLOAD_COST 4684 RTE_SET_USED(ad); 4685 RTE_SET_USED(op_params); 4686 printf("Offload latency test is disabled.\n"); 4687 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4688 return TEST_SKIPPED; 4689 #else 4690 int iter; 4691 uint16_t burst_sz = op_params->burst_sz; 4692 const uint16_t num_to_process = op_params->num_to_process; 4693 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4694 const uint16_t queue_id = ad->queue_ids[0]; 4695 struct test_buffers *bufs = NULL; 4696 struct rte_bbdev_info info; 4697 const char *op_type_str; 4698 struct test_time_stats time_st; 4699 4700 memset(&time_st, 0, sizeof(struct test_time_stats)); 4701 time_st.enq_sw_min_time = UINT64_MAX; 4702 time_st.enq_acc_min_time = UINT64_MAX; 4703 time_st.deq_min_time = UINT64_MAX; 4704 4705 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4706 "BURST_SIZE should be <= %u", MAX_BURST); 4707 4708 rte_bbdev_info_get(ad->dev_id, &info); 4709 bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; 4710 4711 op_type_str = rte_bbdev_op_type_str(op_type); 4712 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4713 4714 printf("+ ------------------------------------------------------- +\n"); 4715 printf("== test: offload latency test\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4716 info.dev_name, burst_sz, num_to_process, op_type_str); 4717 4718 if (op_type == RTE_BBDEV_OP_TURBO_DEC) 4719 iter = offload_latency_test_dec(op_params->mp, bufs, 4720 op_params->ref_dec_op, ad->dev_id, queue_id, 4721 num_to_process, burst_sz, &time_st); 4722 else if (op_type == RTE_BBDEV_OP_TURBO_ENC) 4723 iter = offload_latency_test_enc(op_params->mp, bufs, 4724 op_params->ref_enc_op, ad->dev_id, queue_id, 4725 num_to_process, burst_sz, &time_st); 4726 else if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4727 iter = offload_latency_test_ldpc_enc(op_params->mp, bufs, 4728 op_params->ref_enc_op, ad->dev_id, queue_id, 4729 num_to_process, burst_sz, &time_st); 4730 else if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4731 iter = offload_latency_test_ldpc_dec(op_params->mp, bufs, 4732 op_params->ref_dec_op, ad->dev_id, queue_id, 4733 num_to_process, burst_sz, &time_st); 4734 else 4735 iter = offload_latency_test_enc(op_params->mp, bufs, 4736 op_params->ref_enc_op, ad->dev_id, queue_id, 4737 num_to_process, burst_sz, &time_st); 4738 4739 if (iter <= 0) 4740 return TEST_FAILED; 4741 4742 printf("Enqueue driver offload cost latency:\n" 4743 "\tavg: %lg cycles, %lg us\n" 4744 "\tmin: %lg cycles, %lg us\n" 4745 "\tmax: %lg cycles, %lg us\n" 4746 "Enqueue accelerator offload cost latency:\n" 4747 "\tavg: %lg cycles, %lg us\n" 4748 "\tmin: %lg cycles, %lg us\n" 4749 "\tmax: %lg cycles, %lg us\n", 4750 (double)time_st.enq_sw_total_time / (double)iter, 4751 (double)(time_st.enq_sw_total_time * 1000000) / 4752 (double)iter / (double)rte_get_tsc_hz(), 4753 (double)time_st.enq_sw_min_time, 4754 (double)(time_st.enq_sw_min_time * 1000000) / 4755 rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, 4756 (double)(time_st.enq_sw_max_time * 1000000) / 4757 rte_get_tsc_hz(), (double)time_st.enq_acc_total_time / 4758 (double)iter, 4759 (double)(time_st.enq_acc_total_time * 1000000) / 4760 (double)iter / (double)rte_get_tsc_hz(), 4761 (double)time_st.enq_acc_min_time, 4762 (double)(time_st.enq_acc_min_time * 1000000) / 4763 rte_get_tsc_hz(), (double)time_st.enq_acc_max_time, 4764 (double)(time_st.enq_acc_max_time * 1000000) / 4765 rte_get_tsc_hz()); 4766 4767 printf("Dequeue offload cost latency - one op:\n" 4768 "\tavg: %lg cycles, %lg us\n" 4769 "\tmin: %lg cycles, %lg us\n" 4770 "\tmax: %lg cycles, %lg us\n", 4771 (double)time_st.deq_total_time / (double)iter, 4772 (double)(time_st.deq_total_time * 1000000) / 4773 (double)iter / (double)rte_get_tsc_hz(), 4774 (double)time_st.deq_min_time, 4775 (double)(time_st.deq_min_time * 1000000) / 4776 rte_get_tsc_hz(), (double)time_st.deq_max_time, 4777 (double)(time_st.deq_max_time * 1000000) / 4778 rte_get_tsc_hz()); 4779 4780 struct rte_bbdev_stats stats = {0}; 4781 get_bbdev_queue_stats(ad->dev_id, queue_id, &stats); 4782 if (op_type != RTE_BBDEV_OP_LDPC_DEC) { 4783 TEST_ASSERT_SUCCESS(stats.enqueued_count != num_to_process, 4784 "Mismatch in enqueue count %10"PRIu64" %d", 4785 stats.enqueued_count, num_to_process); 4786 TEST_ASSERT_SUCCESS(stats.dequeued_count != num_to_process, 4787 "Mismatch in dequeue count %10"PRIu64" %d", 4788 stats.dequeued_count, num_to_process); 4789 } 4790 TEST_ASSERT_SUCCESS(stats.enqueue_err_count != 0, 4791 "Enqueue count Error %10"PRIu64"", 4792 stats.enqueue_err_count); 4793 TEST_ASSERT_SUCCESS(stats.dequeue_err_count != 0, 4794 "Dequeue count Error (%10"PRIu64"", 4795 stats.dequeue_err_count); 4796 4797 return TEST_SUCCESS; 4798 #endif 4799 } 4800 4801 #ifdef RTE_BBDEV_OFFLOAD_COST 4802 static int 4803 offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, 4804 const uint16_t num_to_process, uint16_t burst_sz, 4805 uint64_t *deq_total_time, uint64_t *deq_min_time, 4806 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4807 { 4808 int i, deq_total; 4809 struct rte_bbdev_dec_op *ops[MAX_BURST]; 4810 uint64_t deq_start_time, deq_last_time; 4811 4812 /* Test deq offload latency from an empty queue */ 4813 4814 for (i = 0, deq_total = 0; deq_total < num_to_process; 4815 ++i, deq_total += burst_sz) { 4816 deq_start_time = rte_rdtsc_precise(); 4817 4818 if (unlikely(num_to_process - deq_total < burst_sz)) 4819 burst_sz = num_to_process - deq_total; 4820 if (op_type == RTE_BBDEV_OP_LDPC_DEC) 4821 rte_bbdev_dequeue_ldpc_dec_ops(dev_id, queue_id, ops, 4822 burst_sz); 4823 else 4824 rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, 4825 burst_sz); 4826 4827 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4828 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4829 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4830 *deq_total_time += deq_last_time; 4831 } 4832 4833 return i; 4834 } 4835 4836 static int 4837 offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, 4838 const uint16_t num_to_process, uint16_t burst_sz, 4839 uint64_t *deq_total_time, uint64_t *deq_min_time, 4840 uint64_t *deq_max_time, const enum rte_bbdev_op_type op_type) 4841 { 4842 int i, deq_total; 4843 struct rte_bbdev_enc_op *ops[MAX_BURST]; 4844 uint64_t deq_start_time, deq_last_time; 4845 4846 /* Test deq offload latency from an empty queue */ 4847 for (i = 0, deq_total = 0; deq_total < num_to_process; 4848 ++i, deq_total += burst_sz) { 4849 deq_start_time = rte_rdtsc_precise(); 4850 4851 if (unlikely(num_to_process - deq_total < burst_sz)) 4852 burst_sz = num_to_process - deq_total; 4853 if (op_type == RTE_BBDEV_OP_LDPC_ENC) 4854 rte_bbdev_dequeue_ldpc_enc_ops(dev_id, queue_id, ops, 4855 burst_sz); 4856 else 4857 rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, 4858 burst_sz); 4859 4860 deq_last_time = rte_rdtsc_precise() - deq_start_time; 4861 *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); 4862 *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); 4863 *deq_total_time += deq_last_time; 4864 } 4865 4866 return i; 4867 } 4868 4869 #endif 4870 4871 static int 4872 offload_latency_empty_q_test(struct active_device *ad, 4873 struct test_op_params *op_params) 4874 { 4875 #ifndef RTE_BBDEV_OFFLOAD_COST 4876 RTE_SET_USED(ad); 4877 RTE_SET_USED(op_params); 4878 printf("Offload latency empty dequeue test is disabled.\n"); 4879 printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); 4880 return TEST_SKIPPED; 4881 #else 4882 int iter; 4883 uint64_t deq_total_time, deq_min_time, deq_max_time; 4884 uint16_t burst_sz = op_params->burst_sz; 4885 const uint16_t num_to_process = op_params->num_to_process; 4886 const enum rte_bbdev_op_type op_type = test_vector.op_type; 4887 const uint16_t queue_id = ad->queue_ids[0]; 4888 struct rte_bbdev_info info; 4889 const char *op_type_str; 4890 4891 deq_total_time = deq_max_time = 0; 4892 deq_min_time = UINT64_MAX; 4893 4894 TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), 4895 "BURST_SIZE should be <= %u", MAX_BURST); 4896 4897 rte_bbdev_info_get(ad->dev_id, &info); 4898 4899 op_type_str = rte_bbdev_op_type_str(op_type); 4900 TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); 4901 4902 printf("+ ------------------------------------------------------- +\n"); 4903 printf("== test: offload latency empty dequeue\ndev: %s, burst size: %u, num ops: %u, op type: %s\n", 4904 info.dev_name, burst_sz, num_to_process, op_type_str); 4905 4906 if (op_type == RTE_BBDEV_OP_TURBO_DEC || 4907 op_type == RTE_BBDEV_OP_LDPC_DEC) 4908 iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, 4909 num_to_process, burst_sz, &deq_total_time, 4910 &deq_min_time, &deq_max_time, op_type); 4911 else 4912 iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, 4913 num_to_process, burst_sz, &deq_total_time, 4914 &deq_min_time, &deq_max_time, op_type); 4915 4916 if (iter <= 0) 4917 return TEST_FAILED; 4918 4919 printf("Empty dequeue offload:\n" 4920 "\tavg: %lg cycles, %lg us\n" 4921 "\tmin: %lg cycles, %lg us\n" 4922 "\tmax: %lg cycles, %lg us\n", 4923 (double)deq_total_time / (double)iter, 4924 (double)(deq_total_time * 1000000) / (double)iter / 4925 (double)rte_get_tsc_hz(), (double)deq_min_time, 4926 (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), 4927 (double)deq_max_time, (double)(deq_max_time * 1000000) / 4928 rte_get_tsc_hz()); 4929 4930 return TEST_SUCCESS; 4931 #endif 4932 } 4933 4934 static int 4935 bler_tc(void) 4936 { 4937 return run_test_case(bler_test); 4938 } 4939 4940 static int 4941 throughput_tc(void) 4942 { 4943 return run_test_case(throughput_test); 4944 } 4945 4946 static int 4947 offload_cost_tc(void) 4948 { 4949 return run_test_case(offload_cost_test); 4950 } 4951 4952 static int 4953 offload_latency_empty_q_tc(void) 4954 { 4955 return run_test_case(offload_latency_empty_q_test); 4956 } 4957 4958 static int 4959 latency_tc(void) 4960 { 4961 return run_test_case(latency_test); 4962 } 4963 4964 static int 4965 validation_tc(void) 4966 { 4967 return run_test_case(validation_test); 4968 } 4969 4970 static int 4971 interrupt_tc(void) 4972 { 4973 return run_test_case(throughput_test); 4974 } 4975 4976 static struct unit_test_suite bbdev_bler_testsuite = { 4977 .suite_name = "BBdev BLER Tests", 4978 .setup = testsuite_setup, 4979 .teardown = testsuite_teardown, 4980 .unit_test_cases = { 4981 TEST_CASE_ST(ut_setup, ut_teardown, bler_tc), 4982 TEST_CASES_END() /**< NULL terminate unit test array */ 4983 } 4984 }; 4985 4986 static struct unit_test_suite bbdev_throughput_testsuite = { 4987 .suite_name = "BBdev Throughput Tests", 4988 .setup = testsuite_setup, 4989 .teardown = testsuite_teardown, 4990 .unit_test_cases = { 4991 TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), 4992 TEST_CASES_END() /**< NULL terminate unit test array */ 4993 } 4994 }; 4995 4996 static struct unit_test_suite bbdev_validation_testsuite = { 4997 .suite_name = "BBdev Validation Tests", 4998 .setup = testsuite_setup, 4999 .teardown = testsuite_teardown, 5000 .unit_test_cases = { 5001 TEST_CASE_ST(ut_setup, ut_teardown, validation_tc), 5002 TEST_CASES_END() /**< NULL terminate unit test array */ 5003 } 5004 }; 5005 5006 static struct unit_test_suite bbdev_latency_testsuite = { 5007 .suite_name = "BBdev Latency Tests", 5008 .setup = testsuite_setup, 5009 .teardown = testsuite_teardown, 5010 .unit_test_cases = { 5011 TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), 5012 TEST_CASES_END() /**< NULL terminate unit test array */ 5013 } 5014 }; 5015 5016 static struct unit_test_suite bbdev_offload_cost_testsuite = { 5017 .suite_name = "BBdev Offload Cost Tests", 5018 .setup = testsuite_setup, 5019 .teardown = testsuite_teardown, 5020 .unit_test_cases = { 5021 TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), 5022 TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), 5023 TEST_CASES_END() /**< NULL terminate unit test array */ 5024 } 5025 }; 5026 5027 static struct unit_test_suite bbdev_interrupt_testsuite = { 5028 .suite_name = "BBdev Interrupt Tests", 5029 .setup = interrupt_testsuite_setup, 5030 .teardown = testsuite_teardown, 5031 .unit_test_cases = { 5032 TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), 5033 TEST_CASES_END() /**< NULL terminate unit test array */ 5034 } 5035 }; 5036 5037 REGISTER_TEST_COMMAND(bler, bbdev_bler_testsuite); 5038 REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); 5039 REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); 5040 REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); 5041 REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); 5042 REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); 5043