1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_eal.h>
7 #include <rte_log.h>
8 #include <rte_cycles.h>
9 #include "rte_spinlock.h"
10 #include <rte_compressdev.h>
11
12 #include "comp_perf_test_cyclecount.h"
13
14 struct cperf_cyclecount_ctx {
15 struct cperf_verify_ctx ver;
16
17 uint32_t ops_enq_retries;
18 uint32_t ops_deq_retries;
19
20 uint64_t duration_op;
21 uint64_t duration_enq;
22 uint64_t duration_deq;
23 };
24
25 void
cperf_cyclecount_test_destructor(void * arg)26 cperf_cyclecount_test_destructor(void *arg)
27 {
28 struct cperf_cyclecount_ctx *ctx = arg;
29
30 if (arg) {
31 comp_perf_free_memory(ctx->ver.options, &ctx->ver.mem);
32 rte_free(arg);
33 }
34 }
35
36 void *
cperf_cyclecount_test_constructor(uint8_t dev_id,uint16_t qp_id,struct comp_test_data * options)37 cperf_cyclecount_test_constructor(uint8_t dev_id, uint16_t qp_id,
38 struct comp_test_data *options)
39 {
40 struct cperf_cyclecount_ctx *ctx = NULL;
41
42 ctx = rte_malloc(NULL, sizeof(struct cperf_cyclecount_ctx), 0);
43
44 if (ctx == NULL)
45 return NULL;
46
47 ctx->ver.mem.dev_id = dev_id;
48 ctx->ver.mem.qp_id = qp_id;
49 ctx->ver.options = options;
50 ctx->ver.silent = 1; /* ver. part will be silent */
51
52 if (!comp_perf_allocate_memory(ctx->ver.options, &ctx->ver.mem)
53 && !prepare_bufs(ctx->ver.options, &ctx->ver.mem))
54 return ctx;
55
56 cperf_cyclecount_test_destructor(ctx);
57 return NULL;
58 }
59
60 static int
cperf_cyclecount_op_setup(struct rte_comp_op ** ops,struct cperf_cyclecount_ctx * ctx,struct rte_mbuf ** input_bufs,struct rte_mbuf ** output_bufs,void * priv_xform,uint32_t out_seg_sz)61 cperf_cyclecount_op_setup(struct rte_comp_op **ops,
62 struct cperf_cyclecount_ctx *ctx,
63 struct rte_mbuf **input_bufs,
64 struct rte_mbuf **output_bufs,
65 void *priv_xform,
66 uint32_t out_seg_sz)
67 {
68 struct comp_test_data *test_data = ctx->ver.options;
69 struct cperf_mem_resources *mem = &ctx->ver.mem;
70
71 uint32_t i, iter, num_iter;
72 int res = 0;
73 uint16_t ops_needed;
74
75 num_iter = test_data->num_iter;
76
77 for (iter = 0; iter < num_iter; iter++) {
78 uint32_t remaining_ops = mem->total_bufs;
79 uint32_t total_enq_ops = 0;
80 uint16_t num_enq = 0;
81 uint16_t num_deq = 0;
82
83 while (remaining_ops > 0) {
84 uint16_t num_ops = RTE_MIN(remaining_ops,
85 test_data->burst_sz);
86 ops_needed = num_ops;
87
88 /* Allocate compression operations */
89 if (ops_needed && rte_mempool_get_bulk(
90 mem->op_pool,
91 (void **)ops,
92 ops_needed) != 0) {
93 RTE_LOG(ERR, USER1,
94 "Cyclecount: could not allocate enough operations\n");
95 res = -1;
96 goto end;
97 }
98
99 for (i = 0; i < ops_needed; i++) {
100
101 /* Calculate next buffer to attach */
102 /* to operation */
103 uint32_t buf_id = total_enq_ops + i;
104 uint16_t op_id = i;
105
106 /* Reset all data in output buffers */
107 struct rte_mbuf *m = output_bufs[buf_id];
108
109 m->pkt_len = out_seg_sz * m->nb_segs;
110 while (m) {
111 m->data_len = m->buf_len - m->data_off;
112 m = m->next;
113 }
114 ops[op_id]->m_src = input_bufs[buf_id];
115 ops[op_id]->m_dst = output_bufs[buf_id];
116 ops[op_id]->src.offset = 0;
117 ops[op_id]->src.length =
118 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
119 ops[op_id]->dst.offset = 0;
120 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
121 ops[op_id]->input_chksum = buf_id;
122 ops[op_id]->private_xform = priv_xform;
123 }
124
125 /* E N Q U E U I N G */
126 /* assuming that all ops are enqueued */
127 /* instead of the real enqueue operation */
128 num_enq = num_ops;
129
130 remaining_ops -= num_enq;
131 total_enq_ops += num_enq;
132
133 /* D E Q U E U I N G */
134 /* assuming that all ops dequeued */
135 /* instead of the real dequeue operation */
136 num_deq = num_ops;
137
138 rte_mempool_put_bulk(mem->op_pool,
139 (void **)ops, num_deq);
140 }
141 }
142 return res;
143 end:
144 rte_mempool_put_bulk(mem->op_pool, (void **)ops, ops_needed);
145 rte_free(ops);
146
147 return res;
148 }
149
150 static int
main_loop(struct cperf_cyclecount_ctx * ctx,enum rte_comp_xform_type type)151 main_loop(struct cperf_cyclecount_ctx *ctx, enum rte_comp_xform_type type)
152 {
153 struct comp_test_data *test_data = ctx->ver.options;
154 struct cperf_mem_resources *mem = &ctx->ver.mem;
155 uint8_t dev_id = mem->dev_id;
156 uint32_t i, iter, num_iter;
157 struct rte_comp_op **ops, **deq_ops;
158 void *priv_xform = NULL;
159 struct rte_comp_xform xform;
160 struct rte_mbuf **input_bufs, **output_bufs;
161 int ret, res = 0;
162 int allocated = 0;
163 uint32_t out_seg_sz;
164
165 uint64_t tsc_start, tsc_end, tsc_duration;
166
167 if (test_data == NULL || !test_data->burst_sz) {
168 RTE_LOG(ERR, USER1, "Unknown burst size\n");
169 return -1;
170 }
171 ctx->duration_enq = 0;
172 ctx->duration_deq = 0;
173 ctx->ops_enq_retries = 0;
174 ctx->ops_deq_retries = 0;
175
176 /* one array for both enqueue and dequeue */
177 ops = rte_zmalloc_socket(NULL,
178 (test_data->burst_sz + mem->total_bufs) *
179 sizeof(struct rte_comp_op *),
180 0, rte_socket_id());
181
182 if (ops == NULL) {
183 RTE_LOG(ERR, USER1,
184 "Can't allocate memory for ops structures\n");
185 return -1;
186 }
187
188 deq_ops = &ops[test_data->burst_sz];
189
190 if (type == RTE_COMP_COMPRESS) {
191 xform = (struct rte_comp_xform) {
192 .type = RTE_COMP_COMPRESS,
193 .compress = {
194 .algo = RTE_COMP_ALGO_DEFLATE,
195 .deflate.huffman = test_data->huffman_enc,
196 .level = test_data->level,
197 .window_size = test_data->window_sz,
198 .chksum = RTE_COMP_CHECKSUM_NONE,
199 .hash_algo = RTE_COMP_HASH_ALGO_NONE
200 }
201 };
202 input_bufs = mem->decomp_bufs;
203 output_bufs = mem->comp_bufs;
204 out_seg_sz = test_data->out_seg_sz;
205 } else {
206 xform = (struct rte_comp_xform) {
207 .type = RTE_COMP_DECOMPRESS,
208 .decompress = {
209 .algo = RTE_COMP_ALGO_DEFLATE,
210 .chksum = RTE_COMP_CHECKSUM_NONE,
211 .window_size = test_data->window_sz,
212 .hash_algo = RTE_COMP_HASH_ALGO_NONE
213 }
214 };
215 input_bufs = mem->comp_bufs;
216 output_bufs = mem->decomp_bufs;
217 out_seg_sz = test_data->seg_sz;
218 }
219
220 /* Create private xform */
221 if (rte_compressdev_private_xform_create(dev_id, &xform,
222 &priv_xform) < 0) {
223 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
224 res = -1;
225 goto end;
226 }
227
228 tsc_start = rte_rdtsc_precise();
229 ret = cperf_cyclecount_op_setup(ops,
230 ctx,
231 input_bufs,
232 output_bufs,
233 priv_xform,
234 out_seg_sz);
235
236 tsc_end = rte_rdtsc_precise();
237
238 /* ret value check postponed a bit to cancel extra 'if' bias */
239 if (ret < 0) {
240 RTE_LOG(ERR, USER1, "Setup function failed\n");
241 res = -1;
242 goto end;
243 }
244
245 tsc_duration = tsc_end - tsc_start;
246 ctx->duration_op = tsc_duration;
247
248 num_iter = test_data->num_iter;
249 for (iter = 0; iter < num_iter; iter++) {
250 uint32_t total_ops = mem->total_bufs;
251 uint32_t remaining_ops = mem->total_bufs;
252 uint32_t total_deq_ops = 0;
253 uint32_t total_enq_ops = 0;
254 uint16_t ops_unused = 0;
255 uint16_t num_enq = 0;
256 uint16_t num_deq = 0;
257
258 while (remaining_ops > 0) {
259 uint16_t num_ops = RTE_MIN(remaining_ops,
260 test_data->burst_sz);
261 uint16_t ops_needed = num_ops - ops_unused;
262
263 /*
264 * Move the unused operations from the previous
265 * enqueue_burst call to the front, to maintain order
266 */
267 if ((ops_unused > 0) && (num_enq > 0)) {
268 size_t nb_b_to_mov =
269 ops_unused * sizeof(struct rte_comp_op *);
270
271 memmove(ops, &ops[num_enq], nb_b_to_mov);
272 }
273
274 /* Allocate compression operations */
275 if (ops_needed && rte_mempool_get_bulk(
276 mem->op_pool,
277 (void **)&ops[ops_unused],
278 ops_needed) != 0) {
279 RTE_LOG(ERR, USER1,
280 "Could not allocate enough operations\n");
281 res = -1;
282 goto end;
283 }
284 allocated += ops_needed;
285
286 for (i = 0; i < ops_needed; i++) {
287 /*
288 * Calculate next buffer to attach to operation
289 */
290 uint32_t buf_id = total_enq_ops + i +
291 ops_unused;
292 uint16_t op_id = ops_unused + i;
293 /* Reset all data in output buffers */
294 struct rte_mbuf *m = output_bufs[buf_id];
295
296 m->pkt_len = out_seg_sz * m->nb_segs;
297 while (m) {
298 m->data_len = m->buf_len - m->data_off;
299 m = m->next;
300 }
301 ops[op_id]->m_src = input_bufs[buf_id];
302 ops[op_id]->m_dst = output_bufs[buf_id];
303 ops[op_id]->src.offset = 0;
304 ops[op_id]->src.length =
305 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
306 ops[op_id]->dst.offset = 0;
307 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
308 ops[op_id]->input_chksum = buf_id;
309 ops[op_id]->private_xform = priv_xform;
310 }
311
312 if (unlikely(test_data->perf_comp_force_stop))
313 goto end;
314
315 tsc_start = rte_rdtsc_precise();
316 num_enq = rte_compressdev_enqueue_burst(dev_id,
317 mem->qp_id, ops,
318 num_ops);
319 tsc_end = rte_rdtsc_precise();
320 tsc_duration = tsc_end - tsc_start;
321 ctx->duration_enq += tsc_duration;
322
323 if (num_enq < num_ops)
324 ctx->ops_enq_retries++;
325
326 if (test_data->cyclecount_delay)
327 rte_delay_us_block(test_data->cyclecount_delay);
328
329 if (num_enq == 0) {
330 struct rte_compressdev_stats stats;
331
332 rte_compressdev_stats_get(dev_id, &stats);
333 if (stats.enqueue_err_count) {
334 res = -1;
335 goto end;
336 }
337 }
338
339 ops_unused = num_ops - num_enq;
340 remaining_ops -= num_enq;
341 total_enq_ops += num_enq;
342
343 tsc_start = rte_rdtsc_precise();
344 num_deq = rte_compressdev_dequeue_burst(dev_id,
345 mem->qp_id,
346 deq_ops,
347 allocated);
348 tsc_end = rte_rdtsc_precise();
349 tsc_duration = tsc_end - tsc_start;
350 ctx->duration_deq += tsc_duration;
351
352 if (num_deq < allocated)
353 ctx->ops_deq_retries++;
354
355 total_deq_ops += num_deq;
356
357 if (iter == num_iter - 1) {
358 for (i = 0; i < num_deq; i++) {
359 struct rte_comp_op *op = deq_ops[i];
360
361 if (op->status !=
362 RTE_COMP_OP_STATUS_SUCCESS) {
363 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
364 goto end;
365 }
366
367 struct rte_mbuf *m = op->m_dst;
368
369 m->pkt_len = op->produced;
370 uint32_t remaining_data = op->produced;
371 uint16_t data_to_append;
372
373 while (remaining_data > 0) {
374 data_to_append =
375 RTE_MIN(remaining_data,
376 out_seg_sz);
377 m->data_len = data_to_append;
378 remaining_data -=
379 data_to_append;
380 m = m->next;
381 }
382 }
383 }
384 rte_mempool_put_bulk(mem->op_pool,
385 (void **)deq_ops, num_deq);
386 allocated -= num_deq;
387 }
388
389 /* Dequeue the last operations */
390 while (total_deq_ops < total_ops) {
391 if (unlikely(test_data->perf_comp_force_stop))
392 goto end;
393
394 tsc_start = rte_rdtsc_precise();
395 num_deq = rte_compressdev_dequeue_burst(dev_id,
396 mem->qp_id,
397 deq_ops,
398 test_data->burst_sz);
399 tsc_end = rte_rdtsc_precise();
400 tsc_duration = tsc_end - tsc_start;
401 ctx->duration_deq += tsc_duration;
402 ctx->ops_deq_retries++;
403
404 if (num_deq == 0) {
405 struct rte_compressdev_stats stats;
406
407 rte_compressdev_stats_get(dev_id, &stats);
408 if (stats.dequeue_err_count) {
409 res = -1;
410 goto end;
411 }
412 }
413 total_deq_ops += num_deq;
414
415 if (iter == num_iter - 1) {
416 for (i = 0; i < num_deq; i++) {
417 struct rte_comp_op *op = deq_ops[i];
418
419 if (op->status !=
420 RTE_COMP_OP_STATUS_SUCCESS) {
421 RTE_LOG(ERR, USER1, "Some operations were not successful\n");
422 goto end;
423 }
424
425 struct rte_mbuf *m = op->m_dst;
426
427 m->pkt_len = op->produced;
428 uint32_t remaining_data = op->produced;
429 uint16_t data_to_append;
430
431 while (remaining_data > 0) {
432 data_to_append =
433 RTE_MIN(remaining_data,
434 out_seg_sz);
435 m->data_len = data_to_append;
436 remaining_data -=
437 data_to_append;
438 m = m->next;
439 }
440 }
441 }
442 rte_mempool_put_bulk(mem->op_pool,
443 (void **)deq_ops, num_deq);
444 allocated -= num_deq;
445 }
446 }
447 allocated = 0;
448
449 end:
450 if (allocated)
451 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated);
452 rte_compressdev_private_xform_free(dev_id, priv_xform);
453 rte_free(ops);
454
455 if (test_data->perf_comp_force_stop) {
456 RTE_LOG(ERR, USER1,
457 "lcore: %d Perf. test has been aborted by user\n",
458 mem->lcore_id);
459 res = -1;
460 }
461 return res;
462 }
463
464 int
cperf_cyclecount_test_runner(void * test_ctx)465 cperf_cyclecount_test_runner(void *test_ctx)
466 {
467 struct cperf_cyclecount_ctx *ctx = test_ctx;
468 struct comp_test_data *test_data = ctx->ver.options;
469 uint32_t lcore = rte_lcore_id();
470 static uint16_t display_once;
471 static rte_spinlock_t print_spinlock;
472 int i;
473
474 uint32_t ops_enq_retries_comp;
475 uint32_t ops_deq_retries_comp;
476
477 uint32_t ops_enq_retries_decomp;
478 uint32_t ops_deq_retries_decomp;
479
480 uint32_t duration_setup_per_op;
481
482 uint32_t duration_enq_per_op_comp;
483 uint32_t duration_deq_per_op_comp;
484
485 uint32_t duration_enq_per_op_decomp;
486 uint32_t duration_deq_per_op_decomp;
487
488 ctx->ver.mem.lcore_id = lcore;
489
490 uint16_t exp = 0;
491 /*
492 * printing information about current compression thread
493 */
494 if (__atomic_compare_exchange_n(&ctx->ver.mem.print_info_once, &exp,
495 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED))
496 printf(" lcore: %u,"
497 " driver name: %s,"
498 " device name: %s,"
499 " device id: %u,"
500 " socket id: %u,"
501 " queue pair id: %u\n",
502 lcore,
503 ctx->ver.options->driver_name,
504 rte_compressdev_name_get(ctx->ver.mem.dev_id),
505 ctx->ver.mem.dev_id,
506 rte_compressdev_socket_id(ctx->ver.mem.dev_id),
507 ctx->ver.mem.qp_id);
508
509 /*
510 * First the verification part is needed
511 */
512 if (cperf_verify_test_runner(&ctx->ver))
513 return EXIT_FAILURE;
514
515 /*
516 * Run the tests twice, discarding the first performance
517 * results, before the cache is warmed up
518 */
519
520 /* C O M P R E S S */
521 for (i = 0; i < 2; i++) {
522 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0)
523 return EXIT_FAILURE;
524 }
525
526 ops_enq_retries_comp = ctx->ops_enq_retries;
527 ops_deq_retries_comp = ctx->ops_deq_retries;
528
529 duration_enq_per_op_comp = ctx->duration_enq /
530 (ctx->ver.mem.total_bufs * test_data->num_iter);
531 duration_deq_per_op_comp = ctx->duration_deq /
532 (ctx->ver.mem.total_bufs * test_data->num_iter);
533
534 /* D E C O M P R E S S */
535 for (i = 0; i < 2; i++) {
536 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0)
537 return EXIT_FAILURE;
538 }
539
540 ops_enq_retries_decomp = ctx->ops_enq_retries;
541 ops_deq_retries_decomp = ctx->ops_deq_retries;
542
543 duration_enq_per_op_decomp = ctx->duration_enq /
544 (ctx->ver.mem.total_bufs * test_data->num_iter);
545 duration_deq_per_op_decomp = ctx->duration_deq /
546 (ctx->ver.mem.total_bufs * test_data->num_iter);
547
548 duration_setup_per_op = ctx->duration_op /
549 (ctx->ver.mem.total_bufs * test_data->num_iter);
550
551 /* R E P O R T processing */
552 rte_spinlock_lock(&print_spinlock);
553
554 if (display_once == 0) {
555 display_once = 1;
556
557 printf("\nLegend for the table\n"
558 " - Retries section: number of retries for the following operations:\n"
559 " [C-e] - compression enqueue\n"
560 " [C-d] - compression dequeue\n"
561 " [D-e] - decompression enqueue\n"
562 " [D-d] - decompression dequeue\n"
563 " - Cycles section: number of cycles per 'op' for the following operations:\n"
564 " setup/op - memory allocation, op configuration and memory dealocation\n"
565 " [C-e] - compression enqueue\n"
566 " [C-d] - compression dequeue\n"
567 " [D-e] - decompression enqueue\n"
568 " [D-d] - decompression dequeue\n\n");
569
570 printf("\n%12s%6s%12s%17s",
571 "lcore id", "Level", "Comp size", "Comp ratio [%]");
572
573 printf(" |%10s %6s %8s %6s %8s",
574 " Retries:",
575 "[C-e]", "[C-d]",
576 "[D-e]", "[D-d]");
577
578 printf(" |%9s %9s %9s %9s %9s %9s\n",
579 " Cycles:",
580 "setup/op",
581 "[C-e]", "[C-d]",
582 "[D-e]", "[D-d]");
583 }
584
585 printf("%12u"
586 "%6u"
587 "%12zu"
588 "%17.2f",
589 ctx->ver.mem.lcore_id,
590 test_data->level,
591 ctx->ver.comp_data_sz,
592 ctx->ver.ratio);
593
594 printf(" |%10s %6u %8u %6u %8u",
595 " ",
596 ops_enq_retries_comp,
597 ops_deq_retries_comp,
598 ops_enq_retries_decomp,
599 ops_deq_retries_decomp);
600
601 printf(" |%9s %9u %9u %9u %9u %9u\n",
602 " ",
603 duration_setup_per_op,
604 duration_enq_per_op_comp,
605 duration_deq_per_op_comp,
606 duration_enq_per_op_decomp,
607 duration_deq_per_op_decomp);
608
609 rte_spinlock_unlock(&print_spinlock);
610
611 return EXIT_SUCCESS;
612 }
613