1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4 #include <isa-l.h>
5
6 #include <rte_bus_vdev.h>
7 #include <rte_common.h>
8 #include <rte_malloc.h>
9 #include <rte_mbuf.h>
10 #include <rte_compressdev_pmd.h>
11
12 #include "isal_compress_pmd_private.h"
13
14 #define RTE_COMP_ISAL_WINDOW_SIZE 15
15 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
16 #define RTE_COMP_ISAL_LEVEL_ONE 1
17 #define RTE_COMP_ISAL_LEVEL_TWO 2
18 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
19 #define CHKSUM_SZ_CRC 8
20 #define CHKSUM_SZ_ADLER 4
21
22 #define STRINGIFY(s) #s
23 #define ISAL_TOSTRING(maj, min, patch) \
24 STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
25 #define ISAL_VERSION_STRING \
26 ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
27
28 /* Verify and set private xform parameters */
29 int
isal_comp_set_priv_xform_parameters(struct isal_priv_xform * priv_xform,const struct rte_comp_xform * xform)30 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
31 const struct rte_comp_xform *xform)
32 {
33 if (xform == NULL)
34 return -EINVAL;
35
36 /* Set compression private xform variables */
37 if (xform->type == RTE_COMP_COMPRESS) {
38 /* Set private xform type - COMPRESS/DECOMPRESS */
39 priv_xform->type = RTE_COMP_COMPRESS;
40
41 /* Set private xform algorithm */
42 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
43 if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
44 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
45 return -ENOTSUP;
46 }
47 ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
48 return -ENOTSUP;
49 }
50 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
51
52 /* Set private xform window size, 32K supported */
53 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
54 priv_xform->compress.window_size =
55 RTE_COMP_ISAL_WINDOW_SIZE;
56 else {
57 ISAL_PMD_LOG(ERR, "Window size not supported\n");
58 return -ENOTSUP;
59 }
60
61 /* Set private xform huffman type */
62 switch (xform->compress.deflate.huffman) {
63 case(RTE_COMP_HUFFMAN_DEFAULT):
64 priv_xform->compress.deflate.huffman =
65 RTE_COMP_HUFFMAN_DEFAULT;
66 break;
67 case(RTE_COMP_HUFFMAN_FIXED):
68 priv_xform->compress.deflate.huffman =
69 RTE_COMP_HUFFMAN_FIXED;
70 break;
71 case(RTE_COMP_HUFFMAN_DYNAMIC):
72 priv_xform->compress.deflate.huffman =
73 RTE_COMP_HUFFMAN_DYNAMIC;
74 break;
75 default:
76 ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
77 return -ENOTSUP;
78 }
79
80 /* Set private xform checksum */
81 switch (xform->compress.chksum) {
82 /* Raw deflate by default */
83 case(RTE_COMP_CHECKSUM_NONE):
84 priv_xform->compress.chksum = IGZIP_DEFLATE;
85 break;
86 case(RTE_COMP_CHECKSUM_CRC32):
87 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
88 break;
89 case(RTE_COMP_CHECKSUM_ADLER32):
90 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
91 break;
92 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
93 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
94 " supported\n");
95 return -ENOTSUP;
96 default:
97 ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
98 priv_xform->compress.chksum = IGZIP_DEFLATE;
99 break;
100 }
101
102 /* Set private xform level.
103 * Checking compliance with compressdev API, -1 <= level => 9
104 */
105 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
106 xform->compress.level > RTE_COMP_LEVEL_MAX) {
107 ISAL_PMD_LOG(ERR, "Compression level out of range\n");
108 return -EINVAL;
109 }
110 /* Check for Compressdev API level 0, No compression
111 * not supported in ISA-L
112 */
113 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
114 ISAL_PMD_LOG(ERR, "No Compression not supported\n");
115 return -ENOTSUP;
116 }
117 /* If using fixed huffman code, level must be 0 */
118 else if (priv_xform->compress.deflate.huffman ==
119 RTE_COMP_HUFFMAN_FIXED) {
120 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
121 " fixed huffman code\n");
122 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
123 priv_xform->level_buffer_size =
124 ISAL_DEF_LVL0_DEFAULT;
125 } else {
126 /* Mapping API levels to ISA-L levels 1,2 & 3 */
127 switch (xform->compress.level) {
128 case RTE_COMP_LEVEL_PMD_DEFAULT:
129 /* Default is 1 if not using fixed huffman */
130 priv_xform->compress.level =
131 RTE_COMP_ISAL_LEVEL_ONE;
132 priv_xform->level_buffer_size =
133 ISAL_DEF_LVL1_DEFAULT;
134 break;
135 case RTE_COMP_LEVEL_MIN:
136 priv_xform->compress.level =
137 RTE_COMP_ISAL_LEVEL_ONE;
138 priv_xform->level_buffer_size =
139 ISAL_DEF_LVL1_DEFAULT;
140 break;
141 case RTE_COMP_ISAL_LEVEL_TWO:
142 priv_xform->compress.level =
143 RTE_COMP_ISAL_LEVEL_TWO;
144 priv_xform->level_buffer_size =
145 ISAL_DEF_LVL2_DEFAULT;
146 break;
147 /* Level 3 or higher requested */
148 default:
149 /* Check for AVX512, to use ISA-L level 3 */
150 if (rte_cpu_get_flag_enabled(
151 RTE_CPUFLAG_AVX512F)) {
152 priv_xform->compress.level =
153 RTE_COMP_ISAL_LEVEL_THREE;
154 priv_xform->level_buffer_size =
155 ISAL_DEF_LVL3_DEFAULT;
156 }
157 /* Check for AVX2, to use ISA-L level 3 */
158 else if (rte_cpu_get_flag_enabled(
159 RTE_CPUFLAG_AVX2)) {
160 priv_xform->compress.level =
161 RTE_COMP_ISAL_LEVEL_THREE;
162 priv_xform->level_buffer_size =
163 ISAL_DEF_LVL3_DEFAULT;
164 } else {
165 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
166 " 3 or above; Level 3 optimized"
167 " for AVX512 & AVX2 only."
168 " level changed to 2.\n");
169 priv_xform->compress.level =
170 RTE_COMP_ISAL_LEVEL_TWO;
171 priv_xform->level_buffer_size =
172 ISAL_DEF_LVL2_DEFAULT;
173 }
174 }
175 }
176 }
177
178 /* Set decompression private xform variables */
179 else if (xform->type == RTE_COMP_DECOMPRESS) {
180
181 /* Set private xform type - COMPRESS/DECOMPRESS */
182 priv_xform->type = RTE_COMP_DECOMPRESS;
183
184 /* Set private xform algorithm */
185 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
186 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
187 ISAL_PMD_LOG(ERR, "By pass not supported\n");
188 return -ENOTSUP;
189 }
190 ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
191 return -ENOTSUP;
192 }
193 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
194
195 /* Set private xform checksum */
196 switch (xform->decompress.chksum) {
197 /* Raw deflate by default */
198 case(RTE_COMP_CHECKSUM_NONE):
199 priv_xform->decompress.chksum = ISAL_DEFLATE;
200 break;
201 case(RTE_COMP_CHECKSUM_CRC32):
202 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
203 break;
204 case(RTE_COMP_CHECKSUM_ADLER32):
205 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
206 break;
207 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
208 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
209 " supported\n");
210 return -ENOTSUP;
211 default:
212 ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
213 priv_xform->decompress.chksum = ISAL_DEFLATE;
214 break;
215 }
216
217 /* Set private xform window size, 32K supported */
218 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
219 priv_xform->decompress.window_size =
220 RTE_COMP_ISAL_WINDOW_SIZE;
221 else {
222 ISAL_PMD_LOG(ERR, "Window size not supported\n");
223 return -ENOTSUP;
224 }
225 }
226 return 0;
227 }
228
229 /* Compression using chained mbufs for input/output data */
230 static int
chained_mbuf_compression(struct rte_comp_op * op,struct isal_comp_qp * qp)231 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
232 {
233 int ret;
234 uint32_t remaining_offset;
235 uint32_t remaining_data = op->src.length;
236 struct rte_mbuf *src = op->m_src;
237 struct rte_mbuf *dst = op->m_dst;
238
239 /* check for source/destination offset passing multiple segments
240 * and point compression stream to input/output buffer.
241 */
242 remaining_offset = op->src.offset;
243 while (remaining_offset >= src->data_len) {
244 remaining_offset -= src->data_len;
245 src = src->next;
246 }
247 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
248 op->src.length);
249 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
250 remaining_offset);
251
252 remaining_offset = op->dst.offset;
253 while (remaining_offset >= dst->data_len) {
254 remaining_offset -= dst->data_len;
255 dst = dst->next;
256 }
257 qp->stream->avail_out = dst->data_len - remaining_offset;
258 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
259 remaining_offset);
260
261 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
262 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
263 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
264 return -1;
265 }
266
267 while (qp->stream->internal_state.state != ZSTATE_END) {
268 /* Last segment of data */
269 if (remaining_data <= src->data_len)
270 qp->stream->end_of_stream = 1;
271
272 /* Execute compression operation */
273 ret = isal_deflate(qp->stream);
274
275 remaining_data = op->src.length - qp->stream->total_in;
276
277 if (ret != COMP_OK) {
278 ISAL_PMD_LOG(ERR, "Compression operation failed\n");
279 op->status = RTE_COMP_OP_STATUS_ERROR;
280 return ret;
281 }
282
283 if (qp->stream->avail_in == 0 &&
284 qp->stream->total_in != op->src.length) {
285 if (src->next != NULL) {
286 src = src->next;
287 qp->stream->next_in =
288 rte_pktmbuf_mtod(src, uint8_t *);
289 qp->stream->avail_in =
290 RTE_MIN(remaining_data, src->data_len);
291 } else {
292 ISAL_PMD_LOG(ERR,
293 "Not enough input buffer segments\n");
294 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
295 return -1;
296 }
297 }
298
299 if (qp->stream->avail_out == 0 &&
300 qp->stream->internal_state.state != ZSTATE_END) {
301 if (dst->next != NULL) {
302 dst = dst->next;
303 qp->stream->next_out =
304 rte_pktmbuf_mtod(dst, uint8_t *);
305 qp->stream->avail_out = dst->data_len;
306 } else {
307 ISAL_PMD_LOG(ERR,
308 "Not enough output buffer segments\n");
309 op->status =
310 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
311 return -1;
312 }
313 }
314 }
315
316 return 0;
317 }
318
319 /* Decompression using chained mbufs for input/output data */
320 static int
chained_mbuf_decompression(struct rte_comp_op * op,struct isal_comp_qp * qp)321 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
322 {
323 int ret;
324 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
325 uint32_t remaining_data = op->src.length;
326 struct rte_mbuf *src = op->m_src;
327 struct rte_mbuf *dst = op->m_dst;
328
329 /* check for offset passing multiple segments
330 * and point decompression state to input/output buffer
331 */
332 src_remaining_offset = op->src.offset;
333 while (src_remaining_offset >= src->data_len) {
334 src_remaining_offset -= src->data_len;
335 src = src->next;
336 }
337 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
338 op->src.length);
339 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
340 src_remaining_offset);
341
342 dst_remaining_offset = op->dst.offset;
343 while (dst_remaining_offset >= dst->data_len) {
344 dst_remaining_offset -= dst->data_len;
345 dst = dst->next;
346 }
347 qp->state->avail_out = dst->data_len - dst_remaining_offset;
348 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
349 dst_remaining_offset);
350
351 while (qp->state->block_state != ISAL_BLOCK_FINISH) {
352
353 ret = isal_inflate(qp->state);
354
355 /* Check for first segment, offset needs to be accounted for */
356 if (remaining_data == op->src.length) {
357 consumed_data = src->data_len - src_remaining_offset;
358 } else
359 consumed_data = src->data_len;
360
361 if (qp->state->avail_in == 0
362 && op->consumed != op->src.length) {
363 op->consumed += consumed_data;
364 remaining_data -= consumed_data;
365
366 if (src->next != NULL) {
367 src = src->next;
368 qp->state->next_in =
369 rte_pktmbuf_mtod(src, uint8_t *);
370 qp->state->avail_in =
371 RTE_MIN(remaining_data, src->data_len);
372 }
373 }
374
375 if (ret == ISAL_OUT_OVERFLOW) {
376 ISAL_PMD_LOG(ERR, "Decompression operation ran "
377 "out of space, but can be recovered.\n%d bytes "
378 "consumed\t%d bytes produced\n",
379 consumed_data, qp->state->total_out);
380 op->status =
381 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
382 return ret;
383 } else if (ret < 0) {
384 ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
385 op->status = RTE_COMP_OP_STATUS_ERROR;
386 return ret;
387 }
388
389 if (qp->state->avail_out == 0 &&
390 qp->state->block_state != ISAL_BLOCK_FINISH) {
391 if (dst->next != NULL) {
392 dst = dst->next;
393 qp->state->next_out =
394 rte_pktmbuf_mtod(dst, uint8_t *);
395 qp->state->avail_out = dst->data_len;
396 } else {
397 ISAL_PMD_LOG(ERR,
398 "Not enough output buffer segments\n");
399 op->status =
400 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
401 return -1;
402 }
403 }
404 }
405
406 return 0;
407 }
408
409 /* Stateless Compression Function */
410 static int
process_isal_deflate(struct rte_comp_op * op,struct isal_comp_qp * qp,struct isal_priv_xform * priv_xform)411 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
412 struct isal_priv_xform *priv_xform)
413 {
414 int ret = 0;
415 op->status = RTE_COMP_OP_STATUS_SUCCESS;
416
417 /* Required due to init clearing level_buf */
418 uint8_t *temp_level_buf = qp->stream->level_buf;
419
420 /* Initialize compression stream */
421 isal_deflate_init(qp->stream);
422
423 qp->stream->level_buf = temp_level_buf;
424
425 /* Set Checksum flag */
426 qp->stream->gzip_flag = priv_xform->compress.chksum;
427
428 /* Stateless operation, input will be consumed in one go */
429 qp->stream->flush = NO_FLUSH;
430
431 /* set compression level & intermediate level buffer size */
432 qp->stream->level = priv_xform->compress.level;
433 qp->stream->level_buf_size = priv_xform->level_buffer_size;
434
435 /* Set op huffman code */
436 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
437 isal_deflate_set_hufftables(qp->stream, NULL,
438 IGZIP_HUFFTABLE_STATIC);
439 else if (priv_xform->compress.deflate.huffman ==
440 RTE_COMP_HUFFMAN_DEFAULT)
441 isal_deflate_set_hufftables(qp->stream, NULL,
442 IGZIP_HUFFTABLE_DEFAULT);
443 /* Dynamically change the huffman code to suit the input data */
444 else if (priv_xform->compress.deflate.huffman ==
445 RTE_COMP_HUFFMAN_DYNAMIC)
446 isal_deflate_set_hufftables(qp->stream, NULL,
447 IGZIP_HUFFTABLE_DEFAULT);
448
449 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
450 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
451 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
452 return -1;
453 }
454
455 if (op->dst.offset >= op->m_dst->pkt_len) {
456 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
457 " for offset provided.\n");
458 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
459 return -1;
460 }
461
462 /* Chained mbufs */
463 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
464 ret = chained_mbuf_compression(op, qp);
465 if (ret < 0)
466 return ret;
467 } else {
468 /* Linear buffer */
469 qp->stream->end_of_stream = 1; /* All input consumed in one */
470 /* Point compression stream to input buffer */
471 qp->stream->avail_in = op->src.length;
472 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
473 uint8_t *, op->src.offset);
474
475 /* Point compression stream to output buffer */
476 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
477 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
478 uint8_t *, op->dst.offset);
479
480 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
481 ISAL_PMD_LOG(ERR, "Invalid source or destination"
482 " buffers\n");
483 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
484 return -1;
485 }
486
487 /* Execute compression operation */
488 ret = isal_deflate_stateless(qp->stream);
489
490 /* Check that output buffer did not run out of space */
491 if (ret == STATELESS_OVERFLOW) {
492 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
493 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
494 return ret;
495 }
496
497 /* Check that input buffer has been fully consumed */
498 if (qp->stream->avail_in != (uint32_t)0) {
499 ISAL_PMD_LOG(ERR, "Input buffer could not be read"
500 " entirely\n");
501 op->status = RTE_COMP_OP_STATUS_ERROR;
502 return -1;
503 }
504
505 if (ret != COMP_OK) {
506 ISAL_PMD_LOG(ERR, "Compression operation failed\n");
507 op->status = RTE_COMP_OP_STATUS_ERROR;
508 return ret;
509 }
510 }
511
512 op->consumed = qp->stream->total_in;
513 if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
514 op->produced = qp->stream->total_out;
515 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
516 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
517 op->output_chksum = qp->stream->internal_state.crc + 1;
518 } else {
519 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
520 op->output_chksum = qp->stream->internal_state.crc;
521 }
522
523 return ret;
524 }
525
526 /* Stateless Decompression Function */
527 static int
process_isal_inflate(struct rte_comp_op * op,struct isal_comp_qp * qp,struct isal_priv_xform * priv_xform)528 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
529 struct isal_priv_xform *priv_xform)
530 {
531 int ret = 0;
532
533 op->status = RTE_COMP_OP_STATUS_SUCCESS;
534
535 /* Initialize decompression state */
536 isal_inflate_init(qp->state);
537
538 /* Set Checksum flag */
539 qp->state->crc_flag = priv_xform->decompress.chksum;
540
541 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
542 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
543 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
544 return -1;
545 }
546
547 if (op->dst.offset >= op->m_dst->pkt_len) {
548 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
549 "offset provided.\n");
550 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
551 return -1;
552 }
553
554 /* Chained mbufs */
555 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
556 ret = chained_mbuf_decompression(op, qp);
557 if (ret != 0)
558 return ret;
559 } else {
560 /* Linear buffer */
561 /* Point decompression state to input buffer */
562 qp->state->avail_in = op->src.length;
563 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
564 uint8_t *, op->src.offset);
565
566 /* Point decompression state to output buffer */
567 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
568 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
569 uint8_t *, op->dst.offset);
570
571 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
572 ISAL_PMD_LOG(ERR, "Invalid source or destination"
573 " buffers\n");
574 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
575 return -1;
576 }
577
578 /* Execute decompression operation */
579 ret = isal_inflate_stateless(qp->state);
580
581 if (ret == ISAL_OUT_OVERFLOW) {
582 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
583 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
584 return ret;
585 }
586
587 /* Check that input buffer has been fully consumed */
588 if (qp->state->avail_in != (uint32_t)0) {
589 ISAL_PMD_LOG(ERR, "Input buffer could not be read"
590 " entirely\n");
591 op->status = RTE_COMP_OP_STATUS_ERROR;
592 return -1;
593 }
594
595 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
596 ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
597 op->status = RTE_COMP_OP_STATUS_ERROR;
598 return ret;
599 }
600 op->consumed = op->src.length - qp->state->avail_in;
601 }
602 op->produced = qp->state->total_out;
603 op->output_chksum = qp->state->crc;
604
605 return ret;
606 }
607
608 /* Process compression/decompression operation */
609 static int
process_op(struct isal_comp_qp * qp,struct rte_comp_op * op,struct isal_priv_xform * priv_xform)610 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
611 struct isal_priv_xform *priv_xform)
612 {
613 switch (priv_xform->type) {
614 case RTE_COMP_COMPRESS:
615 process_isal_deflate(op, qp, priv_xform);
616 break;
617 case RTE_COMP_DECOMPRESS:
618 process_isal_inflate(op, qp, priv_xform);
619 break;
620 default:
621 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
622 return -ENOTSUP;
623 }
624 return 0;
625 }
626
627 /* Enqueue burst */
628 static uint16_t
isal_comp_pmd_enqueue_burst(void * queue_pair,struct rte_comp_op ** ops,uint16_t nb_ops)629 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
630 uint16_t nb_ops)
631 {
632 struct isal_comp_qp *qp = queue_pair;
633 uint16_t i;
634 int retval;
635 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
636
637 for (i = 0; i < num_enq; i++) {
638 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
639 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
640 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
641 qp->qp_stats.enqueue_err_count++;
642 continue;
643 }
644 retval = process_op(qp, ops[i], ops[i]->private_xform);
645 if (unlikely(retval < 0) ||
646 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
647 qp->qp_stats.enqueue_err_count++;
648 }
649 }
650
651 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
652 num_enq, NULL);
653 qp->num_free_elements -= retval;
654 qp->qp_stats.enqueued_count += retval;
655
656 return retval;
657 }
658
659 /* Dequeue burst */
660 static uint16_t
isal_comp_pmd_dequeue_burst(void * queue_pair,struct rte_comp_op ** ops,uint16_t nb_ops)661 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
662 uint16_t nb_ops)
663 {
664 struct isal_comp_qp *qp = queue_pair;
665 uint16_t nb_dequeued;
666
667 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
668 nb_ops, NULL);
669 qp->num_free_elements += nb_dequeued;
670 qp->qp_stats.dequeued_count += nb_dequeued;
671
672 return nb_dequeued;
673 }
674
675 /* Create ISA-L compression device */
676 static int
compdev_isal_create(const char * name,struct rte_vdev_device * vdev,struct rte_compressdev_pmd_init_params * init_params)677 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
678 struct rte_compressdev_pmd_init_params *init_params)
679 {
680 struct rte_compressdev *dev;
681
682 dev = rte_compressdev_pmd_create(name, &vdev->device,
683 sizeof(struct isal_comp_private), init_params);
684 if (dev == NULL) {
685 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
686 return -EFAULT;
687 }
688
689 dev->dev_ops = isal_compress_pmd_ops;
690
691 /* register rx/tx burst functions for data path */
692 dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
693 dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
694
695 ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
696
697 return 0;
698 }
699
700 /** Remove compression device */
701 static int
compdev_isal_remove_dev(struct rte_vdev_device * vdev)702 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
703 {
704 struct rte_compressdev *compdev;
705 const char *name;
706
707 name = rte_vdev_device_name(vdev);
708 if (name == NULL)
709 return -EINVAL;
710
711 compdev = rte_compressdev_pmd_get_named_dev(name);
712 if (compdev == NULL)
713 return -ENODEV;
714
715 return rte_compressdev_pmd_destroy(compdev);
716 }
717
718 /** Initialise ISA-L compression device */
719 static int
compdev_isal_probe(struct rte_vdev_device * dev)720 compdev_isal_probe(struct rte_vdev_device *dev)
721 {
722 struct rte_compressdev_pmd_init_params init_params = {
723 "",
724 rte_socket_id(),
725 };
726 const char *name, *args;
727 int retval;
728
729 name = rte_vdev_device_name(dev);
730 if (name == NULL)
731 return -EINVAL;
732
733 args = rte_vdev_device_args(dev);
734
735 retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
736 if (retval) {
737 ISAL_PMD_LOG(ERR,
738 "Failed to parse initialisation arguments[%s]\n", args);
739 return -EINVAL;
740 }
741
742 return compdev_isal_create(name, dev, &init_params);
743 }
744
745 static struct rte_vdev_driver compdev_isal_pmd_drv = {
746 .probe = compdev_isal_probe,
747 .remove = compdev_isal_remove_dev,
748 };
749
750 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
751 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
752 "socket_id=<int>");
753 RTE_LOG_REGISTER(isal_logtype_driver, pmd.compress.isal, INFO);
754