1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4 #include <isa-l.h>
5
6 #include <rte_bus_vdev.h>
7 #include <rte_common.h>
8 #include <rte_cpuflags.h>
9 #include <rte_malloc.h>
10 #include <rte_mbuf.h>
11 #include <rte_compressdev_pmd.h>
12
13 #include "isal_compress_pmd_private.h"
14
15 #define RTE_COMP_ISAL_WINDOW_SIZE 15
16 #define RTE_COMP_ISAL_LEVEL_ZERO 0 /* ISA-L Level 0 used for fixed Huffman */
17 #define RTE_COMP_ISAL_LEVEL_ONE 1
18 #define RTE_COMP_ISAL_LEVEL_TWO 2
19 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
20 #define CHKSUM_SZ_CRC 8
21 #define CHKSUM_SZ_ADLER 4
22
23 #define STRINGIFY(s) #s
24 #define ISAL_TOSTRING(maj, min, patch) \
25 STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
26 #define ISAL_VERSION_STRING \
27 ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
28
29 /* Verify and set private xform parameters */
30 int
isal_comp_set_priv_xform_parameters(struct isal_priv_xform * priv_xform,const struct rte_comp_xform * xform)31 isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
32 const struct rte_comp_xform *xform)
33 {
34 if (xform == NULL)
35 return -EINVAL;
36
37 /* Set compression private xform variables */
38 if (xform->type == RTE_COMP_COMPRESS) {
39 /* Set private xform type - COMPRESS/DECOMPRESS */
40 priv_xform->type = RTE_COMP_COMPRESS;
41
42 /* Set private xform algorithm */
43 if (xform->compress.algo != RTE_COMP_ALGO_DEFLATE) {
44 if (xform->compress.algo == RTE_COMP_ALGO_NULL) {
45 ISAL_PMD_LOG(ERR, "By-pass not supported\n");
46 return -ENOTSUP;
47 }
48 ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
49 return -ENOTSUP;
50 }
51 priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
52
53 /* Set private xform window size, 32K supported */
54 if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
55 priv_xform->compress.window_size =
56 RTE_COMP_ISAL_WINDOW_SIZE;
57 else {
58 ISAL_PMD_LOG(ERR, "Window size not supported\n");
59 return -ENOTSUP;
60 }
61
62 /* Set private xform huffman type */
63 switch (xform->compress.deflate.huffman) {
64 case(RTE_COMP_HUFFMAN_DEFAULT):
65 priv_xform->compress.deflate.huffman =
66 RTE_COMP_HUFFMAN_DEFAULT;
67 break;
68 case(RTE_COMP_HUFFMAN_FIXED):
69 priv_xform->compress.deflate.huffman =
70 RTE_COMP_HUFFMAN_FIXED;
71 break;
72 case(RTE_COMP_HUFFMAN_DYNAMIC):
73 priv_xform->compress.deflate.huffman =
74 RTE_COMP_HUFFMAN_DYNAMIC;
75 break;
76 default:
77 ISAL_PMD_LOG(ERR, "Huffman code not supported\n");
78 return -ENOTSUP;
79 }
80
81 /* Set private xform checksum */
82 switch (xform->compress.chksum) {
83 /* Raw deflate by default */
84 case(RTE_COMP_CHECKSUM_NONE):
85 priv_xform->compress.chksum = IGZIP_DEFLATE;
86 break;
87 case(RTE_COMP_CHECKSUM_CRC32):
88 priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
89 break;
90 case(RTE_COMP_CHECKSUM_ADLER32):
91 priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
92 break;
93 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
94 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
95 " supported\n");
96 return -ENOTSUP;
97 default:
98 ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
99 priv_xform->compress.chksum = IGZIP_DEFLATE;
100 break;
101 }
102
103 /* Set private xform level.
104 * Checking compliance with compressdev API, -1 <= level => 9
105 */
106 if (xform->compress.level < RTE_COMP_LEVEL_PMD_DEFAULT ||
107 xform->compress.level > RTE_COMP_LEVEL_MAX) {
108 ISAL_PMD_LOG(ERR, "Compression level out of range\n");
109 return -EINVAL;
110 }
111 /* Check for Compressdev API level 0, No compression
112 * not supported in ISA-L
113 */
114 else if (xform->compress.level == RTE_COMP_LEVEL_NONE) {
115 ISAL_PMD_LOG(ERR, "No Compression not supported\n");
116 return -ENOTSUP;
117 }
118 /* If using fixed huffman code, level must be 0 */
119 else if (priv_xform->compress.deflate.huffman ==
120 RTE_COMP_HUFFMAN_FIXED) {
121 ISAL_PMD_LOG(DEBUG, "ISA-L level 0 used due to a"
122 " fixed huffman code\n");
123 priv_xform->compress.level = RTE_COMP_ISAL_LEVEL_ZERO;
124 priv_xform->level_buffer_size =
125 ISAL_DEF_LVL0_DEFAULT;
126 } else {
127 /* Mapping API levels to ISA-L levels 1,2 & 3 */
128 switch (xform->compress.level) {
129 case RTE_COMP_LEVEL_PMD_DEFAULT:
130 /* Default is 1 if not using fixed huffman */
131 priv_xform->compress.level =
132 RTE_COMP_ISAL_LEVEL_ONE;
133 priv_xform->level_buffer_size =
134 ISAL_DEF_LVL1_DEFAULT;
135 break;
136 case RTE_COMP_LEVEL_MIN:
137 priv_xform->compress.level =
138 RTE_COMP_ISAL_LEVEL_ONE;
139 priv_xform->level_buffer_size =
140 ISAL_DEF_LVL1_DEFAULT;
141 break;
142 case RTE_COMP_ISAL_LEVEL_TWO:
143 priv_xform->compress.level =
144 RTE_COMP_ISAL_LEVEL_TWO;
145 priv_xform->level_buffer_size =
146 ISAL_DEF_LVL2_DEFAULT;
147 break;
148 /* Level 3 or higher requested */
149 default:
150 #ifdef RTE_ARCH_X86
151 /* Check for AVX512, to use ISA-L level 3 */
152 if (rte_cpu_get_flag_enabled(
153 RTE_CPUFLAG_AVX512F)) {
154 priv_xform->compress.level =
155 RTE_COMP_ISAL_LEVEL_THREE;
156 priv_xform->level_buffer_size =
157 ISAL_DEF_LVL3_DEFAULT;
158 }
159 /* Check for AVX2, to use ISA-L level 3 */
160 else if (rte_cpu_get_flag_enabled(
161 RTE_CPUFLAG_AVX2)) {
162 priv_xform->compress.level =
163 RTE_COMP_ISAL_LEVEL_THREE;
164 priv_xform->level_buffer_size =
165 ISAL_DEF_LVL3_DEFAULT;
166 } else
167 #endif
168 {
169 ISAL_PMD_LOG(DEBUG, "Requested ISA-L level"
170 " 3 or above; Level 3 optimized"
171 " for AVX512 & AVX2 only."
172 " level changed to 2.\n");
173 priv_xform->compress.level =
174 RTE_COMP_ISAL_LEVEL_TWO;
175 priv_xform->level_buffer_size =
176 ISAL_DEF_LVL2_DEFAULT;
177 }
178 }
179 }
180 }
181
182 /* Set decompression private xform variables */
183 else if (xform->type == RTE_COMP_DECOMPRESS) {
184
185 /* Set private xform type - COMPRESS/DECOMPRESS */
186 priv_xform->type = RTE_COMP_DECOMPRESS;
187
188 /* Set private xform algorithm */
189 if (xform->decompress.algo != RTE_COMP_ALGO_DEFLATE) {
190 if (xform->decompress.algo == RTE_COMP_ALGO_NULL) {
191 ISAL_PMD_LOG(ERR, "By pass not supported\n");
192 return -ENOTSUP;
193 }
194 ISAL_PMD_LOG(ERR, "Algorithm not supported\n");
195 return -ENOTSUP;
196 }
197 priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
198
199 /* Set private xform checksum */
200 switch (xform->decompress.chksum) {
201 /* Raw deflate by default */
202 case(RTE_COMP_CHECKSUM_NONE):
203 priv_xform->decompress.chksum = ISAL_DEFLATE;
204 break;
205 case(RTE_COMP_CHECKSUM_CRC32):
206 priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
207 break;
208 case(RTE_COMP_CHECKSUM_ADLER32):
209 priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
210 break;
211 case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
212 ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
213 " supported\n");
214 return -ENOTSUP;
215 default:
216 ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
217 priv_xform->decompress.chksum = ISAL_DEFLATE;
218 break;
219 }
220
221 /* Set private xform window size, 32K supported */
222 if (xform->decompress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
223 priv_xform->decompress.window_size =
224 RTE_COMP_ISAL_WINDOW_SIZE;
225 else {
226 ISAL_PMD_LOG(ERR, "Window size not supported\n");
227 return -ENOTSUP;
228 }
229 }
230 return 0;
231 }
232
233 /* Compression using chained mbufs for input/output data */
234 static int
chained_mbuf_compression(struct rte_comp_op * op,struct isal_comp_qp * qp)235 chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
236 {
237 int ret;
238 uint32_t remaining_offset;
239 uint32_t remaining_data = op->src.length;
240 struct rte_mbuf *src = op->m_src;
241 struct rte_mbuf *dst = op->m_dst;
242
243 /* check for source/destination offset passing multiple segments
244 * and point compression stream to input/output buffer.
245 */
246 remaining_offset = op->src.offset;
247 while (remaining_offset >= src->data_len) {
248 remaining_offset -= src->data_len;
249 src = src->next;
250 }
251 qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
252 op->src.length);
253 qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
254 remaining_offset);
255
256 remaining_offset = op->dst.offset;
257 while (remaining_offset >= dst->data_len) {
258 remaining_offset -= dst->data_len;
259 dst = dst->next;
260 }
261 qp->stream->avail_out = dst->data_len - remaining_offset;
262 qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
263 remaining_offset);
264
265 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
266 ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
267 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
268 return -1;
269 }
270
271 while (qp->stream->internal_state.state != ZSTATE_END) {
272 /* Last segment of data */
273 if (remaining_data <= src->data_len)
274 qp->stream->end_of_stream = 1;
275
276 /* Execute compression operation */
277 ret = isal_deflate(qp->stream);
278
279 remaining_data = op->src.length - qp->stream->total_in;
280
281 if (ret != COMP_OK) {
282 ISAL_PMD_LOG(ERR, "Compression operation failed\n");
283 op->status = RTE_COMP_OP_STATUS_ERROR;
284 return ret;
285 }
286
287 if (qp->stream->avail_in == 0 &&
288 qp->stream->total_in != op->src.length) {
289 if (src->next != NULL) {
290 src = src->next;
291 qp->stream->next_in =
292 rte_pktmbuf_mtod(src, uint8_t *);
293 qp->stream->avail_in =
294 RTE_MIN(remaining_data, src->data_len);
295 } else {
296 ISAL_PMD_LOG(ERR,
297 "Not enough input buffer segments\n");
298 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
299 return -1;
300 }
301 }
302
303 if (qp->stream->avail_out == 0 &&
304 qp->stream->internal_state.state != ZSTATE_END) {
305 if (dst->next != NULL) {
306 dst = dst->next;
307 qp->stream->next_out =
308 rte_pktmbuf_mtod(dst, uint8_t *);
309 qp->stream->avail_out = dst->data_len;
310 } else {
311 ISAL_PMD_LOG(ERR,
312 "Not enough output buffer segments\n");
313 op->status =
314 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
315 return -1;
316 }
317 }
318 }
319
320 return 0;
321 }
322
323 /* Decompression using chained mbufs for input/output data */
324 static int
chained_mbuf_decompression(struct rte_comp_op * op,struct isal_comp_qp * qp)325 chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
326 {
327 int ret;
328 uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
329 uint32_t remaining_data = op->src.length;
330 struct rte_mbuf *src = op->m_src;
331 struct rte_mbuf *dst = op->m_dst;
332
333 /* check for offset passing multiple segments
334 * and point decompression state to input/output buffer
335 */
336 src_remaining_offset = op->src.offset;
337 while (src_remaining_offset >= src->data_len) {
338 src_remaining_offset -= src->data_len;
339 src = src->next;
340 }
341 qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
342 op->src.length);
343 qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
344 src_remaining_offset);
345
346 dst_remaining_offset = op->dst.offset;
347 while (dst_remaining_offset >= dst->data_len) {
348 dst_remaining_offset -= dst->data_len;
349 dst = dst->next;
350 }
351 qp->state->avail_out = dst->data_len - dst_remaining_offset;
352 qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
353 dst_remaining_offset);
354
355 while (qp->state->block_state != ISAL_BLOCK_FINISH) {
356
357 ret = isal_inflate(qp->state);
358
359 /* Check for first segment, offset needs to be accounted for */
360 if (remaining_data == op->src.length) {
361 consumed_data = src->data_len - src_remaining_offset;
362 } else
363 consumed_data = src->data_len;
364
365 if (qp->state->avail_in == 0
366 && op->consumed != op->src.length) {
367 op->consumed += consumed_data;
368 remaining_data -= consumed_data;
369
370 if (src->next != NULL) {
371 src = src->next;
372 qp->state->next_in =
373 rte_pktmbuf_mtod(src, uint8_t *);
374 qp->state->avail_in =
375 RTE_MIN(remaining_data, src->data_len);
376 }
377 }
378
379 if (ret == ISAL_OUT_OVERFLOW) {
380 ISAL_PMD_LOG(ERR, "Decompression operation ran "
381 "out of space, but can be recovered.\n%d bytes "
382 "consumed\t%d bytes produced\n",
383 consumed_data, qp->state->total_out);
384 op->status =
385 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
386 return ret;
387 } else if (ret < 0) {
388 ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
389 op->status = RTE_COMP_OP_STATUS_ERROR;
390 return ret;
391 }
392
393 if (qp->state->avail_out == 0 &&
394 qp->state->block_state != ISAL_BLOCK_FINISH) {
395 if (dst->next != NULL) {
396 dst = dst->next;
397 qp->state->next_out =
398 rte_pktmbuf_mtod(dst, uint8_t *);
399 qp->state->avail_out = dst->data_len;
400 } else {
401 ISAL_PMD_LOG(ERR,
402 "Not enough output buffer segments\n");
403 op->status =
404 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
405 return -1;
406 }
407 }
408 }
409
410 return 0;
411 }
412
413 /* Stateless Compression Function */
414 static int
process_isal_deflate(struct rte_comp_op * op,struct isal_comp_qp * qp,struct isal_priv_xform * priv_xform)415 process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
416 struct isal_priv_xform *priv_xform)
417 {
418 int ret = 0;
419 op->status = RTE_COMP_OP_STATUS_SUCCESS;
420
421 /* Required due to init clearing level_buf */
422 uint8_t *temp_level_buf = qp->stream->level_buf;
423
424 /* Initialize compression stream */
425 isal_deflate_init(qp->stream);
426
427 qp->stream->level_buf = temp_level_buf;
428
429 /* Set Checksum flag */
430 qp->stream->gzip_flag = priv_xform->compress.chksum;
431
432 /* Stateless operation, input will be consumed in one go */
433 qp->stream->flush = NO_FLUSH;
434
435 /* set compression level & intermediate level buffer size */
436 qp->stream->level = priv_xform->compress.level;
437 qp->stream->level_buf_size = priv_xform->level_buffer_size;
438
439 /* Set op huffman code */
440 if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
441 isal_deflate_set_hufftables(qp->stream, NULL,
442 IGZIP_HUFFTABLE_STATIC);
443 else if (priv_xform->compress.deflate.huffman ==
444 RTE_COMP_HUFFMAN_DEFAULT)
445 isal_deflate_set_hufftables(qp->stream, NULL,
446 IGZIP_HUFFTABLE_DEFAULT);
447 /* Dynamically change the huffman code to suit the input data */
448 else if (priv_xform->compress.deflate.huffman ==
449 RTE_COMP_HUFFMAN_DYNAMIC)
450 isal_deflate_set_hufftables(qp->stream, NULL,
451 IGZIP_HUFFTABLE_DEFAULT);
452
453 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
454 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
455 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
456 return -1;
457 }
458
459 if (op->dst.offset >= op->m_dst->pkt_len) {
460 ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
461 " for offset provided.\n");
462 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
463 return -1;
464 }
465
466 /* Chained mbufs */
467 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
468 ret = chained_mbuf_compression(op, qp);
469 if (ret < 0)
470 return ret;
471 } else {
472 /* Linear buffer */
473 qp->stream->end_of_stream = 1; /* All input consumed in one */
474 /* Point compression stream to input buffer */
475 qp->stream->avail_in = op->src.length;
476 qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
477 uint8_t *, op->src.offset);
478
479 /* Point compression stream to output buffer */
480 qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
481 qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
482 uint8_t *, op->dst.offset);
483
484 if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
485 ISAL_PMD_LOG(ERR, "Invalid source or destination"
486 " buffers\n");
487 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
488 return -1;
489 }
490
491 /* Execute compression operation */
492 ret = isal_deflate_stateless(qp->stream);
493
494 /* Check that output buffer did not run out of space */
495 if (ret == STATELESS_OVERFLOW) {
496 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
497 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
498 return ret;
499 }
500
501 /* Check that input buffer has been fully consumed */
502 if (qp->stream->avail_in != (uint32_t)0) {
503 ISAL_PMD_LOG(ERR, "Input buffer could not be read"
504 " entirely\n");
505 op->status = RTE_COMP_OP_STATUS_ERROR;
506 return -1;
507 }
508
509 if (ret != COMP_OK) {
510 ISAL_PMD_LOG(ERR, "Compression operation failed\n");
511 op->status = RTE_COMP_OP_STATUS_ERROR;
512 return ret;
513 }
514 }
515
516 op->consumed = qp->stream->total_in;
517 if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
518 op->produced = qp->stream->total_out;
519 } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
520 op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
521 op->output_chksum = qp->stream->internal_state.crc + 1;
522 } else {
523 op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
524 op->output_chksum = qp->stream->internal_state.crc;
525 }
526
527 return ret;
528 }
529
530 /* Stateless Decompression Function */
531 static int
process_isal_inflate(struct rte_comp_op * op,struct isal_comp_qp * qp,struct isal_priv_xform * priv_xform)532 process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
533 struct isal_priv_xform *priv_xform)
534 {
535 int ret = 0;
536
537 op->status = RTE_COMP_OP_STATUS_SUCCESS;
538
539 /* Initialize decompression state */
540 isal_inflate_init(qp->state);
541
542 /* Set Checksum flag */
543 qp->state->crc_flag = priv_xform->decompress.chksum;
544
545 if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
546 ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
547 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
548 return -1;
549 }
550
551 if (op->dst.offset >= op->m_dst->pkt_len) {
552 ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
553 "offset provided.\n");
554 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
555 return -1;
556 }
557
558 /* Chained mbufs */
559 if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
560 ret = chained_mbuf_decompression(op, qp);
561 if (ret != 0)
562 return ret;
563 } else {
564 /* Linear buffer */
565 /* Point decompression state to input buffer */
566 qp->state->avail_in = op->src.length;
567 qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
568 uint8_t *, op->src.offset);
569
570 /* Point decompression state to output buffer */
571 qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
572 qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
573 uint8_t *, op->dst.offset);
574
575 if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
576 ISAL_PMD_LOG(ERR, "Invalid source or destination"
577 " buffers\n");
578 op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
579 return -1;
580 }
581
582 /* Execute decompression operation */
583 ret = isal_inflate_stateless(qp->state);
584
585 if (ret == ISAL_OUT_OVERFLOW) {
586 ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
587 op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
588 return ret;
589 }
590
591 /* Check that input buffer has been fully consumed */
592 if (qp->state->avail_in != (uint32_t)0) {
593 ISAL_PMD_LOG(ERR, "Input buffer could not be read"
594 " entirely\n");
595 op->status = RTE_COMP_OP_STATUS_ERROR;
596 return -1;
597 }
598
599 if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
600 ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
601 op->status = RTE_COMP_OP_STATUS_ERROR;
602 return ret;
603 }
604 op->consumed = op->src.length - qp->state->avail_in;
605 }
606 op->produced = qp->state->total_out;
607 op->output_chksum = qp->state->crc;
608
609 return ret;
610 }
611
612 /* Process compression/decompression operation */
613 static int
process_op(struct isal_comp_qp * qp,struct rte_comp_op * op,struct isal_priv_xform * priv_xform)614 process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
615 struct isal_priv_xform *priv_xform)
616 {
617 switch (priv_xform->type) {
618 case RTE_COMP_COMPRESS:
619 process_isal_deflate(op, qp, priv_xform);
620 break;
621 case RTE_COMP_DECOMPRESS:
622 process_isal_inflate(op, qp, priv_xform);
623 break;
624 default:
625 ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
626 return -ENOTSUP;
627 }
628 return 0;
629 }
630
631 /* Enqueue burst */
632 static uint16_t
isal_comp_pmd_enqueue_burst(void * queue_pair,struct rte_comp_op ** ops,uint16_t nb_ops)633 isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
634 uint16_t nb_ops)
635 {
636 struct isal_comp_qp *qp = queue_pair;
637 uint16_t i;
638 int retval;
639 int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
640
641 for (i = 0; i < num_enq; i++) {
642 if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
643 ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
644 ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
645 qp->qp_stats.enqueue_err_count++;
646 continue;
647 }
648 retval = process_op(qp, ops[i], ops[i]->private_xform);
649 if (unlikely(retval < 0) ||
650 ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
651 qp->qp_stats.enqueue_err_count++;
652 }
653 }
654
655 retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
656 num_enq, NULL);
657 qp->num_free_elements -= retval;
658 qp->qp_stats.enqueued_count += retval;
659
660 return retval;
661 }
662
663 /* Dequeue burst */
664 static uint16_t
isal_comp_pmd_dequeue_burst(void * queue_pair,struct rte_comp_op ** ops,uint16_t nb_ops)665 isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
666 uint16_t nb_ops)
667 {
668 struct isal_comp_qp *qp = queue_pair;
669 uint16_t nb_dequeued;
670
671 nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
672 nb_ops, NULL);
673 qp->num_free_elements += nb_dequeued;
674 qp->qp_stats.dequeued_count += nb_dequeued;
675
676 return nb_dequeued;
677 }
678
679 /* Create ISA-L compression device */
680 static int
compdev_isal_create(const char * name,struct rte_vdev_device * vdev,struct rte_compressdev_pmd_init_params * init_params)681 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
682 struct rte_compressdev_pmd_init_params *init_params)
683 {
684 struct rte_compressdev *dev;
685
686 dev = rte_compressdev_pmd_create(name, &vdev->device,
687 sizeof(struct isal_comp_private), init_params);
688 if (dev == NULL) {
689 ISAL_PMD_LOG(ERR, "failed to create compressdev vdev");
690 return -EFAULT;
691 }
692
693 dev->dev_ops = isal_compress_pmd_ops;
694
695 /* register rx/tx burst functions for data path */
696 dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
697 dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
698
699 ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
700
701 return 0;
702 }
703
704 /** Remove compression device */
705 static int
compdev_isal_remove_dev(struct rte_vdev_device * vdev)706 compdev_isal_remove_dev(struct rte_vdev_device *vdev)
707 {
708 struct rte_compressdev *compdev;
709 const char *name;
710
711 name = rte_vdev_device_name(vdev);
712 if (name == NULL)
713 return -EINVAL;
714
715 compdev = rte_compressdev_pmd_get_named_dev(name);
716 if (compdev == NULL)
717 return -ENODEV;
718
719 return rte_compressdev_pmd_destroy(compdev);
720 }
721
722 /** Initialise ISA-L compression device */
723 static int
compdev_isal_probe(struct rte_vdev_device * dev)724 compdev_isal_probe(struct rte_vdev_device *dev)
725 {
726 struct rte_compressdev_pmd_init_params init_params = {
727 "",
728 rte_socket_id(),
729 };
730 const char *name, *args;
731 int retval;
732
733 name = rte_vdev_device_name(dev);
734 if (name == NULL)
735 return -EINVAL;
736
737 args = rte_vdev_device_args(dev);
738
739 retval = rte_compressdev_pmd_parse_input_args(&init_params, args);
740 if (retval) {
741 ISAL_PMD_LOG(ERR,
742 "Failed to parse initialisation arguments[%s]\n", args);
743 return -EINVAL;
744 }
745
746 return compdev_isal_create(name, dev, &init_params);
747 }
748
749 static struct rte_vdev_driver compdev_isal_pmd_drv = {
750 .probe = compdev_isal_probe,
751 .remove = compdev_isal_remove_dev,
752 };
753
754 RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
755 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
756 "socket_id=<int>");
757 RTE_LOG_REGISTER_DEFAULT(isal_logtype_driver, INFO);
758