1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4 #include <isa-l.h>
5
6 #include <rte_common.h>
7 #include <rte_compressdev_pmd.h>
8 #include <rte_malloc.h>
9
10 #include "isal_compress_pmd_private.h"
11
12 static const struct rte_compressdev_capabilities isal_pmd_capabilities[] = {
13 {
14 .algo = RTE_COMP_ALGO_DEFLATE,
15 .comp_feature_flags = RTE_COMP_FF_OOP_SGL_IN_SGL_OUT |
16 RTE_COMP_FF_OOP_SGL_IN_LB_OUT |
17 RTE_COMP_FF_OOP_LB_IN_SGL_OUT |
18 RTE_COMP_FF_SHAREABLE_PRIV_XFORM |
19 RTE_COMP_FF_HUFFMAN_FIXED |
20 RTE_COMP_FF_HUFFMAN_DYNAMIC |
21 RTE_COMP_FF_CRC32_CHECKSUM |
22 RTE_COMP_FF_ADLER32_CHECKSUM,
23 .window_size = {
24 .min = 15,
25 .max = 15,
26 .increment = 0
27 },
28 },
29 RTE_COMP_END_OF_CAPABILITIES_LIST()
30 };
31
32 /** Configure device */
33 static int
isal_comp_pmd_config(struct rte_compressdev * dev,struct rte_compressdev_config * config)34 isal_comp_pmd_config(struct rte_compressdev *dev,
35 struct rte_compressdev_config *config)
36 {
37 int ret = 0;
38 unsigned int n;
39 char mp_name[RTE_COMPRESSDEV_NAME_MAX_LEN];
40 unsigned int elt_size = sizeof(struct isal_priv_xform);
41 struct isal_comp_private *internals = dev->data->dev_private;
42
43 n = snprintf(mp_name, sizeof(mp_name), "compdev_%d_xform_mp",
44 dev->data->dev_id);
45 if (n > sizeof(mp_name)) {
46 ISAL_PMD_LOG(ERR,
47 "Unable to create unique name for xform mempool");
48 return -ENOMEM;
49 }
50
51 internals->priv_xform_mp = rte_mempool_lookup(mp_name);
52
53 if (internals->priv_xform_mp != NULL) {
54 if (((internals->priv_xform_mp)->elt_size != elt_size) ||
55 ((internals->priv_xform_mp)->size <
56 config->max_nb_priv_xforms)) {
57
58 ISAL_PMD_LOG(ERR, "%s mempool already exists with different"
59 " initialization parameters", mp_name);
60 internals->priv_xform_mp = NULL;
61 return -ENOMEM;
62 }
63 } else { /* First time configuration */
64 internals->priv_xform_mp = rte_mempool_create(
65 mp_name, /* mempool name */
66 /* number of elements*/
67 config->max_nb_priv_xforms,
68 elt_size, /* element size*/
69 0, /* Cache size*/
70 0, /* private data size */
71 NULL, /* obj initialization constructor */
72 NULL, /* obj initialization constructor arg */
73 NULL, /**< obj constructor*/
74 NULL, /* obj constructor arg */
75 config->socket_id, /* socket id */
76 0); /* flags */
77 }
78
79 if (internals->priv_xform_mp == NULL) {
80 ISAL_PMD_LOG(ERR, "%s mempool allocation failed", mp_name);
81 return -ENOMEM;
82 }
83
84 dev->data->dev_private = internals;
85
86 return ret;
87 }
88
89 /** Start device */
90 static int
isal_comp_pmd_start(__rte_unused struct rte_compressdev * dev)91 isal_comp_pmd_start(__rte_unused struct rte_compressdev *dev)
92 {
93 return 0;
94 }
95
96 /** Stop device */
97 static void
isal_comp_pmd_stop(__rte_unused struct rte_compressdev * dev)98 isal_comp_pmd_stop(__rte_unused struct rte_compressdev *dev)
99 {
100 }
101
102 /** Close device */
103 static int
isal_comp_pmd_close(struct rte_compressdev * dev)104 isal_comp_pmd_close(struct rte_compressdev *dev)
105 {
106 /* Free private data */
107 struct isal_comp_private *internals = dev->data->dev_private;
108
109 rte_mempool_free(internals->priv_xform_mp);
110 return 0;
111 }
112
113 /** Get device statistics */
114 static void
isal_comp_pmd_stats_get(struct rte_compressdev * dev,struct rte_compressdev_stats * stats)115 isal_comp_pmd_stats_get(struct rte_compressdev *dev,
116 struct rte_compressdev_stats *stats)
117 {
118 uint16_t qp_id;
119
120 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
121 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
122
123 stats->enqueued_count += qp->qp_stats.enqueued_count;
124 stats->dequeued_count += qp->qp_stats.dequeued_count;
125
126 stats->enqueue_err_count += qp->qp_stats.enqueue_err_count;
127 stats->dequeue_err_count += qp->qp_stats.dequeue_err_count;
128 }
129 }
130
131 /** Get device info */
132 static void
isal_comp_pmd_info_get(struct rte_compressdev * dev __rte_unused,struct rte_compressdev_info * dev_info)133 isal_comp_pmd_info_get(struct rte_compressdev *dev __rte_unused,
134 struct rte_compressdev_info *dev_info)
135 {
136 if (dev_info != NULL) {
137 dev_info->capabilities = isal_pmd_capabilities;
138
139 /* Check CPU for supported vector instruction and set
140 * feature_flags
141 */
142 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
143 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX512;
144 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
145 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX2;
146 else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
147 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_AVX;
148 else
149 dev_info->feature_flags |= RTE_COMPDEV_FF_CPU_SSE;
150 }
151 }
152
153 /** Reset device statistics */
154 static void
isal_comp_pmd_stats_reset(struct rte_compressdev * dev)155 isal_comp_pmd_stats_reset(struct rte_compressdev *dev)
156 {
157 uint16_t qp_id;
158
159 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
160 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
161 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
162 }
163 }
164
165 /** Release queue pair */
166 static int
isal_comp_pmd_qp_release(struct rte_compressdev * dev,uint16_t qp_id)167 isal_comp_pmd_qp_release(struct rte_compressdev *dev, uint16_t qp_id)
168 {
169 struct isal_comp_qp *qp = dev->data->queue_pairs[qp_id];
170
171 if (qp == NULL)
172 return -EINVAL;
173
174 if (qp->stream)
175 rte_free(qp->stream->level_buf);
176
177 rte_free(qp->state);
178 rte_ring_free(qp->processed_pkts);
179 rte_free(qp->stream);
180 rte_free(qp);
181 dev->data->queue_pairs[qp_id] = NULL;
182
183 return 0;
184 }
185
186 /** Create a ring to place process packets on */
187 static struct rte_ring *
isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp * qp,unsigned int ring_size,int socket_id)188 isal_comp_pmd_qp_create_processed_pkts_ring(struct isal_comp_qp *qp,
189 unsigned int ring_size, int socket_id)
190 {
191 struct rte_ring *r;
192
193 r = rte_ring_lookup(qp->name);
194 if (r) {
195 if (rte_ring_get_size(r) >= ring_size) {
196 ISAL_PMD_LOG(DEBUG,
197 "Reusing existing ring %s for processed packets",
198 qp->name);
199 return r;
200 }
201
202 ISAL_PMD_LOG(ERR,
203 "Unable to reuse existing ring %s"
204 " for processed packets",
205 qp->name);
206 return NULL;
207 }
208
209 return rte_ring_create(qp->name, ring_size, socket_id,
210 RING_F_SP_ENQ | RING_F_SC_DEQ);
211 }
212
213 /** set a unique name for the queue pair based on its name, dev_id and qp_id */
214 static int
isal_comp_pmd_qp_set_unique_name(struct rte_compressdev * dev,struct isal_comp_qp * qp)215 isal_comp_pmd_qp_set_unique_name(struct rte_compressdev *dev,
216 struct isal_comp_qp *qp)
217 {
218 unsigned int n = snprintf(qp->name, sizeof(qp->name),
219 "isal_comp_pmd_%u_qp_%u",
220 dev->data->dev_id, qp->id);
221
222 if (n >= sizeof(qp->name))
223 return -1;
224
225 return 0;
226 }
227
228 /* Setup a queue pair */
229 static int
isal_comp_pmd_qp_setup(struct rte_compressdev * dev,uint16_t qp_id,uint32_t max_inflight_ops,int socket_id)230 isal_comp_pmd_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
231 uint32_t max_inflight_ops, int socket_id)
232 {
233 struct isal_comp_qp *qp = NULL;
234 int retval;
235
236 /* Free memory prior to re-allocation if needed. */
237 if (dev->data->queue_pairs[qp_id] != NULL)
238 isal_comp_pmd_qp_release(dev, qp_id);
239
240 /* Allocate the queue pair data structure. */
241 qp = rte_zmalloc_socket("Isa-l compression PMD Queue Pair", sizeof(*qp),
242 RTE_CACHE_LINE_SIZE, socket_id);
243 if (qp == NULL) {
244 ISAL_PMD_LOG(ERR, "Failed to allocate queue pair memory");
245 return (-ENOMEM);
246 }
247
248 /* Initialize memory for compression stream structure */
249 qp->stream = rte_zmalloc_socket("Isa-l compression stream ",
250 sizeof(struct isal_zstream), RTE_CACHE_LINE_SIZE,
251 socket_id);
252 if (qp->stream == NULL) {
253 ISAL_PMD_LOG(ERR, "Failed to allocate compression stream memory");
254 goto qp_setup_cleanup;
255 }
256 /* Initialize memory for compression level buffer */
257 qp->stream->level_buf = rte_zmalloc_socket("Isa-l compression lev_buf",
258 ISAL_DEF_LVL3_DEFAULT, RTE_CACHE_LINE_SIZE,
259 socket_id);
260 if (qp->stream->level_buf == NULL) {
261 ISAL_PMD_LOG(ERR, "Failed to allocate compression level_buf memory");
262 goto qp_setup_cleanup;
263 }
264
265 /* Initialize memory for decompression state structure */
266 qp->state = rte_zmalloc_socket("Isa-l decompression state",
267 sizeof(struct inflate_state), RTE_CACHE_LINE_SIZE,
268 socket_id);
269 if (qp->state == NULL) {
270 ISAL_PMD_LOG(ERR, "Failed to allocate decompression state memory");
271 goto qp_setup_cleanup;
272 }
273
274 qp->id = qp_id;
275 dev->data->queue_pairs[qp_id] = qp;
276
277 retval = isal_comp_pmd_qp_set_unique_name(dev, qp);
278 if (retval) {
279 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
280 "compression device");
281 goto qp_setup_cleanup;
282 }
283
284 qp->processed_pkts = isal_comp_pmd_qp_create_processed_pkts_ring(qp,
285 max_inflight_ops, socket_id);
286 if (qp->processed_pkts == NULL) {
287 ISAL_PMD_LOG(ERR, "Failed to create unique name for isal "
288 "compression device");
289 goto qp_setup_cleanup;
290 }
291
292 qp->num_free_elements = rte_ring_free_count(qp->processed_pkts);
293
294 memset(&qp->qp_stats, 0, sizeof(qp->qp_stats));
295 return 0;
296
297 qp_setup_cleanup:
298 if (qp->stream)
299 rte_free(qp->stream->level_buf);
300 rte_free(qp->stream);
301 rte_free(qp->state);
302 rte_free(qp);
303
304 return -1;
305 }
306
307 /** Set private xform data*/
308 static int
isal_comp_pmd_priv_xform_create(struct rte_compressdev * dev,const struct rte_comp_xform * xform,void ** priv_xform)309 isal_comp_pmd_priv_xform_create(struct rte_compressdev *dev,
310 const struct rte_comp_xform *xform, void **priv_xform)
311 {
312 int ret;
313 struct isal_comp_private *internals = dev->data->dev_private;
314
315 if (xform == NULL) {
316 ISAL_PMD_LOG(ERR, "Invalid Xform struct");
317 return -EINVAL;
318 }
319
320 if (rte_mempool_get(internals->priv_xform_mp, priv_xform)) {
321 ISAL_PMD_LOG(ERR,
322 "Couldn't get object from private xform mempool");
323 return -ENOMEM;
324 }
325
326 ret = isal_comp_set_priv_xform_parameters(*priv_xform, xform);
327 if (ret != 0) {
328 ISAL_PMD_LOG(ERR, "Failed to configure private xform parameters");
329
330 /* Return private xform to mempool */
331 rte_mempool_put(internals->priv_xform_mp, priv_xform);
332 return ret;
333 }
334 return 0;
335 }
336
337 /** Clear memory of the private xform so it doesn't leave key material behind */
338 static int
isal_comp_pmd_priv_xform_free(struct rte_compressdev * dev,void * priv_xform)339 isal_comp_pmd_priv_xform_free(struct rte_compressdev *dev, void *priv_xform)
340 {
341 struct isal_comp_private *internals = dev->data->dev_private;
342
343 /* Zero out the whole structure */
344 if (priv_xform) {
345 memset(priv_xform, 0, sizeof(struct isal_priv_xform));
346 rte_mempool_put(internals->priv_xform_mp, priv_xform);
347 }
348 return 0;
349 }
350
351 struct rte_compressdev_ops isal_pmd_ops = {
352 .dev_configure = isal_comp_pmd_config,
353 .dev_start = isal_comp_pmd_start,
354 .dev_stop = isal_comp_pmd_stop,
355 .dev_close = isal_comp_pmd_close,
356
357 .stats_get = isal_comp_pmd_stats_get,
358 .stats_reset = isal_comp_pmd_stats_reset,
359
360 .dev_infos_get = isal_comp_pmd_info_get,
361
362 .queue_pair_setup = isal_comp_pmd_qp_setup,
363 .queue_pair_release = isal_comp_pmd_qp_release,
364
365 .private_xform_create = isal_comp_pmd_priv_xform_create,
366 .private_xform_free = isal_comp_pmd_priv_xform_free,
367 };
368
369 struct rte_compressdev_ops *isal_compress_pmd_ops = &isal_pmd_ops;
370