1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4 #include <string.h>
5
6 #include <rte_common.h>
7 #include <rte_malloc.h>
8
9 #include "rte_port_sym_crypto.h"
10
11 /*
12 * Port Crypto Reader
13 */
14 #ifdef RTE_PORT_STATS_COLLECT
15
16 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val) \
17 (port)->stats.n_pkts_in += (val)
18 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val) \
19 (port)->stats.n_pkts_drop += (val)
20
21 #else
22
23 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(port, val)
24 #define RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(port, val)
25
26 #endif
27
28 struct rte_port_sym_crypto_reader {
29 struct rte_port_in_stats stats;
30
31 uint8_t cryptodev_id;
32 uint16_t queue_id;
33 struct rte_crypto_op *ops[RTE_PORT_IN_BURST_SIZE_MAX];
34 rte_port_sym_crypto_reader_callback_fn f_callback;
35 void *arg_callback;
36 };
37
38 static void *
rte_port_sym_crypto_reader_create(void * params,int socket_id)39 rte_port_sym_crypto_reader_create(void *params, int socket_id)
40 {
41 struct rte_port_sym_crypto_reader_params *conf =
42 params;
43 struct rte_port_sym_crypto_reader *port;
44
45 /* Check input parameters */
46 if (conf == NULL) {
47 RTE_LOG(ERR, PORT, "%s: params is NULL\n", __func__);
48 return NULL;
49 }
50
51 /* Memory allocation */
52 port = rte_zmalloc_socket("PORT", sizeof(*port),
53 RTE_CACHE_LINE_SIZE, socket_id);
54 if (port == NULL) {
55 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
56 return NULL;
57 }
58
59 /* Initialization */
60 port->cryptodev_id = conf->cryptodev_id;
61 port->queue_id = conf->queue_id;
62 port->f_callback = conf->f_callback;
63 port->arg_callback = conf->arg_callback;
64
65 return port;
66 }
67
68 static int
rte_port_sym_crypto_reader_rx(void * port,struct rte_mbuf ** pkts,uint32_t n_pkts)69 rte_port_sym_crypto_reader_rx(void *port, struct rte_mbuf **pkts, uint32_t n_pkts)
70 {
71 struct rte_port_sym_crypto_reader *p =
72 port;
73 uint16_t rx_ops_cnt, i, n = 0;
74
75 rx_ops_cnt = rte_cryptodev_dequeue_burst(p->cryptodev_id, p->queue_id,
76 p->ops, n_pkts);
77
78 for (i = 0; i < rx_ops_cnt; i++) {
79 struct rte_crypto_op *op = p->ops[i];
80
81 /** Drop failed pkts */
82 if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) {
83 rte_pktmbuf_free(op->sym->m_src);
84 continue;
85 }
86
87 pkts[n++] = op->sym->m_src;
88 }
89
90 if (p->f_callback)
91 (*p->f_callback)(pkts, n, p->arg_callback);
92
93 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_IN_ADD(p, n);
94 RTE_PORT_SYM_CRYPTO_READER_STATS_PKTS_DROP_ADD(p, rx_ops_cnt - n);
95
96 return n;
97 }
98
99 static int
rte_port_sym_crypto_reader_free(void * port)100 rte_port_sym_crypto_reader_free(void *port)
101 {
102 if (port == NULL) {
103 RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
104 return -EINVAL;
105 }
106
107 rte_free(port);
108
109 return 0;
110 }
111
rte_port_sym_crypto_reader_stats_read(void * port,struct rte_port_in_stats * stats,int clear)112 static int rte_port_sym_crypto_reader_stats_read(void *port,
113 struct rte_port_in_stats *stats, int clear)
114 {
115 struct rte_port_sym_crypto_reader *p =
116 port;
117
118 if (stats != NULL)
119 memcpy(stats, &p->stats, sizeof(p->stats));
120
121 if (clear)
122 memset(&p->stats, 0, sizeof(p->stats));
123
124 return 0;
125 }
126
127 /*
128 * Port crypto Writer
129 */
130 #ifdef RTE_PORT_STATS_COLLECT
131
132 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val) \
133 (port)->stats.n_pkts_in += (val)
134 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
135 (port)->stats.n_pkts_drop += (val)
136
137 #else
138
139 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(port, val)
140 #define RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(port, val)
141
142 #endif
143
144 struct rte_port_sym_crypto_writer {
145 struct rte_port_out_stats stats;
146
147 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
148
149 uint32_t tx_burst_sz;
150 uint32_t tx_buf_count;
151 uint64_t bsz_mask;
152
153 uint8_t cryptodev_id;
154 uint16_t queue_id;
155 uint16_t crypto_op_offset;
156 };
157
158 static void *
rte_port_sym_crypto_writer_create(void * params,int socket_id)159 rte_port_sym_crypto_writer_create(void *params, int socket_id)
160 {
161 struct rte_port_sym_crypto_writer_params *conf =
162 params;
163 struct rte_port_sym_crypto_writer *port;
164
165 /* Check input parameters */
166 if ((conf == NULL) ||
167 (conf->tx_burst_sz == 0) ||
168 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
169 (!rte_is_power_of_2(conf->tx_burst_sz))) {
170 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
171 return NULL;
172 }
173
174 /* Memory allocation */
175 port = rte_zmalloc_socket("PORT", sizeof(*port),
176 RTE_CACHE_LINE_SIZE, socket_id);
177 if (port == NULL) {
178 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
179 return NULL;
180 }
181
182 /* Initialization */
183 port->tx_burst_sz = conf->tx_burst_sz;
184 port->tx_buf_count = 0;
185 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
186
187 port->cryptodev_id = conf->cryptodev_id;
188 port->queue_id = conf->queue_id;
189 port->crypto_op_offset = conf->crypto_op_offset;
190
191 return port;
192 }
193
194 static inline void
send_burst(struct rte_port_sym_crypto_writer * p)195 send_burst(struct rte_port_sym_crypto_writer *p)
196 {
197 uint32_t nb_tx;
198
199 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
200 p->tx_buf, p->tx_buf_count);
201
202 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count -
203 nb_tx);
204 for (; nb_tx < p->tx_buf_count; nb_tx++)
205 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
206
207 p->tx_buf_count = 0;
208 }
209
210 static int
rte_port_sym_crypto_writer_tx(void * port,struct rte_mbuf * pkt)211 rte_port_sym_crypto_writer_tx(void *port, struct rte_mbuf *pkt)
212 {
213 struct rte_port_sym_crypto_writer *p =
214 port;
215
216 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
217 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
218 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
219 if (p->tx_buf_count >= p->tx_burst_sz)
220 send_burst(p);
221
222 return 0;
223 }
224
225 static int
rte_port_sym_crypto_writer_tx_bulk(void * port,struct rte_mbuf ** pkts,uint64_t pkts_mask)226 rte_port_sym_crypto_writer_tx_bulk(void *port,
227 struct rte_mbuf **pkts,
228 uint64_t pkts_mask)
229 {
230 struct rte_port_sym_crypto_writer *p =
231 port;
232 uint64_t bsz_mask = p->bsz_mask;
233 uint32_t tx_buf_count = p->tx_buf_count;
234 uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
235 ((pkts_mask & bsz_mask) ^ bsz_mask);
236
237 if (expr == 0) {
238 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
239 uint32_t i;
240
241 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, n_pkts);
242
243 for (i = 0; i < n_pkts; i++)
244 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
245 RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
246 p->crypto_op_offset);
247
248 if (p->tx_buf_count >= p->tx_burst_sz)
249 send_burst(p);
250 } else {
251 for (; pkts_mask;) {
252 uint32_t pkt_index = __builtin_ctzll(pkts_mask);
253 uint64_t pkt_mask = 1LLU << pkt_index;
254 struct rte_mbuf *pkt = pkts[pkt_index];
255
256 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
257 RTE_MBUF_METADATA_UINT8_PTR(pkt,
258 p->crypto_op_offset);
259
260 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
261 pkts_mask &= ~pkt_mask;
262 }
263
264 p->tx_buf_count = tx_buf_count;
265 if (tx_buf_count >= p->tx_burst_sz)
266 send_burst(p);
267 }
268
269 return 0;
270 }
271
272 static int
rte_port_sym_crypto_writer_flush(void * port)273 rte_port_sym_crypto_writer_flush(void *port)
274 {
275 struct rte_port_sym_crypto_writer *p =
276 port;
277
278 if (p->tx_buf_count > 0)
279 send_burst(p);
280
281 return 0;
282 }
283
284 static int
rte_port_sym_crypto_writer_free(void * port)285 rte_port_sym_crypto_writer_free(void *port)
286 {
287 if (port == NULL) {
288 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
289 return -EINVAL;
290 }
291
292 rte_port_sym_crypto_writer_flush(port);
293 rte_free(port);
294
295 return 0;
296 }
297
rte_port_sym_crypto_writer_stats_read(void * port,struct rte_port_out_stats * stats,int clear)298 static int rte_port_sym_crypto_writer_stats_read(void *port,
299 struct rte_port_out_stats *stats, int clear)
300 {
301 struct rte_port_sym_crypto_writer *p =
302 port;
303
304 if (stats != NULL)
305 memcpy(stats, &p->stats, sizeof(p->stats));
306
307 if (clear)
308 memset(&p->stats, 0, sizeof(p->stats));
309
310 return 0;
311 }
312
313 /*
314 * Port crypto Writer Nodrop
315 */
316 #ifdef RTE_PORT_STATS_COLLECT
317
318 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val) \
319 (port)->stats.n_pkts_in += (val)
320 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val) \
321 (port)->stats.n_pkts_drop += (val)
322
323 #else
324
325 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(port, val)
326 #define RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(port, val)
327
328 #endif
329
330 struct rte_port_sym_crypto_writer_nodrop {
331 struct rte_port_out_stats stats;
332
333 struct rte_crypto_op *tx_buf[2 * RTE_PORT_IN_BURST_SIZE_MAX];
334 uint32_t tx_burst_sz;
335 uint32_t tx_buf_count;
336 uint64_t bsz_mask;
337 uint64_t n_retries;
338
339 uint8_t cryptodev_id;
340 uint16_t queue_id;
341 uint16_t crypto_op_offset;
342 };
343
344 static void *
rte_port_sym_crypto_writer_nodrop_create(void * params,int socket_id)345 rte_port_sym_crypto_writer_nodrop_create(void *params, int socket_id)
346 {
347 struct rte_port_sym_crypto_writer_nodrop_params *conf =
348 params;
349 struct rte_port_sym_crypto_writer_nodrop *port;
350
351 /* Check input parameters */
352 if ((conf == NULL) ||
353 (conf->tx_burst_sz == 0) ||
354 (conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
355 (!rte_is_power_of_2(conf->tx_burst_sz))) {
356 RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
357 return NULL;
358 }
359
360 /* Memory allocation */
361 port = rte_zmalloc_socket("PORT", sizeof(*port),
362 RTE_CACHE_LINE_SIZE, socket_id);
363 if (port == NULL) {
364 RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
365 return NULL;
366 }
367
368 /* Initialization */
369 port->cryptodev_id = conf->cryptodev_id;
370 port->queue_id = conf->queue_id;
371 port->crypto_op_offset = conf->crypto_op_offset;
372 port->tx_burst_sz = conf->tx_burst_sz;
373 port->tx_buf_count = 0;
374 port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
375
376 /*
377 * When n_retries is 0 it means that we should wait for every packet to
378 * send no matter how many retries should it take. To limit number of
379 * branches in fast path, we use UINT64_MAX instead of branching.
380 */
381 port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
382
383 return port;
384 }
385
386 static inline void
send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop * p)387 send_burst_nodrop(struct rte_port_sym_crypto_writer_nodrop *p)
388 {
389 uint32_t nb_tx = 0, i;
390
391 nb_tx = rte_cryptodev_enqueue_burst(p->cryptodev_id, p->queue_id,
392 p->tx_buf, p->tx_buf_count);
393
394 /* We sent all the packets in a first try */
395 if (nb_tx >= p->tx_buf_count) {
396 p->tx_buf_count = 0;
397 return;
398 }
399
400 for (i = 0; i < p->n_retries; i++) {
401 nb_tx += rte_cryptodev_enqueue_burst(p->cryptodev_id,
402 p->queue_id, p->tx_buf + nb_tx,
403 p->tx_buf_count - nb_tx);
404
405 /* We sent all the packets in more than one try */
406 if (nb_tx >= p->tx_buf_count) {
407 p->tx_buf_count = 0;
408 return;
409 }
410 }
411
412 /* We didn't send the packets in maximum allowed attempts */
413 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_DROP_ADD(p,
414 p->tx_buf_count - nb_tx);
415 for ( ; nb_tx < p->tx_buf_count; nb_tx++)
416 rte_pktmbuf_free(p->tx_buf[nb_tx]->sym->m_src);
417
418 p->tx_buf_count = 0;
419 }
420
421 static int
rte_port_sym_crypto_writer_nodrop_tx(void * port,struct rte_mbuf * pkt)422 rte_port_sym_crypto_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
423 {
424 struct rte_port_sym_crypto_writer_nodrop *p =
425 port;
426
427 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
428 RTE_MBUF_METADATA_UINT8_PTR(pkt, p->crypto_op_offset);
429 RTE_PORT_SYM_CRYPTO_WRITER_STATS_PKTS_IN_ADD(p, 1);
430 if (p->tx_buf_count >= p->tx_burst_sz)
431 send_burst_nodrop(p);
432
433 return 0;
434 }
435
436 static int
rte_port_sym_crypto_writer_nodrop_tx_bulk(void * port,struct rte_mbuf ** pkts,uint64_t pkts_mask)437 rte_port_sym_crypto_writer_nodrop_tx_bulk(void *port,
438 struct rte_mbuf **pkts,
439 uint64_t pkts_mask)
440 {
441 struct rte_port_sym_crypto_writer_nodrop *p =
442 port;
443
444 uint64_t bsz_mask = p->bsz_mask;
445 uint32_t tx_buf_count = p->tx_buf_count;
446 uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
447 ((pkts_mask & bsz_mask) ^ bsz_mask);
448
449 if (expr == 0) {
450 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
451 uint32_t i;
452
453 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
454
455 for (i = 0; i < n_pkts; i++)
456 p->tx_buf[p->tx_buf_count++] = (struct rte_crypto_op *)
457 RTE_MBUF_METADATA_UINT8_PTR(pkts[i],
458 p->crypto_op_offset);
459
460 if (p->tx_buf_count >= p->tx_burst_sz)
461 send_burst_nodrop(p);
462 } else {
463 for ( ; pkts_mask; ) {
464 uint32_t pkt_index = __builtin_ctzll(pkts_mask);
465 uint64_t pkt_mask = 1LLU << pkt_index;
466 struct rte_mbuf *pkt = pkts[pkt_index];
467
468 p->tx_buf[tx_buf_count++] = (struct rte_crypto_op *)
469 RTE_MBUF_METADATA_UINT8_PTR(pkt,
470 p->crypto_op_offset);
471 RTE_PORT_SYM_CRYPTO_WRITER_NODROP_STATS_PKTS_IN_ADD(p,
472 1);
473 pkts_mask &= ~pkt_mask;
474 }
475
476 p->tx_buf_count = tx_buf_count;
477 if (tx_buf_count >= p->tx_burst_sz)
478 send_burst_nodrop(p);
479 }
480
481 return 0;
482 }
483
484 static int
rte_port_sym_crypto_writer_nodrop_flush(void * port)485 rte_port_sym_crypto_writer_nodrop_flush(void *port)
486 {
487 struct rte_port_sym_crypto_writer_nodrop *p =
488 port;
489
490 if (p->tx_buf_count > 0)
491 send_burst_nodrop(p);
492
493 return 0;
494 }
495
496 static int
rte_port_sym_crypto_writer_nodrop_free(void * port)497 rte_port_sym_crypto_writer_nodrop_free(void *port)
498 {
499 if (port == NULL) {
500 RTE_LOG(ERR, PORT, "%s: Port is NULL\n", __func__);
501 return -EINVAL;
502 }
503
504 rte_port_sym_crypto_writer_nodrop_flush(port);
505 rte_free(port);
506
507 return 0;
508 }
509
rte_port_sym_crypto_writer_nodrop_stats_read(void * port,struct rte_port_out_stats * stats,int clear)510 static int rte_port_sym_crypto_writer_nodrop_stats_read(void *port,
511 struct rte_port_out_stats *stats, int clear)
512 {
513 struct rte_port_sym_crypto_writer_nodrop *p =
514 port;
515
516 if (stats != NULL)
517 memcpy(stats, &p->stats, sizeof(p->stats));
518
519 if (clear)
520 memset(&p->stats, 0, sizeof(p->stats));
521
522 return 0;
523 }
524
525
526 /*
527 * Summary of port operations
528 */
529 struct rte_port_in_ops rte_port_sym_crypto_reader_ops = {
530 .f_create = rte_port_sym_crypto_reader_create,
531 .f_free = rte_port_sym_crypto_reader_free,
532 .f_rx = rte_port_sym_crypto_reader_rx,
533 .f_stats = rte_port_sym_crypto_reader_stats_read,
534 };
535
536 struct rte_port_out_ops rte_port_sym_crypto_writer_ops = {
537 .f_create = rte_port_sym_crypto_writer_create,
538 .f_free = rte_port_sym_crypto_writer_free,
539 .f_tx = rte_port_sym_crypto_writer_tx,
540 .f_tx_bulk = rte_port_sym_crypto_writer_tx_bulk,
541 .f_flush = rte_port_sym_crypto_writer_flush,
542 .f_stats = rte_port_sym_crypto_writer_stats_read,
543 };
544
545 struct rte_port_out_ops rte_port_sym_crypto_writer_nodrop_ops = {
546 .f_create = rte_port_sym_crypto_writer_nodrop_create,
547 .f_free = rte_port_sym_crypto_writer_nodrop_free,
548 .f_tx = rte_port_sym_crypto_writer_nodrop_tx,
549 .f_tx_bulk = rte_port_sym_crypto_writer_nodrop_tx_bulk,
550 .f_flush = rte_port_sym_crypto_writer_nodrop_flush,
551 .f_stats = rte_port_sym_crypto_writer_nodrop_stats_read,
552 };
553