1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2021-2022 NXP
3 */
4
5 #include <rte_byteorder.h>
6 #include <rte_common.h>
7 #include <cryptodev_pmd.h>
8 #include <rte_crypto.h>
9 #include <rte_cryptodev.h>
10 #ifdef RTE_LIB_SECURITY
11 #include <rte_security_driver.h>
12 #endif
13
14 /* RTA header files */
15 #include <desc/algo.h>
16 #include <desc/ipsec.h>
17
18 #include <rte_dpaa_bus.h>
19 #include <dpaa_sec.h>
20 #include <dpaa_sec_log.h>
21
22 struct dpaa_sec_raw_dp_ctx {
23 dpaa_sec_session *session;
24 uint32_t tail;
25 uint32_t head;
26 uint16_t cached_enqueue;
27 uint16_t cached_dequeue;
28 };
29
30 static inline int
is_encode(dpaa_sec_session * ses)31 is_encode(dpaa_sec_session *ses)
32 {
33 return ses->dir == DIR_ENC;
34 }
35
is_decode(dpaa_sec_session * ses)36 static inline int is_decode(dpaa_sec_session *ses)
37 {
38 return ses->dir == DIR_DEC;
39 }
40
41 static __rte_always_inline int
dpaa_sec_raw_enqueue_done(void * qp_data,uint8_t * drv_ctx,uint32_t n)42 dpaa_sec_raw_enqueue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n)
43 {
44 RTE_SET_USED(qp_data);
45 RTE_SET_USED(drv_ctx);
46 RTE_SET_USED(n);
47
48 return 0;
49 }
50
51 static __rte_always_inline int
dpaa_sec_raw_dequeue_done(void * qp_data,uint8_t * drv_ctx,uint32_t n)52 dpaa_sec_raw_dequeue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n)
53 {
54 RTE_SET_USED(qp_data);
55 RTE_SET_USED(drv_ctx);
56 RTE_SET_USED(n);
57
58 return 0;
59 }
60
61 static inline struct dpaa_sec_op_ctx *
dpaa_sec_alloc_raw_ctx(dpaa_sec_session * ses,int sg_count)62 dpaa_sec_alloc_raw_ctx(dpaa_sec_session *ses, int sg_count)
63 {
64 struct dpaa_sec_op_ctx *ctx;
65 int i, retval;
66
67 retval = rte_mempool_get(
68 ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
69 (void **)(&ctx));
70 if (!ctx || retval) {
71 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
72 return NULL;
73 }
74 /*
75 * Clear SG memory. There are 16 SG entries of 16 Bytes each.
76 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
77 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
78 * each packet, memset is costlier than dcbz_64().
79 */
80 for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
81 dcbz_64(&ctx->job.sg[i]);
82
83 ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
84 ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
85
86 return ctx;
87 }
88
89 static struct dpaa_sec_job *
build_dpaa_raw_dp_auth_fd(uint8_t * drv_ctx,struct rte_crypto_sgl * sgl,struct rte_crypto_sgl * dest_sgl,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,void * userdata,struct qm_fd * fd)90 build_dpaa_raw_dp_auth_fd(uint8_t *drv_ctx,
91 struct rte_crypto_sgl *sgl,
92 struct rte_crypto_sgl *dest_sgl,
93 struct rte_crypto_va_iova_ptr *iv,
94 struct rte_crypto_va_iova_ptr *digest,
95 struct rte_crypto_va_iova_ptr *auth_iv,
96 union rte_crypto_sym_ofs ofs,
97 void *userdata,
98 struct qm_fd *fd)
99 {
100 RTE_SET_USED(dest_sgl);
101 RTE_SET_USED(iv);
102 RTE_SET_USED(auth_iv);
103 RTE_SET_USED(fd);
104
105 dpaa_sec_session *ses =
106 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
107 struct dpaa_sec_job *cf;
108 struct dpaa_sec_op_ctx *ctx;
109 struct qm_sg_entry *sg, *out_sg, *in_sg;
110 phys_addr_t start_addr;
111 uint8_t *old_digest, extra_segs;
112 int data_len, data_offset, total_len = 0;
113 unsigned int i;
114
115 for (i = 0; i < sgl->num; i++)
116 total_len += sgl->vec[i].len;
117
118 data_len = total_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
119 data_offset = ofs.ofs.auth.head;
120
121 /* Support only length in bits for SNOW3G and ZUC */
122
123 if (is_decode(ses))
124 extra_segs = 3;
125 else
126 extra_segs = 2;
127
128 if (sgl->num > MAX_SG_ENTRIES) {
129 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
130 MAX_SG_ENTRIES);
131 return NULL;
132 }
133 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + extra_segs);
134 if (!ctx)
135 return NULL;
136
137 cf = &ctx->job;
138 ctx->userdata = (void *)userdata;
139 old_digest = ctx->digest;
140
141 /* output */
142 out_sg = &cf->sg[0];
143 qm_sg_entry_set64(out_sg, digest->iova);
144 out_sg->length = ses->digest_length;
145 cpu_to_hw_sg(out_sg);
146
147 /* input */
148 in_sg = &cf->sg[1];
149 /* need to extend the input to a compound frame */
150 in_sg->extension = 1;
151 in_sg->final = 1;
152 in_sg->length = data_len;
153 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
154
155 /* 1st seg */
156 sg = in_sg + 1;
157
158 if (ses->iv.length) {
159 uint8_t *iv_ptr;
160
161 iv_ptr = rte_crypto_op_ctod_offset(userdata, uint8_t *,
162 ses->iv.offset);
163
164 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
165 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
166 sg->length = 12;
167 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
168 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
169 sg->length = 8;
170 } else {
171 sg->length = ses->iv.length;
172 }
173 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
174 in_sg->length += sg->length;
175 cpu_to_hw_sg(sg);
176 sg++;
177 }
178
179 qm_sg_entry_set64(sg, sgl->vec[0].iova);
180 sg->offset = data_offset;
181
182 if (data_len <= (int)(sgl->vec[0].len - data_offset)) {
183 sg->length = data_len;
184 } else {
185 sg->length = sgl->vec[0].len - data_offset;
186
187 /* remaining i/p segs */
188 for (i = 1; i < sgl->num; i++) {
189 cpu_to_hw_sg(sg);
190 sg++;
191 qm_sg_entry_set64(sg, sgl->vec[i].iova);
192 if (data_len > (int)sgl->vec[i].len)
193 sg->length = sgl->vec[0].len;
194 else
195 sg->length = data_len;
196
197 data_len = data_len - sg->length;
198 if (data_len < 1)
199 break;
200 }
201 }
202
203 if (is_decode(ses)) {
204 /* Digest verification case */
205 cpu_to_hw_sg(sg);
206 sg++;
207 rte_memcpy(old_digest, digest->va,
208 ses->digest_length);
209 start_addr = rte_dpaa_mem_vtop(old_digest);
210 qm_sg_entry_set64(sg, start_addr);
211 sg->length = ses->digest_length;
212 in_sg->length += ses->digest_length;
213 }
214 sg->final = 1;
215 cpu_to_hw_sg(sg);
216 cpu_to_hw_sg(in_sg);
217
218 return cf;
219 }
220
221 static inline struct dpaa_sec_job *
build_raw_cipher_auth_gcm_sg(uint8_t * drv_ctx,struct rte_crypto_sgl * sgl,struct rte_crypto_sgl * dest_sgl,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,void * userdata,struct qm_fd * fd)222 build_raw_cipher_auth_gcm_sg(uint8_t *drv_ctx,
223 struct rte_crypto_sgl *sgl,
224 struct rte_crypto_sgl *dest_sgl,
225 struct rte_crypto_va_iova_ptr *iv,
226 struct rte_crypto_va_iova_ptr *digest,
227 struct rte_crypto_va_iova_ptr *auth_iv,
228 union rte_crypto_sym_ofs ofs,
229 void *userdata,
230 struct qm_fd *fd)
231 {
232 dpaa_sec_session *ses =
233 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
234 struct dpaa_sec_job *cf;
235 struct dpaa_sec_op_ctx *ctx;
236 struct qm_sg_entry *sg, *out_sg, *in_sg;
237 uint8_t extra_req_segs;
238 uint8_t *IV_ptr = iv->va;
239 int data_len = 0, aead_len = 0;
240 unsigned int i;
241
242 for (i = 0; i < sgl->num; i++)
243 data_len += sgl->vec[i].len;
244
245 extra_req_segs = 4;
246 aead_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
247
248 if (ses->auth_only_len)
249 extra_req_segs++;
250
251 if (sgl->num > MAX_SG_ENTRIES) {
252 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
253 MAX_SG_ENTRIES);
254 return NULL;
255 }
256
257 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + extra_req_segs);
258 if (!ctx)
259 return NULL;
260
261 cf = &ctx->job;
262 ctx->userdata = (void *)userdata;
263
264 rte_prefetch0(cf->sg);
265
266 /* output */
267 out_sg = &cf->sg[0];
268 out_sg->extension = 1;
269 if (is_encode(ses))
270 out_sg->length = aead_len + ses->digest_length;
271 else
272 out_sg->length = aead_len;
273
274 /* output sg entries */
275 sg = &cf->sg[2];
276 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
277 cpu_to_hw_sg(out_sg);
278
279 if (dest_sgl) {
280 /* 1st seg */
281 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
282 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head;
283 sg->offset = ofs.ofs.cipher.head;
284
285 /* Successive segs */
286 for (i = 1; i < dest_sgl->num; i++) {
287 cpu_to_hw_sg(sg);
288 sg++;
289 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
290 sg->length = dest_sgl->vec[i].len;
291 }
292 } else {
293 /* 1st seg */
294 qm_sg_entry_set64(sg, sgl->vec[0].iova);
295 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
296 sg->offset = ofs.ofs.cipher.head;
297
298 /* Successive segs */
299 for (i = 1; i < sgl->num; i++) {
300 cpu_to_hw_sg(sg);
301 sg++;
302 qm_sg_entry_set64(sg, sgl->vec[i].iova);
303 sg->length = sgl->vec[i].len;
304 }
305
306 }
307
308 if (is_encode(ses)) {
309 cpu_to_hw_sg(sg);
310 /* set auth output */
311 sg++;
312 qm_sg_entry_set64(sg, digest->iova);
313 sg->length = ses->digest_length;
314 }
315 sg->final = 1;
316 cpu_to_hw_sg(sg);
317
318 /* input */
319 in_sg = &cf->sg[1];
320 in_sg->extension = 1;
321 in_sg->final = 1;
322 if (is_encode(ses))
323 in_sg->length = ses->iv.length + aead_len
324 + ses->auth_only_len;
325 else
326 in_sg->length = ses->iv.length + aead_len
327 + ses->auth_only_len + ses->digest_length;
328
329 /* input sg entries */
330 sg++;
331 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
332 cpu_to_hw_sg(in_sg);
333
334 /* 1st seg IV */
335 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
336 sg->length = ses->iv.length;
337 cpu_to_hw_sg(sg);
338
339 /* 2 seg auth only */
340 if (ses->auth_only_len) {
341 sg++;
342 qm_sg_entry_set64(sg, auth_iv->iova);
343 sg->length = ses->auth_only_len;
344 cpu_to_hw_sg(sg);
345 }
346
347 /* 3rd seg */
348 sg++;
349 qm_sg_entry_set64(sg, sgl->vec[0].iova);
350 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
351 sg->offset = ofs.ofs.cipher.head;
352
353 /* Successive segs */
354 for (i = 1; i < sgl->num; i++) {
355 cpu_to_hw_sg(sg);
356 sg++;
357 qm_sg_entry_set64(sg, sgl->vec[i].iova);
358 sg->length = sgl->vec[i].len;
359 }
360
361 if (is_decode(ses)) {
362 cpu_to_hw_sg(sg);
363 sg++;
364 memcpy(ctx->digest, digest->va,
365 ses->digest_length);
366 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
367 sg->length = ses->digest_length;
368 }
369 sg->final = 1;
370 cpu_to_hw_sg(sg);
371
372 if (ses->auth_only_len)
373 fd->cmd = 0x80000000 | ses->auth_only_len;
374
375 return cf;
376 }
377
378 static inline struct dpaa_sec_job *
build_dpaa_raw_dp_chain_fd(uint8_t * drv_ctx,struct rte_crypto_sgl * sgl,struct rte_crypto_sgl * dest_sgl,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,void * userdata,struct qm_fd * fd)379 build_dpaa_raw_dp_chain_fd(uint8_t *drv_ctx,
380 struct rte_crypto_sgl *sgl,
381 struct rte_crypto_sgl *dest_sgl,
382 struct rte_crypto_va_iova_ptr *iv,
383 struct rte_crypto_va_iova_ptr *digest,
384 struct rte_crypto_va_iova_ptr *auth_iv,
385 union rte_crypto_sym_ofs ofs,
386 void *userdata,
387 struct qm_fd *fd)
388 {
389 RTE_SET_USED(auth_iv);
390
391 dpaa_sec_session *ses =
392 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
393 struct dpaa_sec_job *cf;
394 struct dpaa_sec_op_ctx *ctx;
395 struct qm_sg_entry *sg, *out_sg, *in_sg;
396 uint8_t *IV_ptr = iv->va;
397 unsigned int i;
398 uint16_t auth_hdr_len = ofs.ofs.cipher.head -
399 ofs.ofs.auth.head;
400 uint16_t auth_tail_len;
401 uint32_t auth_only_len;
402 int data_len = 0, auth_len = 0, cipher_len = 0;
403
404 for (i = 0; i < sgl->num; i++)
405 data_len += sgl->vec[i].len;
406
407 cipher_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
408 auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
409 auth_tail_len = auth_len - cipher_len - auth_hdr_len;
410 auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
411
412 if (sgl->num > MAX_SG_ENTRIES) {
413 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
414 MAX_SG_ENTRIES);
415 return NULL;
416 }
417
418 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4);
419 if (!ctx)
420 return NULL;
421
422 cf = &ctx->job;
423 ctx->userdata = (void *)userdata;
424
425 rte_prefetch0(cf->sg);
426
427 /* output */
428 out_sg = &cf->sg[0];
429 out_sg->extension = 1;
430 if (is_encode(ses))
431 out_sg->length = cipher_len + ses->digest_length;
432 else
433 out_sg->length = cipher_len;
434
435 /* output sg entries */
436 sg = &cf->sg[2];
437 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
438 cpu_to_hw_sg(out_sg);
439
440 /* 1st seg */
441 if (dest_sgl) {
442 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
443 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head;
444 sg->offset = ofs.ofs.cipher.head;
445
446 /* Successive segs */
447 for (i = 1; i < dest_sgl->num; i++) {
448 cpu_to_hw_sg(sg);
449 sg++;
450 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
451 sg->length = dest_sgl->vec[i].len;
452 }
453 sg->length -= ofs.ofs.cipher.tail;
454 } else {
455 qm_sg_entry_set64(sg, sgl->vec[0].iova);
456 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
457 sg->offset = ofs.ofs.cipher.head;
458
459 /* Successive segs */
460 for (i = 1; i < sgl->num; i++) {
461 cpu_to_hw_sg(sg);
462 sg++;
463 qm_sg_entry_set64(sg, sgl->vec[i].iova);
464 sg->length = sgl->vec[i].len;
465 }
466 sg->length -= ofs.ofs.cipher.tail;
467 }
468
469 if (is_encode(ses)) {
470 cpu_to_hw_sg(sg);
471 /* set auth output */
472 sg++;
473 qm_sg_entry_set64(sg, digest->iova);
474 sg->length = ses->digest_length;
475 }
476 sg->final = 1;
477 cpu_to_hw_sg(sg);
478
479 /* input */
480 in_sg = &cf->sg[1];
481 in_sg->extension = 1;
482 in_sg->final = 1;
483 if (is_encode(ses))
484 in_sg->length = ses->iv.length + auth_len;
485 else
486 in_sg->length = ses->iv.length + auth_len
487 + ses->digest_length;
488
489 /* input sg entries */
490 sg++;
491 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
492 cpu_to_hw_sg(in_sg);
493
494 /* 1st seg IV */
495 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
496 sg->length = ses->iv.length;
497 cpu_to_hw_sg(sg);
498
499 /* 2 seg */
500 sg++;
501 qm_sg_entry_set64(sg, sgl->vec[0].iova);
502 sg->length = sgl->vec[0].len - ofs.ofs.auth.head;
503 sg->offset = ofs.ofs.auth.head;
504
505 /* Successive segs */
506 for (i = 1; i < sgl->num; i++) {
507 cpu_to_hw_sg(sg);
508 sg++;
509 qm_sg_entry_set64(sg, sgl->vec[i].iova);
510 sg->length = sgl->vec[i].len;
511 }
512
513 if (is_decode(ses)) {
514 cpu_to_hw_sg(sg);
515 sg++;
516 memcpy(ctx->digest, digest->va,
517 ses->digest_length);
518 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
519 sg->length = ses->digest_length;
520 }
521 sg->final = 1;
522 cpu_to_hw_sg(sg);
523
524 if (auth_only_len)
525 fd->cmd = 0x80000000 | auth_only_len;
526
527 return cf;
528 }
529
530 static struct dpaa_sec_job *
build_dpaa_raw_dp_cipher_fd(uint8_t * drv_ctx,struct rte_crypto_sgl * sgl,struct rte_crypto_sgl * dest_sgl,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,void * userdata,struct qm_fd * fd)531 build_dpaa_raw_dp_cipher_fd(uint8_t *drv_ctx,
532 struct rte_crypto_sgl *sgl,
533 struct rte_crypto_sgl *dest_sgl,
534 struct rte_crypto_va_iova_ptr *iv,
535 struct rte_crypto_va_iova_ptr *digest,
536 struct rte_crypto_va_iova_ptr *auth_iv,
537 union rte_crypto_sym_ofs ofs,
538 void *userdata,
539 struct qm_fd *fd)
540 {
541 RTE_SET_USED(digest);
542 RTE_SET_USED(auth_iv);
543 RTE_SET_USED(fd);
544
545 dpaa_sec_session *ses =
546 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
547 struct dpaa_sec_job *cf;
548 struct dpaa_sec_op_ctx *ctx;
549 struct qm_sg_entry *sg, *out_sg, *in_sg;
550 unsigned int i;
551 uint8_t *IV_ptr = iv->va;
552 int data_len, total_len = 0, data_offset;
553
554 for (i = 0; i < sgl->num; i++)
555 total_len += sgl->vec[i].len;
556
557 data_len = total_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
558 data_offset = ofs.ofs.cipher.head;
559
560 /* Support lengths in bits only for SNOW3G and ZUC */
561 if (sgl->num > MAX_SG_ENTRIES) {
562 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
563 MAX_SG_ENTRIES);
564 return NULL;
565 }
566
567 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 3);
568 if (!ctx)
569 return NULL;
570
571 cf = &ctx->job;
572 ctx->userdata = (void *)userdata;
573
574 /* output */
575 out_sg = &cf->sg[0];
576 out_sg->extension = 1;
577 out_sg->length = data_len;
578 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
579 cpu_to_hw_sg(out_sg);
580
581 if (dest_sgl) {
582 /* 1st seg */
583 sg = &cf->sg[2];
584 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
585 sg->length = dest_sgl->vec[0].len - data_offset;
586 sg->offset = data_offset;
587
588 /* Successive segs */
589 for (i = 1; i < dest_sgl->num; i++) {
590 cpu_to_hw_sg(sg);
591 sg++;
592 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
593 sg->length = dest_sgl->vec[i].len;
594 }
595 } else {
596 /* 1st seg */
597 sg = &cf->sg[2];
598 qm_sg_entry_set64(sg, sgl->vec[0].iova);
599 sg->length = sgl->vec[0].len - data_offset;
600 sg->offset = data_offset;
601
602 /* Successive segs */
603 for (i = 1; i < sgl->num; i++) {
604 cpu_to_hw_sg(sg);
605 sg++;
606 qm_sg_entry_set64(sg, sgl->vec[i].iova);
607 sg->length = sgl->vec[i].len;
608 }
609
610 }
611 sg->final = 1;
612 cpu_to_hw_sg(sg);
613
614 /* input */
615 in_sg = &cf->sg[1];
616 in_sg->extension = 1;
617 in_sg->final = 1;
618 in_sg->length = data_len + ses->iv.length;
619
620 sg++;
621 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
622 cpu_to_hw_sg(in_sg);
623
624 /* IV */
625 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
626 sg->length = ses->iv.length;
627 cpu_to_hw_sg(sg);
628
629 /* 1st seg */
630 sg++;
631 qm_sg_entry_set64(sg, sgl->vec[0].iova);
632 sg->length = sgl->vec[0].len - data_offset;
633 sg->offset = data_offset;
634
635 /* Successive segs */
636 for (i = 1; i < sgl->num; i++) {
637 cpu_to_hw_sg(sg);
638 sg++;
639 qm_sg_entry_set64(sg, sgl->vec[i].iova);
640 sg->length = sgl->vec[i].len;
641 }
642 sg->final = 1;
643 cpu_to_hw_sg(sg);
644
645 return cf;
646 }
647
648 #ifdef RTE_LIB_SECURITY
649 static inline struct dpaa_sec_job *
build_dpaa_raw_proto_sg(uint8_t * drv_ctx,struct rte_crypto_sgl * sgl,struct rte_crypto_sgl * dest_sgl,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * auth_iv,union rte_crypto_sym_ofs ofs,void * userdata,struct qm_fd * fd)650 build_dpaa_raw_proto_sg(uint8_t *drv_ctx,
651 struct rte_crypto_sgl *sgl,
652 struct rte_crypto_sgl *dest_sgl,
653 struct rte_crypto_va_iova_ptr *iv,
654 struct rte_crypto_va_iova_ptr *digest,
655 struct rte_crypto_va_iova_ptr *auth_iv,
656 union rte_crypto_sym_ofs ofs,
657 void *userdata,
658 struct qm_fd *fd)
659 {
660 RTE_SET_USED(iv);
661 RTE_SET_USED(digest);
662 RTE_SET_USED(auth_iv);
663 RTE_SET_USED(ofs);
664
665 dpaa_sec_session *ses =
666 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
667 struct dpaa_sec_job *cf;
668 struct dpaa_sec_op_ctx *ctx;
669 struct qm_sg_entry *sg, *out_sg, *in_sg;
670 uint32_t in_len = 0, out_len = 0;
671 unsigned int i;
672
673 if (sgl->num > MAX_SG_ENTRIES) {
674 DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
675 MAX_SG_ENTRIES);
676 return NULL;
677 }
678
679 ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4);
680 if (!ctx)
681 return NULL;
682 cf = &ctx->job;
683 ctx->userdata = (void *)userdata;
684 /* output */
685 out_sg = &cf->sg[0];
686 out_sg->extension = 1;
687 qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
688
689 if (dest_sgl) {
690 /* 1st seg */
691 sg = &cf->sg[2];
692 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
693 sg->offset = 0;
694 sg->length = dest_sgl->vec[0].len;
695 out_len += sg->length;
696
697 for (i = 1; i < dest_sgl->num; i++) {
698 /* Successive segs */
699 cpu_to_hw_sg(sg);
700 sg++;
701 qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
702 sg->offset = 0;
703 sg->length = dest_sgl->vec[i].len;
704 out_len += sg->length;
705 }
706 sg->length = dest_sgl->vec[i - 1].tot_len;
707 } else {
708 /* 1st seg */
709 sg = &cf->sg[2];
710 qm_sg_entry_set64(sg, sgl->vec[0].iova);
711 sg->offset = 0;
712 sg->length = sgl->vec[0].len;
713 out_len += sg->length;
714
715 for (i = 1; i < sgl->num; i++) {
716 /* Successive segs */
717 cpu_to_hw_sg(sg);
718 sg++;
719 qm_sg_entry_set64(sg, sgl->vec[i].iova);
720 sg->offset = 0;
721 sg->length = sgl->vec[i].len;
722 out_len += sg->length;
723 }
724 sg->length = sgl->vec[i - 1].tot_len;
725
726 }
727 out_len += sg->length;
728 sg->final = 1;
729 cpu_to_hw_sg(sg);
730
731 out_sg->length = out_len;
732 cpu_to_hw_sg(out_sg);
733
734 /* input */
735 in_sg = &cf->sg[1];
736 in_sg->extension = 1;
737 in_sg->final = 1;
738 in_len = sgl->vec[0].len;
739
740 sg++;
741 qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
742
743 /* 1st seg */
744 qm_sg_entry_set64(sg, sgl->vec[0].iova);
745 sg->length = sgl->vec[0].len;
746 sg->offset = 0;
747
748 /* Successive segs */
749 for (i = 1; i < sgl->num; i++) {
750 cpu_to_hw_sg(sg);
751 sg++;
752 qm_sg_entry_set64(sg, sgl->vec[i].iova);
753 sg->length = sgl->vec[i].len;
754 sg->offset = 0;
755 in_len += sg->length;
756 }
757 sg->final = 1;
758 cpu_to_hw_sg(sg);
759
760 in_sg->length = in_len;
761 cpu_to_hw_sg(in_sg);
762
763 if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) {
764 fd->cmd = 0x80000000 |
765 *((uint32_t *)((uint8_t *)userdata +
766 ses->pdcp.hfn_ovd_offset));
767 DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u\n",
768 *((uint32_t *)((uint8_t *)userdata +
769 ses->pdcp.hfn_ovd_offset)),
770 ses->pdcp.hfn_ovd);
771 }
772
773 return cf;
774 }
775 #endif
776
777 static uint32_t
dpaa_sec_raw_enqueue_burst(void * qp_data,uint8_t * drv_ctx,struct rte_crypto_sym_vec * vec,union rte_crypto_sym_ofs ofs,void * user_data[],int * status)778 dpaa_sec_raw_enqueue_burst(void *qp_data, uint8_t *drv_ctx,
779 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
780 void *user_data[], int *status)
781 {
782 /* Function to transmit the frames to given device and queuepair */
783 uint32_t loop;
784 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data;
785 uint16_t num_tx = 0;
786 struct qm_fd fds[DPAA_SEC_BURST], *fd;
787 uint32_t frames_to_send;
788 struct dpaa_sec_job *cf;
789 dpaa_sec_session *ses =
790 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
791 uint32_t flags[DPAA_SEC_BURST] = {0};
792 struct qman_fq *inq[DPAA_SEC_BURST];
793
794 if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
795 if (rte_dpaa_portal_init((void *)0)) {
796 DPAA_SEC_ERR("Failure in affining portal");
797 return 0;
798 }
799 }
800
801 while (vec->num) {
802 frames_to_send = (vec->num > DPAA_SEC_BURST) ?
803 DPAA_SEC_BURST : vec->num;
804 for (loop = 0; loop < frames_to_send; loop++) {
805 if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) {
806 if (dpaa_sec_attach_sess_q(dpaa_qp, ses)) {
807 frames_to_send = loop;
808 goto send_pkts;
809 }
810 } else if (unlikely(ses->qp[rte_lcore_id() %
811 MAX_DPAA_CORES] != dpaa_qp)) {
812 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
813 " New qp = %p\n",
814 ses->qp[rte_lcore_id() %
815 MAX_DPAA_CORES], dpaa_qp);
816 frames_to_send = loop;
817 goto send_pkts;
818 }
819
820 /*Clear the unused FD fields before sending*/
821 fd = &fds[loop];
822 memset(fd, 0, sizeof(struct qm_fd));
823 cf = ses->build_raw_dp_fd(drv_ctx,
824 &vec->src_sgl[loop],
825 &vec->dest_sgl[loop],
826 &vec->iv[loop],
827 &vec->digest[loop],
828 &vec->auth_iv[loop],
829 ofs,
830 user_data[loop],
831 fd);
832 if (!cf) {
833 DPAA_SEC_ERR("error: Improper packet contents"
834 " for crypto operation");
835 goto skip_tx;
836 }
837 inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
838 qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg));
839 fd->_format1 = qm_fd_compound;
840 fd->length29 = 2 * sizeof(struct qm_sg_entry);
841
842 status[loop] = 1;
843 }
844 send_pkts:
845 loop = 0;
846 while (loop < frames_to_send) {
847 loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
848 &flags[loop], frames_to_send - loop);
849 }
850 vec->num -= frames_to_send;
851 num_tx += frames_to_send;
852 }
853
854 skip_tx:
855 dpaa_qp->tx_pkts += num_tx;
856 dpaa_qp->tx_errs += vec->num - num_tx;
857
858 return num_tx;
859 }
860
861 static int
dpaa_sec_deq_raw(struct dpaa_sec_qp * qp,void ** out_user_data,uint8_t is_user_data_array,rte_cryptodev_raw_post_dequeue_t post_dequeue,int nb_ops)862 dpaa_sec_deq_raw(struct dpaa_sec_qp *qp, void **out_user_data,
863 uint8_t is_user_data_array,
864 rte_cryptodev_raw_post_dequeue_t post_dequeue,
865 int nb_ops)
866 {
867 struct qman_fq *fq;
868 unsigned int pkts = 0;
869 int num_rx_bufs, ret;
870 struct qm_dqrr_entry *dq;
871 uint32_t vdqcr_flags = 0;
872 uint8_t is_success = 0;
873
874 fq = &qp->outq;
875 /*
876 * Until request for four buffers, we provide exact number of buffers.
877 * Otherwise we do not set the QM_VDQCR_EXACT flag.
878 * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
879 * requested, so we request two less in this case.
880 */
881 if (nb_ops < 4) {
882 vdqcr_flags = QM_VDQCR_EXACT;
883 num_rx_bufs = nb_ops;
884 } else {
885 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
886 (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
887 }
888 ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
889 if (ret)
890 return 0;
891
892 do {
893 const struct qm_fd *fd;
894 struct dpaa_sec_job *job;
895 struct dpaa_sec_op_ctx *ctx;
896
897 dq = qman_dequeue(fq);
898 if (!dq)
899 continue;
900
901 fd = &dq->fd;
902 /* sg is embedded in an op ctx,
903 * sg[0] is for output
904 * sg[1] for input
905 */
906 job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
907
908 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
909 ctx->fd_status = fd->status;
910 if (is_user_data_array)
911 out_user_data[pkts] = ctx->userdata;
912 else
913 out_user_data[0] = ctx->userdata;
914
915 if (!ctx->fd_status) {
916 is_success = true;
917 } else {
918 is_success = false;
919 DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
920 }
921 post_dequeue(ctx->op, pkts, is_success);
922 pkts++;
923
924 /* report op status to sym->op and then free the ctx memory */
925 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
926
927 qman_dqrr_consume(fq, dq);
928 } while (fq->flags & QMAN_FQ_STATE_VDQCR);
929
930 return pkts;
931 }
932
933
934 static __rte_always_inline uint32_t
dpaa_sec_raw_dequeue_burst(void * qp_data,uint8_t * drv_ctx,rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,uint32_t max_nb_to_dequeue,rte_cryptodev_raw_post_dequeue_t post_dequeue,void ** out_user_data,uint8_t is_user_data_array,uint32_t * n_success,int * dequeue_status)935 dpaa_sec_raw_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
936 rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
937 uint32_t max_nb_to_dequeue,
938 rte_cryptodev_raw_post_dequeue_t post_dequeue,
939 void **out_user_data, uint8_t is_user_data_array,
940 uint32_t *n_success, int *dequeue_status)
941 {
942 RTE_SET_USED(drv_ctx);
943 RTE_SET_USED(get_dequeue_count);
944 uint16_t num_rx;
945 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data;
946 uint32_t nb_ops = max_nb_to_dequeue;
947
948 if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
949 if (rte_dpaa_portal_init((void *)0)) {
950 DPAA_SEC_ERR("Failure in affining portal");
951 return 0;
952 }
953 }
954
955 num_rx = dpaa_sec_deq_raw(dpaa_qp, out_user_data,
956 is_user_data_array, post_dequeue, nb_ops);
957
958 dpaa_qp->rx_pkts += num_rx;
959 *dequeue_status = 1;
960 *n_success = num_rx;
961
962 DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
963
964 return num_rx;
965 }
966
967 static __rte_always_inline int
dpaa_sec_raw_enqueue(void * qp_data,uint8_t * drv_ctx,struct rte_crypto_vec * data_vec,uint16_t n_data_vecs,union rte_crypto_sym_ofs ofs,struct rte_crypto_va_iova_ptr * iv,struct rte_crypto_va_iova_ptr * digest,struct rte_crypto_va_iova_ptr * aad_or_auth_iv,void * user_data)968 dpaa_sec_raw_enqueue(void *qp_data, uint8_t *drv_ctx,
969 struct rte_crypto_vec *data_vec,
970 uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs,
971 struct rte_crypto_va_iova_ptr *iv,
972 struct rte_crypto_va_iova_ptr *digest,
973 struct rte_crypto_va_iova_ptr *aad_or_auth_iv,
974 void *user_data)
975 {
976 RTE_SET_USED(qp_data);
977 RTE_SET_USED(drv_ctx);
978 RTE_SET_USED(data_vec);
979 RTE_SET_USED(n_data_vecs);
980 RTE_SET_USED(ofs);
981 RTE_SET_USED(iv);
982 RTE_SET_USED(digest);
983 RTE_SET_USED(aad_or_auth_iv);
984 RTE_SET_USED(user_data);
985
986 return 0;
987 }
988
989 static __rte_always_inline void *
dpaa_sec_raw_dequeue(void * qp_data,uint8_t * drv_ctx,int * dequeue_status,enum rte_crypto_op_status * op_status)990 dpaa_sec_raw_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
991 enum rte_crypto_op_status *op_status)
992 {
993 RTE_SET_USED(qp_data);
994 RTE_SET_USED(drv_ctx);
995 RTE_SET_USED(dequeue_status);
996 RTE_SET_USED(op_status);
997
998 return NULL;
999 }
1000
1001 int
dpaa_sec_configure_raw_dp_ctx(struct rte_cryptodev * dev,uint16_t qp_id,struct rte_crypto_raw_dp_ctx * raw_dp_ctx,enum rte_crypto_op_sess_type sess_type,union rte_cryptodev_session_ctx session_ctx,uint8_t is_update)1002 dpaa_sec_configure_raw_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
1003 struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
1004 enum rte_crypto_op_sess_type sess_type,
1005 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
1006 {
1007 dpaa_sec_session *sess;
1008 struct dpaa_sec_raw_dp_ctx *dp_ctx;
1009 RTE_SET_USED(qp_id);
1010
1011 if (!is_update) {
1012 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx));
1013 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
1014 }
1015
1016 if (sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1017 sess = (dpaa_sec_session *)get_sec_session_private_data(
1018 session_ctx.sec_sess);
1019 else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1020 sess = (dpaa_sec_session *)get_sym_session_private_data(
1021 session_ctx.crypto_sess, dpaa_cryptodev_driver_id);
1022 else
1023 return -ENOTSUP;
1024 raw_dp_ctx->dequeue_burst = dpaa_sec_raw_dequeue_burst;
1025 raw_dp_ctx->dequeue = dpaa_sec_raw_dequeue;
1026 raw_dp_ctx->dequeue_done = dpaa_sec_raw_dequeue_done;
1027 raw_dp_ctx->enqueue_burst = dpaa_sec_raw_enqueue_burst;
1028 raw_dp_ctx->enqueue = dpaa_sec_raw_enqueue;
1029 raw_dp_ctx->enqueue_done = dpaa_sec_raw_enqueue_done;
1030
1031 if (sess->ctxt == DPAA_SEC_CIPHER)
1032 sess->build_raw_dp_fd = build_dpaa_raw_dp_cipher_fd;
1033 else if (sess->ctxt == DPAA_SEC_AUTH)
1034 sess->build_raw_dp_fd = build_dpaa_raw_dp_auth_fd;
1035 else if (sess->ctxt == DPAA_SEC_CIPHER_HASH)
1036 sess->build_raw_dp_fd = build_dpaa_raw_dp_chain_fd;
1037 else if (sess->ctxt == DPAA_SEC_AEAD)
1038 sess->build_raw_dp_fd = build_raw_cipher_auth_gcm_sg;
1039 #ifdef RTE_LIB_SECURITY
1040 else if (sess->ctxt == DPAA_SEC_IPSEC ||
1041 sess->ctxt == DPAA_SEC_PDCP)
1042 sess->build_raw_dp_fd = build_dpaa_raw_proto_sg;
1043 #endif
1044 else
1045 return -ENOTSUP;
1046 dp_ctx = (struct dpaa_sec_raw_dp_ctx *)raw_dp_ctx->drv_ctx_data;
1047 dp_ctx->session = sess;
1048
1049 return 0;
1050 }
1051
1052 int
dpaa_sec_get_dp_ctx_size(__rte_unused struct rte_cryptodev * dev)1053 dpaa_sec_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
1054 {
1055 return sizeof(struct dpaa_sec_raw_dp_ctx);
1056 }
1057