1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2020 Intel Corporation
3 */
4
5 #include <rte_ipsec.h>
6 #include <rte_esp.h>
7 #include <rte_udp.h>
8 #include <rte_errno.h>
9 #include <rte_cryptodev.h>
10
11 #include "sa.h"
12 #include "ipsec_sqn.h"
13 #include "crypto.h"
14 #include "iph.h"
15 #include "misc.h"
16 #include "pad.h"
17
18 typedef int32_t (*esp_outb_prepare_t)(struct rte_ipsec_sa *sa, rte_be64_t sqc,
19 const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
20 union sym_op_data *icv, uint8_t sqh_len, uint8_t tso);
21
22 /*
23 * helper function to fill crypto_sym op for cipher+auth algorithms.
24 * used by outb_cop_prepare(), see below.
25 */
26 static inline void
sop_ciph_auth_prepare(struct rte_crypto_sym_op * sop,const struct rte_ipsec_sa * sa,const union sym_op_data * icv,uint32_t pofs,uint32_t plen)27 sop_ciph_auth_prepare(struct rte_crypto_sym_op *sop,
28 const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
29 uint32_t pofs, uint32_t plen)
30 {
31 sop->cipher.data.offset = sa->ctp.cipher.offset + pofs;
32 sop->cipher.data.length = sa->ctp.cipher.length + plen;
33 sop->auth.data.offset = sa->ctp.auth.offset + pofs;
34 sop->auth.data.length = sa->ctp.auth.length + plen;
35 sop->auth.digest.data = icv->va;
36 sop->auth.digest.phys_addr = icv->pa;
37 }
38
39 /*
40 * helper function to fill crypto_sym op for cipher+auth algorithms.
41 * used by outb_cop_prepare(), see below.
42 */
43 static inline void
sop_aead_prepare(struct rte_crypto_sym_op * sop,const struct rte_ipsec_sa * sa,const union sym_op_data * icv,uint32_t pofs,uint32_t plen)44 sop_aead_prepare(struct rte_crypto_sym_op *sop,
45 const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
46 uint32_t pofs, uint32_t plen)
47 {
48 sop->aead.data.offset = sa->ctp.cipher.offset + pofs;
49 sop->aead.data.length = sa->ctp.cipher.length + plen;
50 sop->aead.digest.data = icv->va;
51 sop->aead.digest.phys_addr = icv->pa;
52 sop->aead.aad.data = icv->va + sa->icv_len;
53 sop->aead.aad.phys_addr = icv->pa + sa->icv_len;
54 }
55
56 /*
57 * setup crypto op and crypto sym op for ESP outbound packet.
58 */
59 static inline void
outb_cop_prepare(struct rte_crypto_op * cop,const struct rte_ipsec_sa * sa,const uint64_t ivp[IPSEC_MAX_IV_QWORD],const union sym_op_data * icv,uint32_t hlen,uint32_t plen)60 outb_cop_prepare(struct rte_crypto_op *cop,
61 const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD],
62 const union sym_op_data *icv, uint32_t hlen, uint32_t plen)
63 {
64 struct rte_crypto_sym_op *sop;
65 struct aead_gcm_iv *gcm;
66 struct aead_ccm_iv *ccm;
67 struct aead_chacha20_poly1305_iv *chacha20_poly1305;
68 struct aesctr_cnt_blk *ctr;
69 uint32_t algo;
70
71 algo = sa->algo_type;
72
73 /* fill sym op fields */
74 sop = cop->sym;
75
76 switch (algo) {
77 case ALGO_TYPE_AES_CBC:
78 /* Cipher-Auth (AES-CBC *) case */
79 case ALGO_TYPE_3DES_CBC:
80 /* Cipher-Auth (3DES-CBC *) case */
81 case ALGO_TYPE_NULL:
82 /* NULL case */
83 sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);
84 break;
85 case ALGO_TYPE_AES_GMAC:
86 /* GMAC case */
87 sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);
88
89 /* fill AAD IV (located inside crypto op) */
90 gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
91 sa->iv_ofs);
92 aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
93 break;
94 case ALGO_TYPE_AES_GCM:
95 /* AEAD (AES_GCM) case */
96 sop_aead_prepare(sop, sa, icv, hlen, plen);
97
98 /* fill AAD IV (located inside crypto op) */
99 gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
100 sa->iv_ofs);
101 aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
102 break;
103 case ALGO_TYPE_AES_CCM:
104 /* AEAD (AES_CCM) case */
105 sop_aead_prepare(sop, sa, icv, hlen, plen);
106
107 /* fill AAD IV (located inside crypto op) */
108 ccm = rte_crypto_op_ctod_offset(cop, struct aead_ccm_iv *,
109 sa->iv_ofs);
110 aead_ccm_iv_fill(ccm, ivp[0], sa->salt);
111 break;
112 case ALGO_TYPE_CHACHA20_POLY1305:
113 /* AEAD (CHACHA20_POLY) case */
114 sop_aead_prepare(sop, sa, icv, hlen, plen);
115
116 /* fill AAD IV (located inside crypto op) */
117 chacha20_poly1305 = rte_crypto_op_ctod_offset(cop,
118 struct aead_chacha20_poly1305_iv *,
119 sa->iv_ofs);
120 aead_chacha20_poly1305_iv_fill(chacha20_poly1305,
121 ivp[0], sa->salt);
122 break;
123 case ALGO_TYPE_AES_CTR:
124 /* Cipher-Auth (AES-CTR *) case */
125 sop_ciph_auth_prepare(sop, sa, icv, hlen, plen);
126
127 /* fill CTR block (located inside crypto op) */
128 ctr = rte_crypto_op_ctod_offset(cop, struct aesctr_cnt_blk *,
129 sa->iv_ofs);
130 aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
131 break;
132 }
133 }
134
135 /*
136 * setup/update packet data and metadata for ESP outbound tunnel case.
137 */
138 static inline int32_t
outb_tun_pkt_prepare(struct rte_ipsec_sa * sa,rte_be64_t sqc,const uint64_t ivp[IPSEC_MAX_IV_QWORD],struct rte_mbuf * mb,union sym_op_data * icv,uint8_t sqh_len,uint8_t tso)139 outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
140 const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
141 union sym_op_data *icv, uint8_t sqh_len, uint8_t tso)
142 {
143 uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen;
144 struct rte_mbuf *ml;
145 struct rte_esp_hdr *esph;
146 struct rte_esp_tail *espt;
147 char *ph, *pt;
148 uint64_t *iv;
149
150 /* calculate extra header space required */
151 hlen = sa->hdr_len + sa->iv_len + sizeof(*esph);
152
153 /* size of ipsec protected data */
154 l2len = mb->l2_len;
155 plen = mb->pkt_len - l2len;
156
157 /* number of bytes to encrypt */
158 clen = plen + sizeof(*espt);
159
160 if (!tso) {
161 clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
162 /* pad length + esp tail */
163 pdlen = clen - plen;
164 tlen = pdlen + sa->icv_len + sqh_len;
165 } else {
166 /* We don't need to pad/align packet or append ICV length
167 * when using TSO offload
168 */
169 pdlen = clen - plen;
170 tlen = pdlen + sqh_len;
171 }
172
173 /* do append and prepend */
174 ml = rte_pktmbuf_lastseg(mb);
175 if (tlen + sa->aad_len > rte_pktmbuf_tailroom(ml))
176 return -ENOSPC;
177
178 /* prepend header */
179 ph = rte_pktmbuf_prepend(mb, hlen - l2len);
180 if (ph == NULL)
181 return -ENOSPC;
182
183 /* append tail */
184 pdofs = ml->data_len;
185 ml->data_len += tlen;
186 mb->pkt_len += tlen;
187 pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
188
189 /* update pkt l2/l3 len */
190 mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) |
191 sa->tx_offload.val;
192
193 /* copy tunnel pkt header */
194 rte_memcpy(ph, sa->hdr, sa->hdr_len);
195
196 /* if UDP encap is enabled update the dgram_len */
197 if (sa->type & RTE_IPSEC_SATP_NATT_ENABLE) {
198 struct rte_udp_hdr *udph = (struct rte_udp_hdr *)
199 (ph - sizeof(struct rte_udp_hdr));
200 udph->dgram_len = rte_cpu_to_be_16(mb->pkt_len - sqh_len -
201 sa->hdr_l3_off - sa->hdr_len);
202 }
203
204 /* update original and new ip header fields */
205 update_tun_outb_l3hdr(sa, ph + sa->hdr_l3_off, ph + hlen,
206 mb->pkt_len - sqh_len, sa->hdr_l3_off, sqn_low16(sqc));
207
208 /* update spi, seqn and iv */
209 esph = (struct rte_esp_hdr *)(ph + sa->hdr_len);
210 iv = (uint64_t *)(esph + 1);
211 copy_iv(iv, ivp, sa->iv_len);
212
213 esph->spi = sa->spi;
214 esph->seq = sqn_low32(sqc);
215
216 /* offset for ICV */
217 pdofs += pdlen + sa->sqh_len;
218
219 /* pad length */
220 pdlen -= sizeof(*espt);
221
222 /* copy padding data */
223 rte_memcpy(pt, esp_pad_bytes, pdlen);
224
225 /* update esp trailer */
226 espt = (struct rte_esp_tail *)(pt + pdlen);
227 espt->pad_len = pdlen;
228 espt->next_proto = sa->proto;
229
230 /* set icv va/pa value(s) */
231 icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
232 icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
233
234 return clen;
235 }
236
237 /*
238 * for pure cryptodev (lookaside none) depending on SA settings,
239 * we might have to write some extra data to the packet.
240 */
241 static inline void
outb_pkt_xprepare(const struct rte_ipsec_sa * sa,rte_be64_t sqc,const union sym_op_data * icv)242 outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
243 const union sym_op_data *icv)
244 {
245 uint32_t *psqh;
246 struct aead_gcm_aad *gaad;
247 struct aead_ccm_aad *caad;
248 struct aead_chacha20_poly1305_aad *chacha20_poly1305_aad;
249
250 /* insert SQN.hi between ESP trailer and ICV */
251 if (sa->sqh_len != 0) {
252 psqh = (uint32_t *)(icv->va - sa->sqh_len);
253 psqh[0] = sqn_hi32(sqc);
254 }
255
256 /*
257 * fill IV and AAD fields, if any (aad fields are placed after icv),
258 * right now we support only one AEAD algorithm: AES-GCM .
259 */
260 switch (sa->algo_type) {
261 case ALGO_TYPE_AES_GCM:
262 if (sa->aad_len != 0) {
263 gaad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
264 aead_gcm_aad_fill(gaad, sa->spi, sqc, IS_ESN(sa));
265 }
266 break;
267 case ALGO_TYPE_AES_CCM:
268 if (sa->aad_len != 0) {
269 caad = (struct aead_ccm_aad *)(icv->va + sa->icv_len);
270 aead_ccm_aad_fill(caad, sa->spi, sqc, IS_ESN(sa));
271 }
272 break;
273 case ALGO_TYPE_CHACHA20_POLY1305:
274 if (sa->aad_len != 0) {
275 chacha20_poly1305_aad = (struct aead_chacha20_poly1305_aad *)
276 (icv->va + sa->icv_len);
277 aead_chacha20_poly1305_aad_fill(chacha20_poly1305_aad,
278 sa->spi, sqc, IS_ESN(sa));
279 }
280 break;
281 default:
282 break;
283 }
284 }
285
286 /*
287 * setup/update packets and crypto ops for ESP outbound tunnel case.
288 */
289 uint16_t
esp_outb_tun_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],struct rte_crypto_op * cop[],uint16_t num)290 esp_outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
291 struct rte_crypto_op *cop[], uint16_t num)
292 {
293 int32_t rc;
294 uint32_t i, k, n;
295 uint64_t sqn;
296 rte_be64_t sqc;
297 struct rte_ipsec_sa *sa;
298 struct rte_cryptodev_sym_session *cs;
299 union sym_op_data icv;
300 uint64_t iv[IPSEC_MAX_IV_QWORD];
301 uint32_t dr[num];
302
303 sa = ss->sa;
304 cs = ss->crypto.ses;
305
306 n = num;
307 sqn = esn_outb_update_sqn(sa, &n);
308 if (n != num)
309 rte_errno = EOVERFLOW;
310
311 k = 0;
312 for (i = 0; i != n; i++) {
313
314 sqc = rte_cpu_to_be_64(sqn + i);
315 gen_iv(iv, sqc);
316
317 /* try to update the packet itself */
318 rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv,
319 sa->sqh_len, 0);
320 /* success, setup crypto op */
321 if (rc >= 0) {
322 outb_pkt_xprepare(sa, sqc, &icv);
323 lksd_none_cop_prepare(cop[k], cs, mb[i]);
324 outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc);
325 k++;
326 /* failure, put packet into the death-row */
327 } else {
328 dr[i - k] = i;
329 rte_errno = -rc;
330 }
331 }
332
333 /* copy not prepared mbufs beyond good ones */
334 if (k != n && k != 0)
335 move_bad_mbufs(mb, dr, n, n - k);
336
337 return k;
338 }
339
340 /*
341 * setup/update packet data and metadata for ESP outbound transport case.
342 */
343 static inline int32_t
outb_trs_pkt_prepare(struct rte_ipsec_sa * sa,rte_be64_t sqc,const uint64_t ivp[IPSEC_MAX_IV_QWORD],struct rte_mbuf * mb,union sym_op_data * icv,uint8_t sqh_len,uint8_t tso)344 outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
345 const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
346 union sym_op_data *icv, uint8_t sqh_len, uint8_t tso)
347 {
348 uint8_t np;
349 uint32_t clen, hlen, pdlen, pdofs, plen, tlen, uhlen;
350 struct rte_mbuf *ml;
351 struct rte_esp_hdr *esph;
352 struct rte_esp_tail *espt;
353 char *ph, *pt;
354 uint64_t *iv;
355 uint32_t l2len, l3len;
356
357 l2len = mb->l2_len;
358 l3len = mb->l3_len;
359
360 uhlen = l2len + l3len;
361 plen = mb->pkt_len - uhlen;
362
363 /* calculate extra header space required */
364 hlen = sa->iv_len + sizeof(*esph);
365
366 /* number of bytes to encrypt */
367 clen = plen + sizeof(*espt);
368
369 if (!tso) {
370 clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
371 /* pad length + esp tail */
372 pdlen = clen - plen;
373 tlen = pdlen + sa->icv_len + sqh_len;
374 } else {
375 /* We don't need to pad/align packet or append ICV length
376 * when using TSO offload
377 */
378 pdlen = clen - plen;
379 tlen = pdlen + sqh_len;
380 }
381
382 /* do append and insert */
383 ml = rte_pktmbuf_lastseg(mb);
384 if (tlen + sa->aad_len > rte_pktmbuf_tailroom(ml))
385 return -ENOSPC;
386
387 /* prepend space for ESP header */
388 ph = rte_pktmbuf_prepend(mb, hlen);
389 if (ph == NULL)
390 return -ENOSPC;
391
392 /* append tail */
393 pdofs = ml->data_len;
394 ml->data_len += tlen;
395 mb->pkt_len += tlen;
396 pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
397
398 /* shift L2/L3 headers */
399 insert_esph(ph, ph + hlen, uhlen);
400
401 /* update ip header fields */
402 np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len - sqh_len, l2len,
403 l3len, IPPROTO_ESP);
404
405 /* update spi, seqn and iv */
406 esph = (struct rte_esp_hdr *)(ph + uhlen);
407 iv = (uint64_t *)(esph + 1);
408 copy_iv(iv, ivp, sa->iv_len);
409
410 esph->spi = sa->spi;
411 esph->seq = sqn_low32(sqc);
412
413 /* offset for ICV */
414 pdofs += pdlen + sa->sqh_len;
415
416 /* pad length */
417 pdlen -= sizeof(*espt);
418
419 /* copy padding data */
420 rte_memcpy(pt, esp_pad_bytes, pdlen);
421
422 /* update esp trailer */
423 espt = (struct rte_esp_tail *)(pt + pdlen);
424 espt->pad_len = pdlen;
425 espt->next_proto = np;
426
427 /* set icv va/pa value(s) */
428 icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
429 icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
430
431 return clen;
432 }
433
434 /*
435 * setup/update packets and crypto ops for ESP outbound transport case.
436 */
437 uint16_t
esp_outb_trs_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],struct rte_crypto_op * cop[],uint16_t num)438 esp_outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
439 struct rte_crypto_op *cop[], uint16_t num)
440 {
441 int32_t rc;
442 uint32_t i, k, n, l2, l3;
443 uint64_t sqn;
444 rte_be64_t sqc;
445 struct rte_ipsec_sa *sa;
446 struct rte_cryptodev_sym_session *cs;
447 union sym_op_data icv;
448 uint64_t iv[IPSEC_MAX_IV_QWORD];
449 uint32_t dr[num];
450
451 sa = ss->sa;
452 cs = ss->crypto.ses;
453
454 n = num;
455 sqn = esn_outb_update_sqn(sa, &n);
456 if (n != num)
457 rte_errno = EOVERFLOW;
458
459 k = 0;
460 for (i = 0; i != n; i++) {
461
462 l2 = mb[i]->l2_len;
463 l3 = mb[i]->l3_len;
464
465 sqc = rte_cpu_to_be_64(sqn + i);
466 gen_iv(iv, sqc);
467
468 /* try to update the packet itself */
469 rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], &icv,
470 sa->sqh_len, 0);
471 /* success, setup crypto op */
472 if (rc >= 0) {
473 outb_pkt_xprepare(sa, sqc, &icv);
474 lksd_none_cop_prepare(cop[k], cs, mb[i]);
475 outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc);
476 k++;
477 /* failure, put packet into the death-row */
478 } else {
479 dr[i - k] = i;
480 rte_errno = -rc;
481 }
482 }
483
484 /* copy not prepared mbufs beyond good ones */
485 if (k != n && k != 0)
486 move_bad_mbufs(mb, dr, n, n - k);
487
488 return k;
489 }
490
491
492 static inline uint32_t
outb_cpu_crypto_prepare(const struct rte_ipsec_sa * sa,uint32_t * pofs,uint32_t plen,void * iv)493 outb_cpu_crypto_prepare(const struct rte_ipsec_sa *sa, uint32_t *pofs,
494 uint32_t plen, void *iv)
495 {
496 uint64_t *ivp = iv;
497 struct aead_gcm_iv *gcm;
498 struct aead_ccm_iv *ccm;
499 struct aead_chacha20_poly1305_iv *chacha20_poly1305;
500 struct aesctr_cnt_blk *ctr;
501 uint32_t clen;
502
503 switch (sa->algo_type) {
504 case ALGO_TYPE_AES_GCM:
505 gcm = iv;
506 aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
507 break;
508 case ALGO_TYPE_AES_CCM:
509 ccm = iv;
510 aead_ccm_iv_fill(ccm, ivp[0], sa->salt);
511 break;
512 case ALGO_TYPE_CHACHA20_POLY1305:
513 chacha20_poly1305 = iv;
514 aead_chacha20_poly1305_iv_fill(chacha20_poly1305,
515 ivp[0], sa->salt);
516 break;
517 case ALGO_TYPE_AES_CTR:
518 ctr = iv;
519 aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
520 break;
521 }
522
523 *pofs += sa->ctp.auth.offset;
524 clen = plen + sa->ctp.auth.length;
525 return clen;
526 }
527
528 static uint16_t
cpu_outb_pkt_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num,esp_outb_prepare_t prepare,uint32_t cofs_mask)529 cpu_outb_pkt_prepare(const struct rte_ipsec_session *ss,
530 struct rte_mbuf *mb[], uint16_t num,
531 esp_outb_prepare_t prepare, uint32_t cofs_mask)
532 {
533 int32_t rc;
534 uint64_t sqn;
535 rte_be64_t sqc;
536 struct rte_ipsec_sa *sa;
537 uint32_t i, k, n;
538 uint32_t l2, l3;
539 union sym_op_data icv;
540 struct rte_crypto_va_iova_ptr iv[num];
541 struct rte_crypto_va_iova_ptr aad[num];
542 struct rte_crypto_va_iova_ptr dgst[num];
543 uint32_t dr[num];
544 uint32_t l4ofs[num];
545 uint32_t clen[num];
546 uint64_t ivbuf[num][IPSEC_MAX_IV_QWORD];
547
548 sa = ss->sa;
549
550 n = num;
551 sqn = esn_outb_update_sqn(sa, &n);
552 if (n != num)
553 rte_errno = EOVERFLOW;
554
555 for (i = 0, k = 0; i != n; i++) {
556
557 l2 = mb[i]->l2_len;
558 l3 = mb[i]->l3_len;
559
560 /* calculate ESP header offset */
561 l4ofs[k] = (l2 + l3) & cofs_mask;
562
563 sqc = rte_cpu_to_be_64(sqn + i);
564 gen_iv(ivbuf[k], sqc);
565
566 /* try to update the packet itself */
567 rc = prepare(sa, sqc, ivbuf[k], mb[i], &icv, sa->sqh_len, 0);
568
569 /* success, proceed with preparations */
570 if (rc >= 0) {
571
572 outb_pkt_xprepare(sa, sqc, &icv);
573
574 /* get encrypted data offset and length */
575 clen[k] = outb_cpu_crypto_prepare(sa, l4ofs + k, rc,
576 ivbuf[k]);
577
578 /* fill iv, digest and aad */
579 iv[k].va = ivbuf[k];
580 aad[k].va = icv.va + sa->icv_len;
581 dgst[k++].va = icv.va;
582 } else {
583 dr[i - k] = i;
584 rte_errno = -rc;
585 }
586 }
587
588 /* copy not prepared mbufs beyond good ones */
589 if (k != n && k != 0)
590 move_bad_mbufs(mb, dr, n, n - k);
591
592 /* convert mbufs to iovecs and do actual crypto/auth processing */
593 if (k != 0)
594 cpu_crypto_bulk(ss, sa->cofs, mb, iv, aad, dgst,
595 l4ofs, clen, k);
596 return k;
597 }
598
599 uint16_t
cpu_outb_tun_pkt_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)600 cpu_outb_tun_pkt_prepare(const struct rte_ipsec_session *ss,
601 struct rte_mbuf *mb[], uint16_t num)
602 {
603 return cpu_outb_pkt_prepare(ss, mb, num, outb_tun_pkt_prepare, 0);
604 }
605
606 uint16_t
cpu_outb_trs_pkt_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)607 cpu_outb_trs_pkt_prepare(const struct rte_ipsec_session *ss,
608 struct rte_mbuf *mb[], uint16_t num)
609 {
610 return cpu_outb_pkt_prepare(ss, mb, num, outb_trs_pkt_prepare,
611 UINT32_MAX);
612 }
613
614 /*
615 * process outbound packets for SA with ESN support,
616 * for algorithms that require SQN.hibits to be implicitly included
617 * into digest computation.
618 * In that case we have to move ICV bytes back to their proper place.
619 */
620 uint16_t
esp_outb_sqh_process(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)621 esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
622 uint16_t num)
623 {
624 uint32_t i, k, icv_len, *icv, bytes;
625 struct rte_mbuf *ml;
626 struct rte_ipsec_sa *sa;
627 uint32_t dr[num];
628
629 sa = ss->sa;
630
631 k = 0;
632 icv_len = sa->icv_len;
633 bytes = 0;
634
635 for (i = 0; i != num; i++) {
636 if ((mb[i]->ol_flags & RTE_MBUF_F_RX_SEC_OFFLOAD_FAILED) == 0) {
637 ml = rte_pktmbuf_lastseg(mb[i]);
638 /* remove high-order 32 bits of esn from packet len */
639 mb[i]->pkt_len -= sa->sqh_len;
640 ml->data_len -= sa->sqh_len;
641 icv = rte_pktmbuf_mtod_offset(ml, void *,
642 ml->data_len - icv_len);
643 remove_sqh(icv, icv_len);
644 bytes += mb[i]->pkt_len;
645 k++;
646 } else
647 dr[i - k] = i;
648 }
649 sa->statistics.count += k;
650 sa->statistics.bytes += bytes;
651
652 /* handle unprocessed mbufs */
653 if (k != num) {
654 rte_errno = EBADMSG;
655 if (k != 0)
656 move_bad_mbufs(mb, dr, num, num - k);
657 }
658
659 return k;
660 }
661
662 /*
663 * prepare packets for inline ipsec processing:
664 * set ol_flags and attach metadata.
665 */
666 static inline void
inline_outb_mbuf_prepare(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)667 inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss,
668 struct rte_mbuf *mb[], uint16_t num)
669 {
670 uint32_t i, ol_flags, bytes;
671
672 ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA;
673 bytes = 0;
674 for (i = 0; i != num; i++) {
675
676 mb[i]->ol_flags |= RTE_MBUF_F_TX_SEC_OFFLOAD;
677 bytes += mb[i]->pkt_len;
678 if (ol_flags != 0)
679 rte_security_set_pkt_metadata(ss->security.ctx,
680 ss->security.ses, mb[i], NULL);
681 }
682 ss->sa->statistics.count += num;
683 ss->sa->statistics.bytes += bytes;
684 }
685
686
687 static inline int
esn_outb_nb_segments(struct rte_mbuf * m)688 esn_outb_nb_segments(struct rte_mbuf *m)
689 {
690 if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) {
691 uint16_t pkt_l3len = m->pkt_len - m->l2_len;
692 uint16_t segments =
693 (m->tso_segsz > 0 && pkt_l3len > m->tso_segsz) ?
694 (pkt_l3len + m->tso_segsz - 1) / m->tso_segsz : 1;
695 return segments;
696 }
697 return 1; /* no TSO */
698 }
699
700 /* Compute how many packets can be sent before overflow occurs */
701 static inline uint16_t
esn_outb_nb_valid_packets(uint16_t num,uint32_t n_sqn,uint16_t nb_segs[])702 esn_outb_nb_valid_packets(uint16_t num, uint32_t n_sqn, uint16_t nb_segs[])
703 {
704 uint16_t i;
705 uint32_t seg_cnt = 0;
706 for (i = 0; i < num && seg_cnt < n_sqn; i++)
707 seg_cnt += nb_segs[i];
708 return i - 1;
709 }
710
711 /*
712 * process group of ESP outbound tunnel packets destined for
713 * INLINE_CRYPTO type of device.
714 */
715 uint16_t
inline_outb_tun_pkt_process(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)716 inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
717 struct rte_mbuf *mb[], uint16_t num)
718 {
719 int32_t rc;
720 uint32_t i, k, nb_segs_total, n_sqn;
721 uint64_t sqn;
722 rte_be64_t sqc;
723 struct rte_ipsec_sa *sa;
724 union sym_op_data icv;
725 uint64_t iv[IPSEC_MAX_IV_QWORD];
726 uint32_t dr[num];
727 uint16_t nb_segs[num];
728
729 sa = ss->sa;
730 nb_segs_total = 0;
731 /* Calculate number of segments */
732 for (i = 0; i != num; i++) {
733 nb_segs[i] = esn_outb_nb_segments(mb[i]);
734 nb_segs_total += nb_segs[i];
735 }
736
737 n_sqn = nb_segs_total;
738 sqn = esn_outb_update_sqn(sa, &n_sqn);
739 if (n_sqn != nb_segs_total) {
740 rte_errno = EOVERFLOW;
741 /* if there are segmented packets find out how many can be
742 * sent until overflow occurs
743 */
744 if (nb_segs_total > num) /* there is at least 1 */
745 num = esn_outb_nb_valid_packets(num, n_sqn, nb_segs);
746 else
747 num = n_sqn; /* no segmented packets */
748 }
749
750 k = 0;
751 for (i = 0; i != num; i++) {
752
753 sqc = rte_cpu_to_be_64(sqn);
754 gen_iv(iv, sqc);
755 sqn += nb_segs[i];
756
757 /* try to update the packet itself */
758 rc = outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0,
759 (mb[i]->ol_flags &
760 (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) != 0);
761
762 k += (rc >= 0);
763
764 /* failure, put packet into the death-row */
765 if (rc < 0) {
766 dr[i - k] = i;
767 rte_errno = -rc;
768 }
769 }
770
771 /* copy not processed mbufs beyond good ones */
772 if (k != num && k != 0)
773 move_bad_mbufs(mb, dr, num, num - k);
774
775 inline_outb_mbuf_prepare(ss, mb, k);
776 return k;
777 }
778
779 /*
780 * process group of ESP outbound transport packets destined for
781 * INLINE_CRYPTO type of device.
782 */
783 uint16_t
inline_outb_trs_pkt_process(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)784 inline_outb_trs_pkt_process(const struct rte_ipsec_session *ss,
785 struct rte_mbuf *mb[], uint16_t num)
786 {
787 int32_t rc;
788 uint32_t i, k, nb_segs_total, n_sqn;
789 uint64_t sqn;
790 rte_be64_t sqc;
791 struct rte_ipsec_sa *sa;
792 union sym_op_data icv;
793 uint64_t iv[IPSEC_MAX_IV_QWORD];
794 uint32_t dr[num];
795 uint16_t nb_segs[num];
796
797 sa = ss->sa;
798 nb_segs_total = 0;
799 /* Calculate number of segments */
800 for (i = 0; i != num; i++) {
801 nb_segs[i] = esn_outb_nb_segments(mb[i]);
802 nb_segs_total += nb_segs[i];
803 }
804
805 n_sqn = nb_segs_total;
806 sqn = esn_outb_update_sqn(sa, &n_sqn);
807 if (n_sqn != nb_segs_total) {
808 rte_errno = EOVERFLOW;
809 /* if there are segmented packets find out how many can be
810 * sent until overflow occurs
811 */
812 if (nb_segs_total > num) /* there is at least 1 */
813 num = esn_outb_nb_valid_packets(num, n_sqn, nb_segs);
814 else
815 num = n_sqn; /* no segmented packets */
816 }
817
818 k = 0;
819 for (i = 0; i != num; i++) {
820
821 sqc = rte_cpu_to_be_64(sqn);
822 gen_iv(iv, sqc);
823 sqn += nb_segs[i];
824
825 /* try to update the packet itself */
826 rc = outb_trs_pkt_prepare(sa, sqc, iv, mb[i], &icv, 0,
827 (mb[i]->ol_flags &
828 (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) != 0);
829
830 k += (rc >= 0);
831
832 /* failure, put packet into the death-row */
833 if (rc < 0) {
834 dr[i - k] = i;
835 rte_errno = -rc;
836 }
837 }
838
839 /* copy not processed mbufs beyond good ones */
840 if (k != num && k != 0)
841 move_bad_mbufs(mb, dr, num, num - k);
842
843 inline_outb_mbuf_prepare(ss, mb, k);
844 return k;
845 }
846
847 /*
848 * outbound for RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
849 * actual processing is done by HW/PMD, just set flags and metadata.
850 */
851 uint16_t
inline_proto_outb_pkt_process(const struct rte_ipsec_session * ss,struct rte_mbuf * mb[],uint16_t num)852 inline_proto_outb_pkt_process(const struct rte_ipsec_session *ss,
853 struct rte_mbuf *mb[], uint16_t num)
854 {
855 inline_outb_mbuf_prepare(ss, mb, num);
856 return num;
857 }
858