1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
3 */
4 #include <sys/types.h>
5 #include <netinet/in.h>
6 #include <netinet/ip.h>
7
8 #include <rte_branch_prediction.h>
9 #include <rte_log.h>
10 #include <rte_cryptodev.h>
11 #include <rte_ethdev.h>
12 #include <rte_mbuf.h>
13
14 #include "ipsec.h"
15 #include "ipsec-secgw.h"
16
17 #define SATP_OUT_IPV4(t) \
18 ((((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TRANS && \
19 (((t) & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4)) || \
20 ((t) & RTE_IPSEC_SATP_MODE_MASK) == RTE_IPSEC_SATP_MODE_TUNLV4)
21
22 /* helper routine to free bulk of crypto-ops and related packets */
23 static inline void
free_cops(struct rte_crypto_op * cop[],uint32_t n)24 free_cops(struct rte_crypto_op *cop[], uint32_t n)
25 {
26 uint32_t i;
27
28 for (i = 0; i != n; i++)
29 rte_pktmbuf_free(cop[i]->sym->m_src);
30 }
31
32 /* helper routine to enqueue bulk of crypto ops */
33 static inline void
enqueue_cop_bulk(struct cdev_qp * cqp,struct rte_crypto_op * cop[],uint32_t num)34 enqueue_cop_bulk(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
35 {
36 uint32_t i, k, len, n;
37
38 len = cqp->len;
39
40 /*
41 * if cqp is empty and we have enough ops,
42 * then queue them to the PMD straightway.
43 */
44 if (num >= RTE_DIM(cqp->buf) * 3 / 4 && len == 0) {
45 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cop, num);
46 cqp->in_flight += n;
47 free_cops(cop + n, num - n);
48 return;
49 }
50
51 k = 0;
52
53 do {
54 n = RTE_DIM(cqp->buf) - len;
55 n = RTE_MIN(num - k, n);
56
57 /* put packets into cqp */
58 for (i = 0; i != n; i++)
59 cqp->buf[len + i] = cop[k + i];
60
61 len += n;
62 k += n;
63
64 /* if cqp is full then, enqueue crypto-ops to PMD */
65 if (len == RTE_DIM(cqp->buf)) {
66 n = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp,
67 cqp->buf, len);
68 cqp->in_flight += n;
69 free_cops(cqp->buf + n, len - n);
70 len = 0;
71 }
72
73
74 } while (k != num);
75
76 cqp->len = len;
77 }
78
79 static inline int
fill_ipsec_session(struct rte_ipsec_session * ss,struct ipsec_ctx * ctx,struct ipsec_sa * sa)80 fill_ipsec_session(struct rte_ipsec_session *ss, struct ipsec_ctx *ctx,
81 struct ipsec_sa *sa)
82 {
83 int32_t rc;
84
85 /* setup crypto section */
86 if (ss->type == RTE_SECURITY_ACTION_TYPE_NONE ||
87 ss->type == RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO) {
88 RTE_ASSERT(ss->crypto.ses == NULL);
89 rc = create_lookaside_session(ctx, sa, ss);
90 if (rc != 0)
91 return rc;
92 /* setup session action type */
93 } else if (ss->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
94 RTE_ASSERT(ss->security.ses == NULL);
95 rc = create_lookaside_session(ctx, sa, ss);
96 if (rc != 0)
97 return rc;
98 } else
99 RTE_ASSERT(0);
100
101 rc = rte_ipsec_session_prepare(ss);
102 if (rc != 0)
103 memset(ss, 0, sizeof(*ss));
104
105 return rc;
106 }
107
108 /*
109 * group input packets byt the SA they belong to.
110 */
111 static uint32_t
sa_group(void * sa_ptr[],struct rte_mbuf * pkts[],struct rte_ipsec_group grp[],uint32_t num)112 sa_group(void *sa_ptr[], struct rte_mbuf *pkts[],
113 struct rte_ipsec_group grp[], uint32_t num)
114 {
115 uint32_t i, n, spi;
116 void *sa;
117 void * const nosa = &spi;
118
119 sa = nosa;
120 grp[0].m = pkts;
121 for (i = 0, n = 0; i != num; i++) {
122
123 if (sa != sa_ptr[i]) {
124 grp[n].cnt = pkts + i - grp[n].m;
125 n += (sa != nosa);
126 grp[n].id.ptr = sa_ptr[i];
127 grp[n].m = pkts + i;
128 sa = sa_ptr[i];
129 }
130 }
131
132 /* terminate last group */
133 if (sa != nosa) {
134 grp[n].cnt = pkts + i - grp[n].m;
135 n++;
136 }
137
138 return n;
139 }
140
141 /*
142 * helper function, splits processed packets into ipv4/ipv6 traffic.
143 */
144 static inline void
copy_to_trf(struct ipsec_traffic * trf,uint64_t satp,struct rte_mbuf * mb[],uint32_t num)145 copy_to_trf(struct ipsec_traffic *trf, uint64_t satp, struct rte_mbuf *mb[],
146 uint32_t num)
147 {
148 uint32_t j, ofs, s;
149 struct traffic_type *out;
150
151 /*
152 * determine traffic type(ipv4/ipv6) and offset for ACL classify
153 * based on SA type
154 */
155 if ((satp & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) {
156 if ((satp & RTE_IPSEC_SATP_IPV_MASK) == RTE_IPSEC_SATP_IPV4) {
157 out = &trf->ip4;
158 ofs = offsetof(struct ip, ip_p);
159 } else {
160 out = &trf->ip6;
161 ofs = offsetof(struct ip6_hdr, ip6_nxt);
162 }
163 } else if (SATP_OUT_IPV4(satp)) {
164 out = &trf->ip4;
165 ofs = offsetof(struct ip, ip_p);
166 } else {
167 out = &trf->ip6;
168 ofs = offsetof(struct ip6_hdr, ip6_nxt);
169 }
170
171 for (j = 0, s = out->num; j != num; j++) {
172 out->data[s + j] = rte_pktmbuf_mtod_offset(mb[j],
173 void *, ofs);
174 out->pkts[s + j] = mb[j];
175 }
176
177 out->num += num;
178 }
179
180 static uint32_t
ipsec_prepare_crypto_group(struct ipsec_ctx * ctx,struct ipsec_sa * sa,struct rte_ipsec_session * ips,struct rte_mbuf ** m,unsigned int cnt)181 ipsec_prepare_crypto_group(struct ipsec_ctx *ctx, struct ipsec_sa *sa,
182 struct rte_ipsec_session *ips, struct rte_mbuf **m,
183 unsigned int cnt)
184 {
185 struct cdev_qp *cqp;
186 struct rte_crypto_op *cop[cnt];
187 uint32_t j, k;
188 struct ipsec_mbuf_metadata *priv;
189
190 cqp = &ctx->tbl[sa->cdev_id_qp];
191
192 /* for that app each mbuf has it's own crypto op */
193 for (j = 0; j != cnt; j++) {
194 priv = get_priv(m[j]);
195 cop[j] = &priv->cop;
196 /*
197 * this is just to satisfy inbound_sa_check()
198 * should be removed in future.
199 */
200 priv->sa = sa;
201 }
202
203 /* prepare and enqueue crypto ops */
204 k = rte_ipsec_pkt_crypto_prepare(ips, m, cop, cnt);
205 if (k != 0)
206 enqueue_cop_bulk(cqp, cop, k);
207
208 return k;
209 }
210
211 /*
212 * helper routine for inline and cpu(synchronous) processing
213 * this is just to satisfy inbound_sa_check() and get_hop_for_offload_pkt().
214 * Should be removed in future.
215 */
216 static inline void
prep_process_group(void * sa,struct rte_mbuf * mb[],uint32_t cnt)217 prep_process_group(void *sa, struct rte_mbuf *mb[], uint32_t cnt)
218 {
219 uint32_t j;
220 struct ipsec_mbuf_metadata *priv;
221
222 for (j = 0; j != cnt; j++) {
223 priv = get_priv(mb[j]);
224 priv->sa = sa;
225 /* setup TSO related fields if TSO enabled*/
226 if (priv->sa->mss) {
227 uint32_t ptype = mb[j]->packet_type;
228 /* only TCP is supported */
229 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) {
230 mb[j]->tso_segsz = priv->sa->mss;
231 if ((IS_TUNNEL(priv->sa->flags))) {
232 mb[j]->outer_l3_len = mb[j]->l3_len;
233 mb[j]->outer_l2_len = mb[j]->l2_len;
234 mb[j]->ol_flags |=
235 RTE_MBUF_F_TX_TUNNEL_ESP;
236 if (RTE_ETH_IS_IPV4_HDR(ptype))
237 mb[j]->ol_flags |=
238 RTE_MBUF_F_TX_OUTER_IP_CKSUM;
239 }
240 mb[j]->l4_len = sizeof(struct rte_tcp_hdr);
241 mb[j]->ol_flags |= (RTE_MBUF_F_TX_TCP_SEG |
242 RTE_MBUF_F_TX_TCP_CKSUM);
243 if (RTE_ETH_IS_IPV4_HDR(ptype))
244 mb[j]->ol_flags |=
245 RTE_MBUF_F_TX_OUTER_IPV4;
246 else
247 mb[j]->ol_flags |=
248 RTE_MBUF_F_TX_OUTER_IPV6;
249 }
250 }
251 }
252 }
253
254 /*
255 * finish processing of packets successfully decrypted by an inline processor
256 */
257 static uint32_t
ipsec_process_inline_group(struct rte_ipsec_session * ips,void * sa,struct ipsec_traffic * trf,struct rte_mbuf * mb[],uint32_t cnt)258 ipsec_process_inline_group(struct rte_ipsec_session *ips, void *sa,
259 struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
260 {
261 uint64_t satp;
262 uint32_t k;
263
264 /* get SA type */
265 satp = rte_ipsec_sa_type(ips->sa);
266 prep_process_group(sa, mb, cnt);
267
268 k = rte_ipsec_pkt_process(ips, mb, cnt);
269 copy_to_trf(trf, satp, mb, k);
270 return k;
271 }
272
273 /*
274 * process packets synchronously
275 */
276 static uint32_t
ipsec_process_cpu_group(struct rte_ipsec_session * ips,void * sa,struct ipsec_traffic * trf,struct rte_mbuf * mb[],uint32_t cnt)277 ipsec_process_cpu_group(struct rte_ipsec_session *ips, void *sa,
278 struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t cnt)
279 {
280 uint64_t satp;
281 uint32_t k;
282
283 /* get SA type */
284 satp = rte_ipsec_sa_type(ips->sa);
285 prep_process_group(sa, mb, cnt);
286
287 k = rte_ipsec_pkt_cpu_prepare(ips, mb, cnt);
288 k = rte_ipsec_pkt_process(ips, mb, k);
289 copy_to_trf(trf, satp, mb, k);
290 return k;
291 }
292
293 /*
294 * Process ipsec packets.
295 * If packet belong to SA that is subject of inline-crypto,
296 * then process it immediately.
297 * Otherwise do necessary preparations and queue it to related
298 * crypto-dev queue.
299 */
300 void
ipsec_process(struct ipsec_ctx * ctx,struct ipsec_traffic * trf)301 ipsec_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
302 {
303 uint32_t i, k, n;
304 struct ipsec_sa *sa;
305 struct rte_ipsec_group *pg;
306 struct rte_ipsec_session *ips;
307 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
308
309 n = sa_group(trf->ipsec.saptr, trf->ipsec.pkts, grp, trf->ipsec.num);
310
311 for (i = 0; i != n; i++) {
312
313 pg = grp + i;
314 sa = ipsec_mask_saptr(pg->id.ptr);
315
316 /* fallback to cryptodev with RX packets which inline
317 * processor was unable to process
318 */
319 if (sa != NULL)
320 ips = (pg->id.val & IPSEC_SA_OFFLOAD_FALLBACK_FLAG) ?
321 ipsec_get_fallback_session(sa) :
322 ipsec_get_primary_session(sa);
323
324 /* no valid HW session for that SA, try to create one */
325 if (sa == NULL || (ips->crypto.ses == NULL &&
326 fill_ipsec_session(ips, ctx, sa) != 0))
327 k = 0;
328
329 /* process packets inline */
330 else {
331 switch (ips->type) {
332 /* enqueue packets to crypto dev */
333 case RTE_SECURITY_ACTION_TYPE_NONE:
334 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
335 k = ipsec_prepare_crypto_group(ctx, sa, ips,
336 pg->m, pg->cnt);
337 break;
338 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
339 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
340 k = ipsec_process_inline_group(ips, sa,
341 trf, pg->m, pg->cnt);
342 break;
343 case RTE_SECURITY_ACTION_TYPE_CPU_CRYPTO:
344 k = ipsec_process_cpu_group(ips, sa,
345 trf, pg->m, pg->cnt);
346 break;
347 default:
348 k = 0;
349 }
350 }
351
352 /* drop packets that cannot be enqueued/processed */
353 if (k != pg->cnt)
354 free_pkts(pg->m + k, pg->cnt - k);
355 }
356 }
357
358 static inline uint32_t
cqp_dequeue(struct cdev_qp * cqp,struct rte_crypto_op * cop[],uint32_t num)359 cqp_dequeue(struct cdev_qp *cqp, struct rte_crypto_op *cop[], uint32_t num)
360 {
361 uint32_t n;
362
363 if (cqp->in_flight == 0)
364 return 0;
365
366 n = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp, cop, num);
367 RTE_ASSERT(cqp->in_flight >= n);
368 cqp->in_flight -= n;
369
370 return n;
371 }
372
373 static inline uint32_t
ctx_dequeue(struct ipsec_ctx * ctx,struct rte_crypto_op * cop[],uint32_t num)374 ctx_dequeue(struct ipsec_ctx *ctx, struct rte_crypto_op *cop[], uint32_t num)
375 {
376 uint32_t i, n;
377
378 n = 0;
379
380 for (i = ctx->last_qp; n != num && i != ctx->nb_qps; i++)
381 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);
382
383 for (i = 0; n != num && i != ctx->last_qp; i++)
384 n += cqp_dequeue(ctx->tbl + i, cop + n, num - n);
385
386 ctx->last_qp = i;
387 return n;
388 }
389
390 /*
391 * dequeue packets from crypto-queues and finalize processing.
392 */
393 void
ipsec_cqp_process(struct ipsec_ctx * ctx,struct ipsec_traffic * trf)394 ipsec_cqp_process(struct ipsec_ctx *ctx, struct ipsec_traffic *trf)
395 {
396 uint64_t satp;
397 uint32_t i, k, n, ng;
398 struct rte_ipsec_session *ss;
399 struct traffic_type *out;
400 struct rte_ipsec_group *pg;
401 struct rte_crypto_op *cop[RTE_DIM(trf->ipsec.pkts)];
402 struct rte_ipsec_group grp[RTE_DIM(trf->ipsec.pkts)];
403
404 trf->ip4.num = 0;
405 trf->ip6.num = 0;
406
407 out = &trf->ipsec;
408
409 /* dequeue completed crypto-ops */
410 n = ctx_dequeue(ctx, cop, RTE_DIM(cop));
411 if (n == 0)
412 return;
413
414 /* group them by ipsec session */
415 ng = rte_ipsec_pkt_crypto_group((const struct rte_crypto_op **)
416 (uintptr_t)cop, out->pkts, grp, n);
417
418 /* process each group of packets */
419 for (i = 0; i != ng; i++) {
420
421 pg = grp + i;
422 ss = pg->id.ptr;
423 satp = rte_ipsec_sa_type(ss->sa);
424
425 k = rte_ipsec_pkt_process(ss, pg->m, pg->cnt);
426 copy_to_trf(trf, satp, pg->m, k);
427
428 /* free bad packets, if any */
429 free_pkts(pg->m + k, pg->cnt - k);
430
431 n -= pg->cnt;
432 }
433
434 /* we should never have packet with unknown SA here */
435 RTE_VERIFY(n == 0);
436 }
437