1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_mbuf.h>
7 #include <rte_cycles.h>
8 #include <rte_ethdev.h>
9
10 #include "rte_gro.h"
11 #include "gro_tcp4.h"
12 #include "gro_udp4.h"
13 #include "gro_vxlan_tcp4.h"
14 #include "gro_vxlan_udp4.h"
15
16 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id,
17 uint16_t max_flow_num,
18 uint16_t max_item_per_flow);
19 typedef void (*gro_tbl_destroy_fn)(void *tbl);
20 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl);
21
22 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = {
23 gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create,
24 gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL};
25 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = {
26 gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy,
27 gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy,
28 NULL};
29 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = {
30 gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count,
31 gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count,
32 NULL};
33
34 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
35 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \
36 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
37
38 #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
39 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
40 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0))
41
42 #define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
43 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
44 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
45 RTE_PTYPE_TUNNEL_VXLAN) && \
46 ((ptype & RTE_PTYPE_INNER_L4_TCP) == \
47 RTE_PTYPE_INNER_L4_TCP) && \
48 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
49 RTE_PTYPE_INNER_L3_IPV4) || \
50 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
51 RTE_PTYPE_INNER_L3_IPV4_EXT) || \
52 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
53 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
54
55 #define IS_IPV4_VXLAN_UDP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \
56 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \
57 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \
58 RTE_PTYPE_TUNNEL_VXLAN) && \
59 ((ptype & RTE_PTYPE_INNER_L4_UDP) == \
60 RTE_PTYPE_INNER_L4_UDP) && \
61 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \
62 RTE_PTYPE_INNER_L3_IPV4) || \
63 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
64 RTE_PTYPE_INNER_L3_IPV4_EXT) || \
65 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \
66 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN)))
67
68 /*
69 * GRO context structure. It keeps the table structures, which are
70 * used to merge packets, for different GRO types. Before using
71 * rte_gro_reassemble(), applications need to create the GRO context
72 * first.
73 */
74 struct gro_ctx {
75 /* GRO types to perform */
76 uint64_t gro_types;
77 /* reassembly tables */
78 void *tbls[RTE_GRO_TYPE_MAX_NUM];
79 };
80
81 void *
rte_gro_ctx_create(const struct rte_gro_param * param)82 rte_gro_ctx_create(const struct rte_gro_param *param)
83 {
84 struct gro_ctx *gro_ctx;
85 gro_tbl_create_fn create_tbl_fn;
86 uint64_t gro_type_flag = 0;
87 uint64_t gro_types = 0;
88 uint8_t i;
89
90 gro_ctx = rte_zmalloc_socket(__func__,
91 sizeof(struct gro_ctx),
92 RTE_CACHE_LINE_SIZE,
93 param->socket_id);
94 if (gro_ctx == NULL)
95 return NULL;
96
97 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
98 gro_type_flag = 1ULL << i;
99 if ((param->gro_types & gro_type_flag) == 0)
100 continue;
101
102 create_tbl_fn = tbl_create_fn[i];
103 if (create_tbl_fn == NULL)
104 continue;
105
106 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id,
107 param->max_flow_num,
108 param->max_item_per_flow);
109 if (gro_ctx->tbls[i] == NULL) {
110 /* destroy all created tables */
111 gro_ctx->gro_types = gro_types;
112 rte_gro_ctx_destroy(gro_ctx);
113 return NULL;
114 }
115 gro_types |= gro_type_flag;
116 }
117 gro_ctx->gro_types = param->gro_types;
118
119 return gro_ctx;
120 }
121
122 void
rte_gro_ctx_destroy(void * ctx)123 rte_gro_ctx_destroy(void *ctx)
124 {
125 gro_tbl_destroy_fn destroy_tbl_fn;
126 struct gro_ctx *gro_ctx = ctx;
127 uint64_t gro_type_flag;
128 uint8_t i;
129
130 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) {
131 gro_type_flag = 1ULL << i;
132 if ((gro_ctx->gro_types & gro_type_flag) == 0)
133 continue;
134 destroy_tbl_fn = tbl_destroy_fn[i];
135 if (destroy_tbl_fn)
136 destroy_tbl_fn(gro_ctx->tbls[i]);
137 }
138 rte_free(gro_ctx);
139 }
140
141 uint16_t
rte_gro_reassemble_burst(struct rte_mbuf ** pkts,uint16_t nb_pkts,const struct rte_gro_param * param)142 rte_gro_reassemble_burst(struct rte_mbuf **pkts,
143 uint16_t nb_pkts,
144 const struct rte_gro_param *param)
145 {
146 /* allocate a reassembly table for TCP/IPv4 GRO */
147 struct gro_tcp4_tbl tcp_tbl;
148 struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
149 struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
150
151 /* allocate a reassembly table for UDP/IPv4 GRO */
152 struct gro_udp4_tbl udp_tbl;
153 struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
154 struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} };
155
156 /* Allocate a reassembly table for VXLAN TCP GRO */
157 struct gro_vxlan_tcp4_tbl vxlan_tcp_tbl;
158 struct gro_vxlan_tcp4_flow vxlan_tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
159 struct gro_vxlan_tcp4_item vxlan_tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
160 = {{{0}, 0, 0} };
161
162 /* Allocate a reassembly table for VXLAN UDP GRO */
163 struct gro_vxlan_udp4_tbl vxlan_udp_tbl;
164 struct gro_vxlan_udp4_flow vxlan_udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM];
165 struct gro_vxlan_udp4_item vxlan_udp_items[RTE_GRO_MAX_BURST_ITEM_NUM]
166 = {{{0}} };
167
168 struct rte_mbuf *unprocess_pkts[nb_pkts];
169 uint32_t item_num;
170 int32_t ret;
171 uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts;
172 uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0,
173 do_vxlan_udp_gro = 0;
174
175 if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
176 RTE_GRO_TCP_IPV4 |
177 RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
178 RTE_GRO_UDP_IPV4)) == 0))
179 return nb_pkts;
180
181 /* Get the maximum number of packets */
182 item_num = RTE_MIN(nb_pkts, (param->max_flow_num *
183 param->max_item_per_flow));
184 item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM);
185
186 if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
187 for (i = 0; i < item_num; i++)
188 vxlan_tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
189
190 vxlan_tcp_tbl.flows = vxlan_tcp_flows;
191 vxlan_tcp_tbl.items = vxlan_tcp_items;
192 vxlan_tcp_tbl.flow_num = 0;
193 vxlan_tcp_tbl.item_num = 0;
194 vxlan_tcp_tbl.max_flow_num = item_num;
195 vxlan_tcp_tbl.max_item_num = item_num;
196 do_vxlan_tcp_gro = 1;
197 }
198
199 if (param->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) {
200 for (i = 0; i < item_num; i++)
201 vxlan_udp_flows[i].start_index = INVALID_ARRAY_INDEX;
202
203 vxlan_udp_tbl.flows = vxlan_udp_flows;
204 vxlan_udp_tbl.items = vxlan_udp_items;
205 vxlan_udp_tbl.flow_num = 0;
206 vxlan_udp_tbl.item_num = 0;
207 vxlan_udp_tbl.max_flow_num = item_num;
208 vxlan_udp_tbl.max_item_num = item_num;
209 do_vxlan_udp_gro = 1;
210 }
211
212 if (param->gro_types & RTE_GRO_TCP_IPV4) {
213 for (i = 0; i < item_num; i++)
214 tcp_flows[i].start_index = INVALID_ARRAY_INDEX;
215
216 tcp_tbl.flows = tcp_flows;
217 tcp_tbl.items = tcp_items;
218 tcp_tbl.flow_num = 0;
219 tcp_tbl.item_num = 0;
220 tcp_tbl.max_flow_num = item_num;
221 tcp_tbl.max_item_num = item_num;
222 do_tcp4_gro = 1;
223 }
224
225 if (param->gro_types & RTE_GRO_UDP_IPV4) {
226 for (i = 0; i < item_num; i++)
227 udp_flows[i].start_index = INVALID_ARRAY_INDEX;
228
229 udp_tbl.flows = udp_flows;
230 udp_tbl.items = udp_items;
231 udp_tbl.flow_num = 0;
232 udp_tbl.item_num = 0;
233 udp_tbl.max_flow_num = item_num;
234 udp_tbl.max_item_num = item_num;
235 do_udp4_gro = 1;
236 }
237
238
239 for (i = 0; i < nb_pkts; i++) {
240 /*
241 * The timestamp is ignored, since all packets
242 * will be flushed from the tables.
243 */
244 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
245 do_vxlan_tcp_gro) {
246 ret = gro_vxlan_tcp4_reassemble(pkts[i],
247 &vxlan_tcp_tbl, 0);
248 if (ret > 0)
249 /* Merge successfully */
250 nb_after_gro--;
251 else if (ret < 0)
252 unprocess_pkts[unprocess_num++] = pkts[i];
253 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
254 do_vxlan_udp_gro) {
255 ret = gro_vxlan_udp4_reassemble(pkts[i],
256 &vxlan_udp_tbl, 0);
257 if (ret > 0)
258 /* Merge successfully */
259 nb_after_gro--;
260 else if (ret < 0)
261 unprocess_pkts[unprocess_num++] = pkts[i];
262 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
263 do_tcp4_gro) {
264 ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0);
265 if (ret > 0)
266 /* merge successfully */
267 nb_after_gro--;
268 else if (ret < 0)
269 unprocess_pkts[unprocess_num++] = pkts[i];
270 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
271 do_udp4_gro) {
272 ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0);
273 if (ret > 0)
274 /* merge successfully */
275 nb_after_gro--;
276 else if (ret < 0)
277 unprocess_pkts[unprocess_num++] = pkts[i];
278 } else
279 unprocess_pkts[unprocess_num++] = pkts[i];
280 }
281
282 if ((nb_after_gro < nb_pkts)
283 || (unprocess_num < nb_pkts)) {
284 i = 0;
285 /* Flush all packets from the tables */
286 if (do_vxlan_tcp_gro) {
287 i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl,
288 0, pkts, nb_pkts);
289 }
290
291 if (do_vxlan_udp_gro) {
292 i += gro_vxlan_udp4_tbl_timeout_flush(&vxlan_udp_tbl,
293 0, &pkts[i], nb_pkts - i);
294
295 }
296
297 if (do_tcp4_gro) {
298 i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0,
299 &pkts[i], nb_pkts - i);
300 }
301
302 if (do_udp4_gro) {
303 i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0,
304 &pkts[i], nb_pkts - i);
305 }
306 /* Copy unprocessed packets */
307 if (unprocess_num > 0) {
308 memcpy(&pkts[i], unprocess_pkts,
309 sizeof(struct rte_mbuf *) *
310 unprocess_num);
311 }
312 nb_after_gro = i + unprocess_num;
313 }
314
315 return nb_after_gro;
316 }
317
318 uint16_t
rte_gro_reassemble(struct rte_mbuf ** pkts,uint16_t nb_pkts,void * ctx)319 rte_gro_reassemble(struct rte_mbuf **pkts,
320 uint16_t nb_pkts,
321 void *ctx)
322 {
323 struct rte_mbuf *unprocess_pkts[nb_pkts];
324 struct gro_ctx *gro_ctx = ctx;
325 void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl;
326 uint64_t current_time;
327 uint16_t i, unprocess_num = 0;
328 uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro;
329
330 if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 |
331 RTE_GRO_TCP_IPV4 |
332 RTE_GRO_IPV4_VXLAN_UDP_IPV4 |
333 RTE_GRO_UDP_IPV4)) == 0))
334 return nb_pkts;
335
336 tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX];
337 vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX];
338 udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX];
339 vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX];
340
341 do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) ==
342 RTE_GRO_TCP_IPV4;
343 do_vxlan_tcp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) ==
344 RTE_GRO_IPV4_VXLAN_TCP_IPV4;
345 do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) ==
346 RTE_GRO_UDP_IPV4;
347 do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) ==
348 RTE_GRO_IPV4_VXLAN_UDP_IPV4;
349
350 current_time = rte_rdtsc();
351
352 for (i = 0; i < nb_pkts; i++) {
353 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) &&
354 do_vxlan_tcp_gro) {
355 if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tcp_tbl,
356 current_time) < 0)
357 unprocess_pkts[unprocess_num++] = pkts[i];
358 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) &&
359 do_vxlan_udp_gro) {
360 if (gro_vxlan_udp4_reassemble(pkts[i], vxlan_udp_tbl,
361 current_time) < 0)
362 unprocess_pkts[unprocess_num++] = pkts[i];
363 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) &&
364 do_tcp4_gro) {
365 if (gro_tcp4_reassemble(pkts[i], tcp_tbl,
366 current_time) < 0)
367 unprocess_pkts[unprocess_num++] = pkts[i];
368 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) &&
369 do_udp4_gro) {
370 if (gro_udp4_reassemble(pkts[i], udp_tbl,
371 current_time) < 0)
372 unprocess_pkts[unprocess_num++] = pkts[i];
373 } else
374 unprocess_pkts[unprocess_num++] = pkts[i];
375 }
376 if (unprocess_num > 0) {
377 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) *
378 unprocess_num);
379 }
380
381 return unprocess_num;
382 }
383
384 uint16_t
rte_gro_timeout_flush(void * ctx,uint64_t timeout_cycles,uint64_t gro_types,struct rte_mbuf ** out,uint16_t max_nb_out)385 rte_gro_timeout_flush(void *ctx,
386 uint64_t timeout_cycles,
387 uint64_t gro_types,
388 struct rte_mbuf **out,
389 uint16_t max_nb_out)
390 {
391 struct gro_ctx *gro_ctx = ctx;
392 uint64_t flush_timestamp;
393 uint16_t num = 0;
394 uint16_t left_nb_out = max_nb_out;
395
396 gro_types = gro_types & gro_ctx->gro_types;
397 flush_timestamp = rte_rdtsc() - timeout_cycles;
398
399 if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) {
400 num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[
401 RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX],
402 flush_timestamp, out, left_nb_out);
403 left_nb_out = max_nb_out - num;
404 }
405
406 if ((gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) && left_nb_out > 0) {
407 num += gro_vxlan_udp4_tbl_timeout_flush(gro_ctx->tbls[
408 RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX],
409 flush_timestamp, &out[num], left_nb_out);
410 left_nb_out = max_nb_out - num;
411 }
412
413 /* If no available space in 'out', stop flushing. */
414 if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) {
415 num += gro_tcp4_tbl_timeout_flush(
416 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX],
417 flush_timestamp,
418 &out[num], left_nb_out);
419 left_nb_out = max_nb_out - num;
420 }
421
422 /* If no available space in 'out', stop flushing. */
423 if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) {
424 num += gro_udp4_tbl_timeout_flush(
425 gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX],
426 flush_timestamp,
427 &out[num], left_nb_out);
428 }
429
430 return num;
431 }
432
433 uint64_t
rte_gro_get_pkt_count(void * ctx)434 rte_gro_get_pkt_count(void *ctx)
435 {
436 struct gro_ctx *gro_ctx = ctx;
437 gro_tbl_pkt_count_fn pkt_count_fn;
438 uint64_t gro_types = gro_ctx->gro_types, flag;
439 uint64_t item_num = 0;
440 uint8_t i;
441
442 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) {
443 flag = 1ULL << i;
444 if ((gro_types & flag) == 0)
445 continue;
446
447 gro_types ^= flag;
448 pkt_count_fn = tbl_pkt_count_fn[i];
449 if (pkt_count_fn)
450 item_num += pkt_count_fn(gro_ctx->tbls[i]);
451 }
452
453 return item_num;
454 }
455