1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Inspur Corporation
3 */
4
5 #include <rte_malloc.h>
6 #include <rte_mbuf.h>
7 #include <rte_cycles.h>
8 #include <rte_ethdev.h>
9
10 #include "gro_udp4.h"
11
12 void *
gro_udp4_tbl_create(uint16_t socket_id,uint16_t max_flow_num,uint16_t max_item_per_flow)13 gro_udp4_tbl_create(uint16_t socket_id,
14 uint16_t max_flow_num,
15 uint16_t max_item_per_flow)
16 {
17 struct gro_udp4_tbl *tbl;
18 size_t size;
19 uint32_t entries_num, i;
20
21 entries_num = max_flow_num * max_item_per_flow;
22 entries_num = RTE_MIN(entries_num, GRO_UDP4_TBL_MAX_ITEM_NUM);
23
24 if (entries_num == 0)
25 return NULL;
26
27 tbl = rte_zmalloc_socket(__func__,
28 sizeof(struct gro_udp4_tbl),
29 RTE_CACHE_LINE_SIZE,
30 socket_id);
31 if (tbl == NULL)
32 return NULL;
33
34 size = sizeof(struct gro_udp4_item) * entries_num;
35 tbl->items = rte_zmalloc_socket(__func__,
36 size,
37 RTE_CACHE_LINE_SIZE,
38 socket_id);
39 if (tbl->items == NULL) {
40 rte_free(tbl);
41 return NULL;
42 }
43 tbl->max_item_num = entries_num;
44
45 size = sizeof(struct gro_udp4_flow) * entries_num;
46 tbl->flows = rte_zmalloc_socket(__func__,
47 size,
48 RTE_CACHE_LINE_SIZE,
49 socket_id);
50 if (tbl->flows == NULL) {
51 rte_free(tbl->items);
52 rte_free(tbl);
53 return NULL;
54 }
55 /* INVALID_ARRAY_INDEX indicates an empty flow */
56 for (i = 0; i < entries_num; i++)
57 tbl->flows[i].start_index = INVALID_ARRAY_INDEX;
58 tbl->max_flow_num = entries_num;
59
60 return tbl;
61 }
62
63 void
gro_udp4_tbl_destroy(void * tbl)64 gro_udp4_tbl_destroy(void *tbl)
65 {
66 struct gro_udp4_tbl *udp_tbl = tbl;
67
68 if (udp_tbl) {
69 rte_free(udp_tbl->items);
70 rte_free(udp_tbl->flows);
71 }
72 rte_free(udp_tbl);
73 }
74
75 static inline uint32_t
find_an_empty_item(struct gro_udp4_tbl * tbl)76 find_an_empty_item(struct gro_udp4_tbl *tbl)
77 {
78 uint32_t i;
79 uint32_t max_item_num = tbl->max_item_num;
80
81 for (i = 0; i < max_item_num; i++)
82 if (tbl->items[i].firstseg == NULL)
83 return i;
84 return INVALID_ARRAY_INDEX;
85 }
86
87 static inline uint32_t
find_an_empty_flow(struct gro_udp4_tbl * tbl)88 find_an_empty_flow(struct gro_udp4_tbl *tbl)
89 {
90 uint32_t i;
91 uint32_t max_flow_num = tbl->max_flow_num;
92
93 for (i = 0; i < max_flow_num; i++)
94 if (tbl->flows[i].start_index == INVALID_ARRAY_INDEX)
95 return i;
96 return INVALID_ARRAY_INDEX;
97 }
98
99 static inline uint32_t
insert_new_item(struct gro_udp4_tbl * tbl,struct rte_mbuf * pkt,uint64_t start_time,uint32_t prev_idx,uint16_t frag_offset,uint8_t is_last_frag)100 insert_new_item(struct gro_udp4_tbl *tbl,
101 struct rte_mbuf *pkt,
102 uint64_t start_time,
103 uint32_t prev_idx,
104 uint16_t frag_offset,
105 uint8_t is_last_frag)
106 {
107 uint32_t item_idx;
108
109 item_idx = find_an_empty_item(tbl);
110 if (unlikely(item_idx == INVALID_ARRAY_INDEX))
111 return INVALID_ARRAY_INDEX;
112
113 tbl->items[item_idx].firstseg = pkt;
114 tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt);
115 tbl->items[item_idx].start_time = start_time;
116 tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX;
117 tbl->items[item_idx].frag_offset = frag_offset;
118 tbl->items[item_idx].is_last_frag = is_last_frag;
119 tbl->items[item_idx].nb_merged = 1;
120 tbl->item_num++;
121
122 /* if the previous packet exists, chain them together. */
123 if (prev_idx != INVALID_ARRAY_INDEX) {
124 tbl->items[item_idx].next_pkt_idx =
125 tbl->items[prev_idx].next_pkt_idx;
126 tbl->items[prev_idx].next_pkt_idx = item_idx;
127 }
128
129 return item_idx;
130 }
131
132 static inline uint32_t
delete_item(struct gro_udp4_tbl * tbl,uint32_t item_idx,uint32_t prev_item_idx)133 delete_item(struct gro_udp4_tbl *tbl, uint32_t item_idx,
134 uint32_t prev_item_idx)
135 {
136 uint32_t next_idx = tbl->items[item_idx].next_pkt_idx;
137
138 /* NULL indicates an empty item */
139 tbl->items[item_idx].firstseg = NULL;
140 tbl->item_num--;
141 if (prev_item_idx != INVALID_ARRAY_INDEX)
142 tbl->items[prev_item_idx].next_pkt_idx = next_idx;
143
144 return next_idx;
145 }
146
147 static inline uint32_t
insert_new_flow(struct gro_udp4_tbl * tbl,struct udp4_flow_key * src,uint32_t item_idx)148 insert_new_flow(struct gro_udp4_tbl *tbl,
149 struct udp4_flow_key *src,
150 uint32_t item_idx)
151 {
152 struct udp4_flow_key *dst;
153 uint32_t flow_idx;
154
155 flow_idx = find_an_empty_flow(tbl);
156 if (unlikely(flow_idx == INVALID_ARRAY_INDEX))
157 return INVALID_ARRAY_INDEX;
158
159 dst = &(tbl->flows[flow_idx].key);
160
161 rte_ether_addr_copy(&(src->eth_saddr), &(dst->eth_saddr));
162 rte_ether_addr_copy(&(src->eth_daddr), &(dst->eth_daddr));
163 dst->ip_src_addr = src->ip_src_addr;
164 dst->ip_dst_addr = src->ip_dst_addr;
165 dst->ip_id = src->ip_id;
166
167 tbl->flows[flow_idx].start_index = item_idx;
168 tbl->flow_num++;
169
170 return flow_idx;
171 }
172
173 /*
174 * update the packet length for the flushed packet.
175 */
176 static inline void
update_header(struct gro_udp4_item * item)177 update_header(struct gro_udp4_item *item)
178 {
179 struct rte_ipv4_hdr *ipv4_hdr;
180 struct rte_mbuf *pkt = item->firstseg;
181 uint16_t frag_offset;
182
183 ipv4_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) +
184 pkt->l2_len);
185 ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len -
186 pkt->l2_len);
187
188 /* Clear MF bit if it is last fragment */
189 if (item->is_last_frag) {
190 frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
191 ipv4_hdr->fragment_offset =
192 rte_cpu_to_be_16(frag_offset & ~RTE_IPV4_HDR_MF_FLAG);
193 }
194 }
195
196 int32_t
gro_udp4_reassemble(struct rte_mbuf * pkt,struct gro_udp4_tbl * tbl,uint64_t start_time)197 gro_udp4_reassemble(struct rte_mbuf *pkt,
198 struct gro_udp4_tbl *tbl,
199 uint64_t start_time)
200 {
201 struct rte_ether_hdr *eth_hdr;
202 struct rte_ipv4_hdr *ipv4_hdr;
203 uint16_t ip_dl;
204 uint16_t ip_id, hdr_len;
205 uint16_t frag_offset = 0;
206 uint8_t is_last_frag;
207
208 struct udp4_flow_key key;
209 uint32_t cur_idx, prev_idx, item_idx;
210 uint32_t i, max_flow_num, remaining_flow_num;
211 int cmp;
212 uint8_t find;
213
214 eth_hdr = rte_pktmbuf_mtod(pkt, struct rte_ether_hdr *);
215 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)eth_hdr + pkt->l2_len);
216 hdr_len = pkt->l2_len + pkt->l3_len;
217
218 /*
219 * Don't process non-fragment packet.
220 */
221 if (!is_ipv4_fragment(ipv4_hdr))
222 return -1;
223
224 /*
225 * Don't process the packet whose payload length is less than or
226 * equal to 0.
227 */
228 if (pkt->pkt_len <= hdr_len)
229 return -1;
230
231 ip_dl = rte_be_to_cpu_16(ipv4_hdr->total_length);
232 if (ip_dl <= pkt->l3_len)
233 return -1;
234
235 ip_dl -= pkt->l3_len;
236 ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id);
237 frag_offset = rte_be_to_cpu_16(ipv4_hdr->fragment_offset);
238 is_last_frag = ((frag_offset & RTE_IPV4_HDR_MF_FLAG) == 0) ? 1 : 0;
239 frag_offset = (uint16_t)(frag_offset & RTE_IPV4_HDR_OFFSET_MASK) << 3;
240
241 rte_ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr));
242 rte_ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr));
243 key.ip_src_addr = ipv4_hdr->src_addr;
244 key.ip_dst_addr = ipv4_hdr->dst_addr;
245 key.ip_id = ip_id;
246
247 /* Search for a matched flow. */
248 max_flow_num = tbl->max_flow_num;
249 remaining_flow_num = tbl->flow_num;
250 find = 0;
251 for (i = 0; i < max_flow_num && remaining_flow_num; i++) {
252 if (tbl->flows[i].start_index != INVALID_ARRAY_INDEX) {
253 if (is_same_udp4_flow(tbl->flows[i].key, key)) {
254 find = 1;
255 break;
256 }
257 remaining_flow_num--;
258 }
259 }
260
261 /*
262 * Fail to find a matched flow. Insert a new flow and store the
263 * packet into the flow.
264 */
265 if (find == 0) {
266 item_idx = insert_new_item(tbl, pkt, start_time,
267 INVALID_ARRAY_INDEX, frag_offset,
268 is_last_frag);
269 if (unlikely(item_idx == INVALID_ARRAY_INDEX))
270 return -1;
271 if (insert_new_flow(tbl, &key, item_idx) ==
272 INVALID_ARRAY_INDEX) {
273 /*
274 * Fail to insert a new flow, so delete the
275 * stored packet.
276 */
277 delete_item(tbl, item_idx, INVALID_ARRAY_INDEX);
278 return -1;
279 }
280 return 0;
281 }
282
283 /*
284 * Check all packets in the flow and try to find a neighbor for
285 * the input packet.
286 */
287 cur_idx = tbl->flows[i].start_index;
288 prev_idx = cur_idx;
289 do {
290 cmp = udp4_check_neighbor(&(tbl->items[cur_idx]),
291 frag_offset, ip_dl, 0);
292 if (cmp) {
293 if (merge_two_udp4_packets(&(tbl->items[cur_idx]),
294 pkt, cmp, frag_offset,
295 is_last_frag, 0))
296 return 1;
297 /*
298 * Fail to merge the two packets, as the packet
299 * length is greater than the max value. Store
300 * the packet into the flow.
301 */
302 if (insert_new_item(tbl, pkt, start_time, prev_idx,
303 frag_offset, is_last_frag) ==
304 INVALID_ARRAY_INDEX)
305 return -1;
306 return 0;
307 }
308
309 /* Ensure inserted items are ordered by frag_offset */
310 if (frag_offset
311 < tbl->items[cur_idx].frag_offset) {
312 break;
313 }
314
315 prev_idx = cur_idx;
316 cur_idx = tbl->items[cur_idx].next_pkt_idx;
317 } while (cur_idx != INVALID_ARRAY_INDEX);
318
319 /* Fail to find a neighbor, so store the packet into the flow. */
320 if (cur_idx == tbl->flows[i].start_index) {
321 /* Insert it before the first packet of the flow */
322 item_idx = insert_new_item(tbl, pkt, start_time,
323 INVALID_ARRAY_INDEX, frag_offset,
324 is_last_frag);
325 if (unlikely(item_idx == INVALID_ARRAY_INDEX))
326 return -1;
327 tbl->items[item_idx].next_pkt_idx = cur_idx;
328 tbl->flows[i].start_index = item_idx;
329 } else {
330 if (insert_new_item(tbl, pkt, start_time, prev_idx,
331 frag_offset, is_last_frag)
332 == INVALID_ARRAY_INDEX)
333 return -1;
334 }
335
336 return 0;
337 }
338
339 static int
gro_udp4_merge_items(struct gro_udp4_tbl * tbl,uint32_t start_idx)340 gro_udp4_merge_items(struct gro_udp4_tbl *tbl,
341 uint32_t start_idx)
342 {
343 uint16_t frag_offset;
344 uint8_t is_last_frag;
345 int16_t ip_dl;
346 struct rte_mbuf *pkt;
347 int cmp;
348 uint32_t item_idx;
349 uint16_t hdr_len;
350
351 item_idx = tbl->items[start_idx].next_pkt_idx;
352 while (item_idx != INVALID_ARRAY_INDEX) {
353 pkt = tbl->items[item_idx].firstseg;
354 hdr_len = pkt->l2_len + pkt->l3_len;
355 ip_dl = pkt->pkt_len - hdr_len;
356 frag_offset = tbl->items[item_idx].frag_offset;
357 is_last_frag = tbl->items[item_idx].is_last_frag;
358 cmp = udp4_check_neighbor(&(tbl->items[start_idx]),
359 frag_offset, ip_dl, 0);
360 if (cmp) {
361 if (merge_two_udp4_packets(
362 &(tbl->items[start_idx]),
363 pkt, cmp, frag_offset,
364 is_last_frag, 0)) {
365 item_idx = delete_item(tbl, item_idx,
366 INVALID_ARRAY_INDEX);
367 tbl->items[start_idx].next_pkt_idx
368 = item_idx;
369 } else
370 return 0;
371 } else
372 return 0;
373 }
374
375 return 0;
376 }
377
378 uint16_t
gro_udp4_tbl_timeout_flush(struct gro_udp4_tbl * tbl,uint64_t flush_timestamp,struct rte_mbuf ** out,uint16_t nb_out)379 gro_udp4_tbl_timeout_flush(struct gro_udp4_tbl *tbl,
380 uint64_t flush_timestamp,
381 struct rte_mbuf **out,
382 uint16_t nb_out)
383 {
384 uint16_t k = 0;
385 uint32_t i, j;
386 uint32_t max_flow_num = tbl->max_flow_num;
387
388 for (i = 0; i < max_flow_num; i++) {
389 if (unlikely(tbl->flow_num == 0))
390 return k;
391
392 j = tbl->flows[i].start_index;
393 while (j != INVALID_ARRAY_INDEX) {
394 if (tbl->items[j].start_time <= flush_timestamp) {
395 gro_udp4_merge_items(tbl, j);
396 out[k++] = tbl->items[j].firstseg;
397 if (tbl->items[j].nb_merged > 1)
398 update_header(&(tbl->items[j]));
399 /*
400 * Delete the packet and get the next
401 * packet in the flow.
402 */
403 j = delete_item(tbl, j, INVALID_ARRAY_INDEX);
404 tbl->flows[i].start_index = j;
405 if (j == INVALID_ARRAY_INDEX)
406 tbl->flow_num--;
407
408 if (unlikely(k == nb_out))
409 return k;
410 } else
411 /*
412 * Flushing packets does not strictly follow
413 * timestamp. It does not flush left packets of
414 * the flow this time once it finds one item
415 * whose start_time is greater than
416 * flush_timestamp. So go to check other flows.
417 */
418 break;
419 }
420 }
421 return k;
422 }
423
424 uint32_t
gro_udp4_tbl_pkt_count(void * tbl)425 gro_udp4_tbl_pkt_count(void *tbl)
426 {
427 struct gro_udp4_tbl *gro_tbl = tbl;
428
429 if (gro_tbl)
430 return gro_tbl->item_num;
431
432 return 0;
433 }
434