1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <rte_malloc.h> 6 #include <rte_mbuf.h> 7 #include <rte_cycles.h> 8 #include <rte_ethdev.h> 9 10 #include "rte_gro.h" 11 #include "gro_tcp4.h" 12 #include "gro_udp4.h" 13 #include "gro_vxlan_tcp4.h" 14 #include "gro_vxlan_udp4.h" 15 16 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, 17 uint16_t max_flow_num, 18 uint16_t max_item_per_flow); 19 typedef void (*gro_tbl_destroy_fn)(void *tbl); 20 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl); 21 22 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = { 23 gro_tcp4_tbl_create, gro_vxlan_tcp4_tbl_create, 24 gro_udp4_tbl_create, gro_vxlan_udp4_tbl_create, NULL}; 25 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = { 26 gro_tcp4_tbl_destroy, gro_vxlan_tcp4_tbl_destroy, 27 gro_udp4_tbl_destroy, gro_vxlan_udp4_tbl_destroy, 28 NULL}; 29 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = { 30 gro_tcp4_tbl_pkt_count, gro_vxlan_tcp4_tbl_pkt_count, 31 gro_udp4_tbl_pkt_count, gro_vxlan_udp4_tbl_pkt_count, 32 NULL}; 33 34 #define IS_IPV4_TCP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ 35 ((ptype & RTE_PTYPE_L4_TCP) == RTE_PTYPE_L4_TCP) && \ 36 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0)) 37 38 #define IS_IPV4_UDP_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ 39 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \ 40 (RTE_ETH_IS_TUNNEL_PKT(ptype) == 0)) 41 42 #define IS_IPV4_VXLAN_TCP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ 43 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \ 44 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \ 45 RTE_PTYPE_TUNNEL_VXLAN) && \ 46 ((ptype & RTE_PTYPE_INNER_L4_TCP) == \ 47 RTE_PTYPE_INNER_L4_TCP) && \ 48 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 49 RTE_PTYPE_INNER_L3_IPV4) || \ 50 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 51 RTE_PTYPE_INNER_L3_IPV4_EXT) || \ 52 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 53 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN))) 54 55 #define IS_IPV4_VXLAN_UDP4_PKT(ptype) (RTE_ETH_IS_IPV4_HDR(ptype) && \ 56 ((ptype & RTE_PTYPE_L4_UDP) == RTE_PTYPE_L4_UDP) && \ 57 ((ptype & RTE_PTYPE_TUNNEL_VXLAN) == \ 58 RTE_PTYPE_TUNNEL_VXLAN) && \ 59 ((ptype & RTE_PTYPE_INNER_L4_UDP) == \ 60 RTE_PTYPE_INNER_L4_UDP) && \ 61 (((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 62 RTE_PTYPE_INNER_L3_IPV4) || \ 63 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 64 RTE_PTYPE_INNER_L3_IPV4_EXT) || \ 65 ((ptype & RTE_PTYPE_INNER_L3_MASK) == \ 66 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN))) 67 68 /* 69 * GRO context structure. It keeps the table structures, which are 70 * used to merge packets, for different GRO types. Before using 71 * rte_gro_reassemble(), applications need to create the GRO context 72 * first. 73 */ 74 struct gro_ctx { 75 /* GRO types to perform */ 76 uint64_t gro_types; 77 /* reassembly tables */ 78 void *tbls[RTE_GRO_TYPE_MAX_NUM]; 79 }; 80 81 void * 82 rte_gro_ctx_create(const struct rte_gro_param *param) 83 { 84 struct gro_ctx *gro_ctx; 85 gro_tbl_create_fn create_tbl_fn; 86 uint64_t gro_type_flag = 0; 87 uint64_t gro_types = 0; 88 uint8_t i; 89 90 gro_ctx = rte_zmalloc_socket(__func__, 91 sizeof(struct gro_ctx), 92 RTE_CACHE_LINE_SIZE, 93 param->socket_id); 94 if (gro_ctx == NULL) 95 return NULL; 96 97 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { 98 gro_type_flag = 1ULL << i; 99 if ((param->gro_types & gro_type_flag) == 0) 100 continue; 101 102 create_tbl_fn = tbl_create_fn[i]; 103 if (create_tbl_fn == NULL) 104 continue; 105 106 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id, 107 param->max_flow_num, 108 param->max_item_per_flow); 109 if (gro_ctx->tbls[i] == NULL) { 110 /* destroy all created tables */ 111 gro_ctx->gro_types = gro_types; 112 rte_gro_ctx_destroy(gro_ctx); 113 return NULL; 114 } 115 gro_types |= gro_type_flag; 116 } 117 gro_ctx->gro_types = param->gro_types; 118 119 return gro_ctx; 120 } 121 122 void 123 rte_gro_ctx_destroy(void *ctx) 124 { 125 gro_tbl_destroy_fn destroy_tbl_fn; 126 struct gro_ctx *gro_ctx = ctx; 127 uint64_t gro_type_flag; 128 uint8_t i; 129 130 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { 131 gro_type_flag = 1ULL << i; 132 if ((gro_ctx->gro_types & gro_type_flag) == 0) 133 continue; 134 destroy_tbl_fn = tbl_destroy_fn[i]; 135 if (destroy_tbl_fn) 136 destroy_tbl_fn(gro_ctx->tbls[i]); 137 } 138 rte_free(gro_ctx); 139 } 140 141 uint16_t 142 rte_gro_reassemble_burst(struct rte_mbuf **pkts, 143 uint16_t nb_pkts, 144 const struct rte_gro_param *param) 145 { 146 /* allocate a reassembly table for TCP/IPv4 GRO */ 147 struct gro_tcp4_tbl tcp_tbl; 148 struct gro_tcp4_flow tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; 149 struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; 150 151 /* allocate a reassembly table for UDP/IPv4 GRO */ 152 struct gro_udp4_tbl udp_tbl; 153 struct gro_udp4_flow udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; 154 struct gro_udp4_item udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; 155 156 /* Allocate a reassembly table for VXLAN TCP GRO */ 157 struct gro_vxlan_tcp4_tbl vxlan_tcp_tbl; 158 struct gro_vxlan_tcp4_flow vxlan_tcp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; 159 struct gro_vxlan_tcp4_item vxlan_tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] 160 = {{{0}, 0, 0} }; 161 162 /* Allocate a reassembly table for VXLAN UDP GRO */ 163 struct gro_vxlan_udp4_tbl vxlan_udp_tbl; 164 struct gro_vxlan_udp4_flow vxlan_udp_flows[RTE_GRO_MAX_BURST_ITEM_NUM]; 165 struct gro_vxlan_udp4_item vxlan_udp_items[RTE_GRO_MAX_BURST_ITEM_NUM] 166 = {{{0}} }; 167 168 struct rte_mbuf *unprocess_pkts[nb_pkts]; 169 uint32_t item_num; 170 int32_t ret; 171 uint16_t i, unprocess_num = 0, nb_after_gro = nb_pkts; 172 uint8_t do_tcp4_gro = 0, do_vxlan_tcp_gro = 0, do_udp4_gro = 0, 173 do_vxlan_udp_gro = 0; 174 175 if (unlikely((param->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | 176 RTE_GRO_TCP_IPV4 | 177 RTE_GRO_IPV4_VXLAN_UDP_IPV4 | 178 RTE_GRO_UDP_IPV4)) == 0)) 179 return nb_pkts; 180 181 /* Get the maximum number of packets */ 182 item_num = RTE_MIN(nb_pkts, (param->max_flow_num * 183 param->max_item_per_flow)); 184 item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM); 185 186 if (param->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { 187 for (i = 0; i < item_num; i++) 188 vxlan_tcp_flows[i].start_index = INVALID_ARRAY_INDEX; 189 190 vxlan_tcp_tbl.flows = vxlan_tcp_flows; 191 vxlan_tcp_tbl.items = vxlan_tcp_items; 192 vxlan_tcp_tbl.flow_num = 0; 193 vxlan_tcp_tbl.item_num = 0; 194 vxlan_tcp_tbl.max_flow_num = item_num; 195 vxlan_tcp_tbl.max_item_num = item_num; 196 do_vxlan_tcp_gro = 1; 197 } 198 199 if (param->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) { 200 for (i = 0; i < item_num; i++) 201 vxlan_udp_flows[i].start_index = INVALID_ARRAY_INDEX; 202 203 vxlan_udp_tbl.flows = vxlan_udp_flows; 204 vxlan_udp_tbl.items = vxlan_udp_items; 205 vxlan_udp_tbl.flow_num = 0; 206 vxlan_udp_tbl.item_num = 0; 207 vxlan_udp_tbl.max_flow_num = item_num; 208 vxlan_udp_tbl.max_item_num = item_num; 209 do_vxlan_udp_gro = 1; 210 } 211 212 if (param->gro_types & RTE_GRO_TCP_IPV4) { 213 for (i = 0; i < item_num; i++) 214 tcp_flows[i].start_index = INVALID_ARRAY_INDEX; 215 216 tcp_tbl.flows = tcp_flows; 217 tcp_tbl.items = tcp_items; 218 tcp_tbl.flow_num = 0; 219 tcp_tbl.item_num = 0; 220 tcp_tbl.max_flow_num = item_num; 221 tcp_tbl.max_item_num = item_num; 222 do_tcp4_gro = 1; 223 } 224 225 if (param->gro_types & RTE_GRO_UDP_IPV4) { 226 for (i = 0; i < item_num; i++) 227 udp_flows[i].start_index = INVALID_ARRAY_INDEX; 228 229 udp_tbl.flows = udp_flows; 230 udp_tbl.items = udp_items; 231 udp_tbl.flow_num = 0; 232 udp_tbl.item_num = 0; 233 udp_tbl.max_flow_num = item_num; 234 udp_tbl.max_item_num = item_num; 235 do_udp4_gro = 1; 236 } 237 238 239 for (i = 0; i < nb_pkts; i++) { 240 /* 241 * The timestamp is ignored, since all packets 242 * will be flushed from the tables. 243 */ 244 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && 245 do_vxlan_tcp_gro) { 246 ret = gro_vxlan_tcp4_reassemble(pkts[i], 247 &vxlan_tcp_tbl, 0); 248 if (ret > 0) 249 /* Merge successfully */ 250 nb_after_gro--; 251 else if (ret < 0) 252 unprocess_pkts[unprocess_num++] = pkts[i]; 253 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) && 254 do_vxlan_udp_gro) { 255 ret = gro_vxlan_udp4_reassemble(pkts[i], 256 &vxlan_udp_tbl, 0); 257 if (ret > 0) 258 /* Merge successfully */ 259 nb_after_gro--; 260 else if (ret < 0) 261 unprocess_pkts[unprocess_num++] = pkts[i]; 262 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && 263 do_tcp4_gro) { 264 ret = gro_tcp4_reassemble(pkts[i], &tcp_tbl, 0); 265 if (ret > 0) 266 /* merge successfully */ 267 nb_after_gro--; 268 else if (ret < 0) 269 unprocess_pkts[unprocess_num++] = pkts[i]; 270 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) && 271 do_udp4_gro) { 272 ret = gro_udp4_reassemble(pkts[i], &udp_tbl, 0); 273 if (ret > 0) 274 /* merge successfully */ 275 nb_after_gro--; 276 else if (ret < 0) 277 unprocess_pkts[unprocess_num++] = pkts[i]; 278 } else 279 unprocess_pkts[unprocess_num++] = pkts[i]; 280 } 281 282 if ((nb_after_gro < nb_pkts) 283 || (unprocess_num < nb_pkts)) { 284 i = 0; 285 /* Flush all packets from the tables */ 286 if (do_vxlan_tcp_gro) { 287 i = gro_vxlan_tcp4_tbl_timeout_flush(&vxlan_tcp_tbl, 288 0, pkts, nb_pkts); 289 } 290 291 if (do_vxlan_udp_gro) { 292 i += gro_vxlan_udp4_tbl_timeout_flush(&vxlan_udp_tbl, 293 0, &pkts[i], nb_pkts - i); 294 295 } 296 297 if (do_tcp4_gro) { 298 i += gro_tcp4_tbl_timeout_flush(&tcp_tbl, 0, 299 &pkts[i], nb_pkts - i); 300 } 301 302 if (do_udp4_gro) { 303 i += gro_udp4_tbl_timeout_flush(&udp_tbl, 0, 304 &pkts[i], nb_pkts - i); 305 } 306 /* Copy unprocessed packets */ 307 if (unprocess_num > 0) { 308 memcpy(&pkts[i], unprocess_pkts, 309 sizeof(struct rte_mbuf *) * 310 unprocess_num); 311 } 312 nb_after_gro = i + unprocess_num; 313 } 314 315 return nb_after_gro; 316 } 317 318 uint16_t 319 rte_gro_reassemble(struct rte_mbuf **pkts, 320 uint16_t nb_pkts, 321 void *ctx) 322 { 323 struct rte_mbuf *unprocess_pkts[nb_pkts]; 324 struct gro_ctx *gro_ctx = ctx; 325 void *tcp_tbl, *udp_tbl, *vxlan_tcp_tbl, *vxlan_udp_tbl; 326 uint64_t current_time; 327 uint16_t i, unprocess_num = 0; 328 uint8_t do_tcp4_gro, do_vxlan_tcp_gro, do_udp4_gro, do_vxlan_udp_gro; 329 330 if (unlikely((gro_ctx->gro_types & (RTE_GRO_IPV4_VXLAN_TCP_IPV4 | 331 RTE_GRO_TCP_IPV4 | 332 RTE_GRO_IPV4_VXLAN_UDP_IPV4 | 333 RTE_GRO_UDP_IPV4)) == 0)) 334 return nb_pkts; 335 336 tcp_tbl = gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX]; 337 vxlan_tcp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX]; 338 udp_tbl = gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX]; 339 vxlan_udp_tbl = gro_ctx->tbls[RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX]; 340 341 do_tcp4_gro = (gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 342 RTE_GRO_TCP_IPV4; 343 do_vxlan_tcp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) == 344 RTE_GRO_IPV4_VXLAN_TCP_IPV4; 345 do_udp4_gro = (gro_ctx->gro_types & RTE_GRO_UDP_IPV4) == 346 RTE_GRO_UDP_IPV4; 347 do_vxlan_udp_gro = (gro_ctx->gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) == 348 RTE_GRO_IPV4_VXLAN_UDP_IPV4; 349 350 current_time = rte_rdtsc(); 351 352 for (i = 0; i < nb_pkts; i++) { 353 if (IS_IPV4_VXLAN_TCP4_PKT(pkts[i]->packet_type) && 354 do_vxlan_tcp_gro) { 355 if (gro_vxlan_tcp4_reassemble(pkts[i], vxlan_tcp_tbl, 356 current_time) < 0) 357 unprocess_pkts[unprocess_num++] = pkts[i]; 358 } else if (IS_IPV4_VXLAN_UDP4_PKT(pkts[i]->packet_type) && 359 do_vxlan_udp_gro) { 360 if (gro_vxlan_udp4_reassemble(pkts[i], vxlan_udp_tbl, 361 current_time) < 0) 362 unprocess_pkts[unprocess_num++] = pkts[i]; 363 } else if (IS_IPV4_TCP_PKT(pkts[i]->packet_type) && 364 do_tcp4_gro) { 365 if (gro_tcp4_reassemble(pkts[i], tcp_tbl, 366 current_time) < 0) 367 unprocess_pkts[unprocess_num++] = pkts[i]; 368 } else if (IS_IPV4_UDP_PKT(pkts[i]->packet_type) && 369 do_udp4_gro) { 370 if (gro_udp4_reassemble(pkts[i], udp_tbl, 371 current_time) < 0) 372 unprocess_pkts[unprocess_num++] = pkts[i]; 373 } else 374 unprocess_pkts[unprocess_num++] = pkts[i]; 375 } 376 if (unprocess_num > 0) { 377 memcpy(pkts, unprocess_pkts, sizeof(struct rte_mbuf *) * 378 unprocess_num); 379 } 380 381 return unprocess_num; 382 } 383 384 uint16_t 385 rte_gro_timeout_flush(void *ctx, 386 uint64_t timeout_cycles, 387 uint64_t gro_types, 388 struct rte_mbuf **out, 389 uint16_t max_nb_out) 390 { 391 struct gro_ctx *gro_ctx = ctx; 392 uint64_t flush_timestamp; 393 uint16_t num = 0; 394 uint16_t left_nb_out = max_nb_out; 395 396 gro_types = gro_types & gro_ctx->gro_types; 397 flush_timestamp = rte_rdtsc() - timeout_cycles; 398 399 if (gro_types & RTE_GRO_IPV4_VXLAN_TCP_IPV4) { 400 num = gro_vxlan_tcp4_tbl_timeout_flush(gro_ctx->tbls[ 401 RTE_GRO_IPV4_VXLAN_TCP_IPV4_INDEX], 402 flush_timestamp, out, left_nb_out); 403 left_nb_out = max_nb_out - num; 404 } 405 406 if ((gro_types & RTE_GRO_IPV4_VXLAN_UDP_IPV4) && left_nb_out > 0) { 407 num += gro_vxlan_udp4_tbl_timeout_flush(gro_ctx->tbls[ 408 RTE_GRO_IPV4_VXLAN_UDP_IPV4_INDEX], 409 flush_timestamp, &out[num], left_nb_out); 410 left_nb_out = max_nb_out - num; 411 } 412 413 /* If no available space in 'out', stop flushing. */ 414 if ((gro_types & RTE_GRO_TCP_IPV4) && left_nb_out > 0) { 415 num += gro_tcp4_tbl_timeout_flush( 416 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX], 417 flush_timestamp, 418 &out[num], left_nb_out); 419 left_nb_out = max_nb_out - num; 420 } 421 422 /* If no available space in 'out', stop flushing. */ 423 if ((gro_types & RTE_GRO_UDP_IPV4) && left_nb_out > 0) { 424 num += gro_udp4_tbl_timeout_flush( 425 gro_ctx->tbls[RTE_GRO_UDP_IPV4_INDEX], 426 flush_timestamp, 427 &out[num], left_nb_out); 428 } 429 430 return num; 431 } 432 433 uint64_t 434 rte_gro_get_pkt_count(void *ctx) 435 { 436 struct gro_ctx *gro_ctx = ctx; 437 gro_tbl_pkt_count_fn pkt_count_fn; 438 uint64_t gro_types = gro_ctx->gro_types, flag; 439 uint64_t item_num = 0; 440 uint8_t i; 441 442 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM && gro_types; i++) { 443 flag = 1ULL << i; 444 if ((gro_types & flag) == 0) 445 continue; 446 447 gro_types ^= flag; 448 pkt_count_fn = tbl_pkt_count_fn[i]; 449 if (pkt_count_fn) 450 item_num += pkt_count_fn(gro_ctx->tbls[i]); 451 } 452 453 return item_num; 454 } 455