1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_malloc.h> 34 #include <rte_mbuf.h> 35 #include <rte_cycles.h> 36 #include <rte_ethdev.h> 37 38 #include "rte_gro.h" 39 #include "gro_tcp4.h" 40 41 typedef void *(*gro_tbl_create_fn)(uint16_t socket_id, 42 uint16_t max_flow_num, 43 uint16_t max_item_per_flow); 44 typedef void (*gro_tbl_destroy_fn)(void *tbl); 45 typedef uint32_t (*gro_tbl_pkt_count_fn)(void *tbl); 46 47 static gro_tbl_create_fn tbl_create_fn[RTE_GRO_TYPE_MAX_NUM] = { 48 gro_tcp4_tbl_create, NULL}; 49 static gro_tbl_destroy_fn tbl_destroy_fn[RTE_GRO_TYPE_MAX_NUM] = { 50 gro_tcp4_tbl_destroy, NULL}; 51 static gro_tbl_pkt_count_fn tbl_pkt_count_fn[RTE_GRO_TYPE_MAX_NUM] = { 52 gro_tcp4_tbl_pkt_count, NULL}; 53 54 /* 55 * GRO context structure, which is used to merge packets. It keeps 56 * many reassembly tables of desired GRO types. Applications need to 57 * create GRO context objects before using rte_gro_reassemble to 58 * perform GRO. 59 */ 60 struct gro_ctx { 61 /* GRO types to perform */ 62 uint64_t gro_types; 63 /* reassembly tables */ 64 void *tbls[RTE_GRO_TYPE_MAX_NUM]; 65 }; 66 67 void * 68 rte_gro_ctx_create(const struct rte_gro_param *param) 69 { 70 struct gro_ctx *gro_ctx; 71 gro_tbl_create_fn create_tbl_fn; 72 uint64_t gro_type_flag = 0; 73 uint64_t gro_types = 0; 74 uint8_t i; 75 76 gro_ctx = rte_zmalloc_socket(__func__, 77 sizeof(struct gro_ctx), 78 RTE_CACHE_LINE_SIZE, 79 param->socket_id); 80 if (gro_ctx == NULL) 81 return NULL; 82 83 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { 84 gro_type_flag = 1ULL << i; 85 if ((param->gro_types & gro_type_flag) == 0) 86 continue; 87 88 create_tbl_fn = tbl_create_fn[i]; 89 if (create_tbl_fn == NULL) 90 continue; 91 92 gro_ctx->tbls[i] = create_tbl_fn(param->socket_id, 93 param->max_flow_num, 94 param->max_item_per_flow); 95 if (gro_ctx->tbls[i] == NULL) { 96 /* destroy all created tables */ 97 gro_ctx->gro_types = gro_types; 98 rte_gro_ctx_destroy(gro_ctx); 99 return NULL; 100 } 101 gro_types |= gro_type_flag; 102 } 103 gro_ctx->gro_types = param->gro_types; 104 105 return gro_ctx; 106 } 107 108 void 109 rte_gro_ctx_destroy(void *ctx) 110 { 111 gro_tbl_destroy_fn destroy_tbl_fn; 112 struct gro_ctx *gro_ctx = ctx; 113 uint64_t gro_type_flag; 114 uint8_t i; 115 116 if (gro_ctx == NULL) 117 return; 118 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { 119 gro_type_flag = 1ULL << i; 120 if ((gro_ctx->gro_types & gro_type_flag) == 0) 121 continue; 122 destroy_tbl_fn = tbl_destroy_fn[i]; 123 if (destroy_tbl_fn) 124 destroy_tbl_fn(gro_ctx->tbls[i]); 125 } 126 rte_free(gro_ctx); 127 } 128 129 uint16_t 130 rte_gro_reassemble_burst(struct rte_mbuf **pkts, 131 uint16_t nb_pkts, 132 const struct rte_gro_param *param) 133 { 134 uint16_t i; 135 uint16_t nb_after_gro = nb_pkts; 136 uint32_t item_num; 137 138 /* allocate a reassembly table for TCP/IPv4 GRO */ 139 struct gro_tcp4_tbl tcp_tbl; 140 struct gro_tcp4_key tcp_keys[RTE_GRO_MAX_BURST_ITEM_NUM]; 141 struct gro_tcp4_item tcp_items[RTE_GRO_MAX_BURST_ITEM_NUM] = {{0} }; 142 143 struct rte_mbuf *unprocess_pkts[nb_pkts]; 144 uint16_t unprocess_num = 0; 145 int32_t ret; 146 uint64_t current_time; 147 148 if ((param->gro_types & RTE_GRO_TCP_IPV4) == 0) 149 return nb_pkts; 150 151 /* get the actual number of packets */ 152 item_num = RTE_MIN(nb_pkts, (param->max_flow_num * 153 param->max_item_per_flow)); 154 item_num = RTE_MIN(item_num, RTE_GRO_MAX_BURST_ITEM_NUM); 155 156 for (i = 0; i < item_num; i++) 157 tcp_keys[i].start_index = INVALID_ARRAY_INDEX; 158 159 tcp_tbl.keys = tcp_keys; 160 tcp_tbl.items = tcp_items; 161 tcp_tbl.key_num = 0; 162 tcp_tbl.item_num = 0; 163 tcp_tbl.max_key_num = item_num; 164 tcp_tbl.max_item_num = item_num; 165 166 current_time = rte_rdtsc(); 167 168 for (i = 0; i < nb_pkts; i++) { 169 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | 170 RTE_PTYPE_L4_TCP)) == 171 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { 172 ret = gro_tcp4_reassemble(pkts[i], 173 &tcp_tbl, 174 current_time); 175 if (ret > 0) 176 /* merge successfully */ 177 nb_after_gro--; 178 else if (ret < 0) { 179 unprocess_pkts[unprocess_num++] = 180 pkts[i]; 181 } 182 } else 183 unprocess_pkts[unprocess_num++] = pkts[i]; 184 } 185 186 /* re-arrange GROed packets */ 187 if (nb_after_gro < nb_pkts) { 188 i = gro_tcp4_tbl_timeout_flush(&tcp_tbl, current_time, 189 pkts, nb_pkts); 190 if (unprocess_num > 0) { 191 memcpy(&pkts[i], unprocess_pkts, 192 sizeof(struct rte_mbuf *) * 193 unprocess_num); 194 } 195 } 196 197 return nb_after_gro; 198 } 199 200 uint16_t 201 rte_gro_reassemble(struct rte_mbuf **pkts, 202 uint16_t nb_pkts, 203 void *ctx) 204 { 205 uint16_t i, unprocess_num = 0; 206 struct rte_mbuf *unprocess_pkts[nb_pkts]; 207 struct gro_ctx *gro_ctx = ctx; 208 uint64_t current_time; 209 210 if ((gro_ctx->gro_types & RTE_GRO_TCP_IPV4) == 0) 211 return nb_pkts; 212 213 current_time = rte_rdtsc(); 214 215 for (i = 0; i < nb_pkts; i++) { 216 if ((pkts[i]->packet_type & (RTE_PTYPE_L3_IPV4 | 217 RTE_PTYPE_L4_TCP)) == 218 (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP)) { 219 if (gro_tcp4_reassemble(pkts[i], 220 gro_ctx->tbls 221 [RTE_GRO_TCP_IPV4_INDEX], 222 current_time) < 0) 223 unprocess_pkts[unprocess_num++] = pkts[i]; 224 } else 225 unprocess_pkts[unprocess_num++] = pkts[i]; 226 } 227 if (unprocess_num > 0) { 228 memcpy(pkts, unprocess_pkts, 229 sizeof(struct rte_mbuf *) * 230 unprocess_num); 231 } 232 233 return unprocess_num; 234 } 235 236 uint16_t 237 rte_gro_timeout_flush(void *ctx, 238 uint64_t timeout_cycles, 239 uint64_t gro_types, 240 struct rte_mbuf **out, 241 uint16_t max_nb_out) 242 { 243 struct gro_ctx *gro_ctx = ctx; 244 uint64_t flush_timestamp; 245 246 gro_types = gro_types & gro_ctx->gro_types; 247 flush_timestamp = rte_rdtsc() - timeout_cycles; 248 249 if (gro_types & RTE_GRO_TCP_IPV4) { 250 return gro_tcp4_tbl_timeout_flush( 251 gro_ctx->tbls[RTE_GRO_TCP_IPV4_INDEX], 252 flush_timestamp, 253 out, max_nb_out); 254 } 255 return 0; 256 } 257 258 uint64_t 259 rte_gro_get_pkt_count(void *ctx) 260 { 261 struct gro_ctx *gro_ctx = ctx; 262 gro_tbl_pkt_count_fn pkt_count_fn; 263 uint64_t item_num = 0; 264 uint64_t gro_type_flag; 265 uint8_t i; 266 267 for (i = 0; i < RTE_GRO_TYPE_MAX_NUM; i++) { 268 gro_type_flag = 1ULL << i; 269 if ((gro_ctx->gro_types & gro_type_flag) == 0) 270 continue; 271 272 pkt_count_fn = tbl_pkt_count_fn[i]; 273 if (pkt_count_fn == NULL) 274 continue; 275 item_num += pkt_count_fn(gro_ctx->tbls[i]); 276 } 277 return item_num; 278 } 279