1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_malloc.h> 34 #include <rte_mbuf.h> 35 #include <rte_cycles.h> 36 #include <rte_ethdev.h> 37 #include <rte_ip.h> 38 #include <rte_tcp.h> 39 40 #include "gro_tcp4.h" 41 42 void * 43 gro_tcp4_tbl_create(uint16_t socket_id, 44 uint16_t max_flow_num, 45 uint16_t max_item_per_flow) 46 { 47 struct gro_tcp4_tbl *tbl; 48 size_t size; 49 uint32_t entries_num, i; 50 51 entries_num = max_flow_num * max_item_per_flow; 52 entries_num = RTE_MIN(entries_num, GRO_TCP4_TBL_MAX_ITEM_NUM); 53 54 if (entries_num == 0) 55 return NULL; 56 57 tbl = rte_zmalloc_socket(__func__, 58 sizeof(struct gro_tcp4_tbl), 59 RTE_CACHE_LINE_SIZE, 60 socket_id); 61 if (tbl == NULL) 62 return NULL; 63 64 size = sizeof(struct gro_tcp4_item) * entries_num; 65 tbl->items = rte_zmalloc_socket(__func__, 66 size, 67 RTE_CACHE_LINE_SIZE, 68 socket_id); 69 if (tbl->items == NULL) { 70 rte_free(tbl); 71 return NULL; 72 } 73 tbl->max_item_num = entries_num; 74 75 size = sizeof(struct gro_tcp4_key) * entries_num; 76 tbl->keys = rte_zmalloc_socket(__func__, 77 size, 78 RTE_CACHE_LINE_SIZE, 79 socket_id); 80 if (tbl->keys == NULL) { 81 rte_free(tbl->items); 82 rte_free(tbl); 83 return NULL; 84 } 85 /* INVALID_ARRAY_INDEX indicates empty key */ 86 for (i = 0; i < entries_num; i++) 87 tbl->keys[i].start_index = INVALID_ARRAY_INDEX; 88 tbl->max_key_num = entries_num; 89 90 return tbl; 91 } 92 93 void 94 gro_tcp4_tbl_destroy(void *tbl) 95 { 96 struct gro_tcp4_tbl *tcp_tbl = tbl; 97 98 if (tcp_tbl) { 99 rte_free(tcp_tbl->items); 100 rte_free(tcp_tbl->keys); 101 } 102 rte_free(tcp_tbl); 103 } 104 105 /* 106 * merge two TCP/IPv4 packets without updating checksums. 107 * If cmp is larger than 0, append the new packet to the 108 * original packet. Otherwise, pre-pend the new packet to 109 * the original packet. 110 */ 111 static inline int 112 merge_two_tcp4_packets(struct gro_tcp4_item *item_src, 113 struct rte_mbuf *pkt, 114 uint16_t ip_id, 115 uint32_t sent_seq, 116 int cmp) 117 { 118 struct rte_mbuf *pkt_head, *pkt_tail, *lastseg; 119 uint16_t tcp_datalen; 120 121 if (cmp > 0) { 122 pkt_head = item_src->firstseg; 123 pkt_tail = pkt; 124 } else { 125 pkt_head = pkt; 126 pkt_tail = item_src->firstseg; 127 } 128 129 /* check if the packet length will be beyond the max value */ 130 tcp_datalen = pkt_tail->pkt_len - pkt_tail->l2_len - 131 pkt_tail->l3_len - pkt_tail->l4_len; 132 if (pkt_head->pkt_len - pkt_head->l2_len + tcp_datalen > 133 TCP4_MAX_L3_LENGTH) 134 return 0; 135 136 /* remove packet header for the tail packet */ 137 rte_pktmbuf_adj(pkt_tail, 138 pkt_tail->l2_len + 139 pkt_tail->l3_len + 140 pkt_tail->l4_len); 141 142 /* chain two packets together */ 143 if (cmp > 0) { 144 item_src->lastseg->next = pkt; 145 item_src->lastseg = rte_pktmbuf_lastseg(pkt); 146 /* update IP ID to the larger value */ 147 item_src->ip_id = ip_id; 148 } else { 149 lastseg = rte_pktmbuf_lastseg(pkt); 150 lastseg->next = item_src->firstseg; 151 item_src->firstseg = pkt; 152 /* update sent_seq to the smaller value */ 153 item_src->sent_seq = sent_seq; 154 } 155 item_src->nb_merged++; 156 157 /* update mbuf metadata for the merged packet */ 158 pkt_head->nb_segs += pkt_tail->nb_segs; 159 pkt_head->pkt_len += pkt_tail->pkt_len; 160 161 return 1; 162 } 163 164 static inline int 165 check_seq_option(struct gro_tcp4_item *item, 166 struct tcp_hdr *tcp_hdr, 167 uint16_t tcp_hl, 168 uint16_t tcp_dl, 169 uint16_t ip_id, 170 uint32_t sent_seq) 171 { 172 struct rte_mbuf *pkt0 = item->firstseg; 173 struct ipv4_hdr *ipv4_hdr0; 174 struct tcp_hdr *tcp_hdr0; 175 uint16_t tcp_hl0, tcp_dl0; 176 uint16_t len; 177 178 ipv4_hdr0 = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt0, char *) + 179 pkt0->l2_len); 180 tcp_hdr0 = (struct tcp_hdr *)((char *)ipv4_hdr0 + pkt0->l3_len); 181 tcp_hl0 = pkt0->l4_len; 182 183 /* check if TCP option fields equal. If not, return 0. */ 184 len = RTE_MAX(tcp_hl, tcp_hl0) - sizeof(struct tcp_hdr); 185 if ((tcp_hl != tcp_hl0) || 186 ((len > 0) && (memcmp(tcp_hdr + 1, 187 tcp_hdr0 + 1, 188 len) != 0))) 189 return 0; 190 191 /* check if the two packets are neighbors */ 192 tcp_dl0 = pkt0->pkt_len - pkt0->l2_len - pkt0->l3_len - tcp_hl0; 193 if ((sent_seq == (item->sent_seq + tcp_dl0)) && 194 (ip_id == (item->ip_id + 1))) 195 /* append the new packet */ 196 return 1; 197 else if (((sent_seq + tcp_dl) == item->sent_seq) && 198 ((ip_id + item->nb_merged) == item->ip_id)) 199 /* pre-pend the new packet */ 200 return -1; 201 else 202 return 0; 203 } 204 205 static inline uint32_t 206 find_an_empty_item(struct gro_tcp4_tbl *tbl) 207 { 208 uint32_t i; 209 uint32_t max_item_num = tbl->max_item_num; 210 211 for (i = 0; i < max_item_num; i++) 212 if (tbl->items[i].firstseg == NULL) 213 return i; 214 return INVALID_ARRAY_INDEX; 215 } 216 217 static inline uint32_t 218 find_an_empty_key(struct gro_tcp4_tbl *tbl) 219 { 220 uint32_t i; 221 uint32_t max_key_num = tbl->max_key_num; 222 223 for (i = 0; i < max_key_num; i++) 224 if (tbl->keys[i].start_index == INVALID_ARRAY_INDEX) 225 return i; 226 return INVALID_ARRAY_INDEX; 227 } 228 229 static inline uint32_t 230 insert_new_item(struct gro_tcp4_tbl *tbl, 231 struct rte_mbuf *pkt, 232 uint16_t ip_id, 233 uint32_t sent_seq, 234 uint32_t prev_idx, 235 uint64_t start_time) 236 { 237 uint32_t item_idx; 238 239 item_idx = find_an_empty_item(tbl); 240 if (item_idx == INVALID_ARRAY_INDEX) 241 return INVALID_ARRAY_INDEX; 242 243 tbl->items[item_idx].firstseg = pkt; 244 tbl->items[item_idx].lastseg = rte_pktmbuf_lastseg(pkt); 245 tbl->items[item_idx].start_time = start_time; 246 tbl->items[item_idx].next_pkt_idx = INVALID_ARRAY_INDEX; 247 tbl->items[item_idx].sent_seq = sent_seq; 248 tbl->items[item_idx].ip_id = ip_id; 249 tbl->items[item_idx].nb_merged = 1; 250 tbl->item_num++; 251 252 /* if the previous packet exists, chain the new one with it */ 253 if (prev_idx != INVALID_ARRAY_INDEX) { 254 tbl->items[item_idx].next_pkt_idx = 255 tbl->items[prev_idx].next_pkt_idx; 256 tbl->items[prev_idx].next_pkt_idx = item_idx; 257 } 258 259 return item_idx; 260 } 261 262 static inline uint32_t 263 delete_item(struct gro_tcp4_tbl *tbl, uint32_t item_idx, 264 uint32_t prev_item_idx) 265 { 266 uint32_t next_idx = tbl->items[item_idx].next_pkt_idx; 267 268 /* set NULL to firstseg to indicate it's an empty item */ 269 tbl->items[item_idx].firstseg = NULL; 270 tbl->item_num--; 271 if (prev_item_idx != INVALID_ARRAY_INDEX) 272 tbl->items[prev_item_idx].next_pkt_idx = next_idx; 273 274 return next_idx; 275 } 276 277 static inline uint32_t 278 insert_new_key(struct gro_tcp4_tbl *tbl, 279 struct tcp4_key *key_src, 280 uint32_t item_idx) 281 { 282 struct tcp4_key *key_dst; 283 uint32_t key_idx; 284 285 key_idx = find_an_empty_key(tbl); 286 if (key_idx == INVALID_ARRAY_INDEX) 287 return INVALID_ARRAY_INDEX; 288 289 key_dst = &(tbl->keys[key_idx].key); 290 291 ether_addr_copy(&(key_src->eth_saddr), &(key_dst->eth_saddr)); 292 ether_addr_copy(&(key_src->eth_daddr), &(key_dst->eth_daddr)); 293 key_dst->ip_src_addr = key_src->ip_src_addr; 294 key_dst->ip_dst_addr = key_src->ip_dst_addr; 295 key_dst->recv_ack = key_src->recv_ack; 296 key_dst->src_port = key_src->src_port; 297 key_dst->dst_port = key_src->dst_port; 298 299 /* non-INVALID_ARRAY_INDEX value indicates this key is valid */ 300 tbl->keys[key_idx].start_index = item_idx; 301 tbl->key_num++; 302 303 return key_idx; 304 } 305 306 static inline int 307 is_same_key(struct tcp4_key k1, struct tcp4_key k2) 308 { 309 if (is_same_ether_addr(&k1.eth_saddr, &k2.eth_saddr) == 0) 310 return 0; 311 312 if (is_same_ether_addr(&k1.eth_daddr, &k2.eth_daddr) == 0) 313 return 0; 314 315 return ((k1.ip_src_addr == k2.ip_src_addr) && 316 (k1.ip_dst_addr == k2.ip_dst_addr) && 317 (k1.recv_ack == k2.recv_ack) && 318 (k1.src_port == k2.src_port) && 319 (k1.dst_port == k2.dst_port)); 320 } 321 322 /* 323 * update packet length for the flushed packet. 324 */ 325 static inline void 326 update_header(struct gro_tcp4_item *item) 327 { 328 struct ipv4_hdr *ipv4_hdr; 329 struct rte_mbuf *pkt = item->firstseg; 330 331 ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(pkt, char *) + 332 pkt->l2_len); 333 ipv4_hdr->total_length = rte_cpu_to_be_16(pkt->pkt_len - 334 pkt->l2_len); 335 } 336 337 int32_t 338 gro_tcp4_reassemble(struct rte_mbuf *pkt, 339 struct gro_tcp4_tbl *tbl, 340 uint64_t start_time) 341 { 342 struct ether_hdr *eth_hdr; 343 struct ipv4_hdr *ipv4_hdr; 344 struct tcp_hdr *tcp_hdr; 345 uint32_t sent_seq; 346 uint16_t tcp_dl, ip_id; 347 348 struct tcp4_key key; 349 uint32_t cur_idx, prev_idx, item_idx; 350 uint32_t i, max_key_num; 351 int cmp; 352 353 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); 354 ipv4_hdr = (struct ipv4_hdr *)((char *)eth_hdr + pkt->l2_len); 355 tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + pkt->l3_len); 356 357 /* 358 * if FIN, SYN, RST, PSH, URG, ECE or 359 * CWR is set, return immediately. 360 */ 361 if (tcp_hdr->tcp_flags != TCP_ACK_FLAG) 362 return -1; 363 /* if payload length is 0, return immediately */ 364 tcp_dl = rte_be_to_cpu_16(ipv4_hdr->total_length) - pkt->l3_len - 365 pkt->l4_len; 366 if (tcp_dl == 0) 367 return -1; 368 369 ip_id = rte_be_to_cpu_16(ipv4_hdr->packet_id); 370 sent_seq = rte_be_to_cpu_32(tcp_hdr->sent_seq); 371 372 ether_addr_copy(&(eth_hdr->s_addr), &(key.eth_saddr)); 373 ether_addr_copy(&(eth_hdr->d_addr), &(key.eth_daddr)); 374 key.ip_src_addr = ipv4_hdr->src_addr; 375 key.ip_dst_addr = ipv4_hdr->dst_addr; 376 key.src_port = tcp_hdr->src_port; 377 key.dst_port = tcp_hdr->dst_port; 378 key.recv_ack = tcp_hdr->recv_ack; 379 380 /* search for a key */ 381 max_key_num = tbl->max_key_num; 382 for (i = 0; i < max_key_num; i++) { 383 if ((tbl->keys[i].start_index != INVALID_ARRAY_INDEX) && 384 is_same_key(tbl->keys[i].key, key)) 385 break; 386 } 387 388 /* can't find a key, so insert a new key and a new item. */ 389 if (i == tbl->max_key_num) { 390 item_idx = insert_new_item(tbl, pkt, ip_id, sent_seq, 391 INVALID_ARRAY_INDEX, start_time); 392 if (item_idx == INVALID_ARRAY_INDEX) 393 return -1; 394 if (insert_new_key(tbl, &key, item_idx) == 395 INVALID_ARRAY_INDEX) { 396 /* 397 * fail to insert a new key, so 398 * delete the inserted item 399 */ 400 delete_item(tbl, item_idx, INVALID_ARRAY_INDEX); 401 return -1; 402 } 403 return 0; 404 } 405 406 /* traverse all packets in the item group to find one to merge */ 407 cur_idx = tbl->keys[i].start_index; 408 prev_idx = cur_idx; 409 do { 410 cmp = check_seq_option(&(tbl->items[cur_idx]), tcp_hdr, 411 pkt->l4_len, tcp_dl, ip_id, sent_seq); 412 if (cmp) { 413 if (merge_two_tcp4_packets(&(tbl->items[cur_idx]), 414 pkt, ip_id, 415 sent_seq, cmp)) 416 return 1; 417 /* 418 * fail to merge two packets since the packet 419 * length will be greater than the max value. 420 * So insert the packet into the item group. 421 */ 422 if (insert_new_item(tbl, pkt, ip_id, sent_seq, 423 prev_idx, start_time) == 424 INVALID_ARRAY_INDEX) 425 return -1; 426 return 0; 427 } 428 prev_idx = cur_idx; 429 cur_idx = tbl->items[cur_idx].next_pkt_idx; 430 } while (cur_idx != INVALID_ARRAY_INDEX); 431 432 /* 433 * can't find a packet in the item group to merge, 434 * so insert the packet into the item group. 435 */ 436 if (insert_new_item(tbl, pkt, ip_id, sent_seq, prev_idx, 437 start_time) == INVALID_ARRAY_INDEX) 438 return -1; 439 440 return 0; 441 } 442 443 uint16_t 444 gro_tcp4_tbl_timeout_flush(struct gro_tcp4_tbl *tbl, 445 uint64_t flush_timestamp, 446 struct rte_mbuf **out, 447 uint16_t nb_out) 448 { 449 uint16_t k = 0; 450 uint32_t i, j; 451 uint32_t max_key_num = tbl->max_key_num; 452 453 for (i = 0; i < max_key_num; i++) { 454 /* all keys have been checked, return immediately */ 455 if (tbl->key_num == 0) 456 return k; 457 458 j = tbl->keys[i].start_index; 459 while (j != INVALID_ARRAY_INDEX) { 460 if (tbl->items[j].start_time <= flush_timestamp) { 461 out[k++] = tbl->items[j].firstseg; 462 if (tbl->items[j].nb_merged > 1) 463 update_header(&(tbl->items[j])); 464 /* 465 * delete the item and get 466 * the next packet index 467 */ 468 j = delete_item(tbl, j, 469 INVALID_ARRAY_INDEX); 470 471 /* 472 * delete the key as all of 473 * packets are flushed 474 */ 475 if (j == INVALID_ARRAY_INDEX) { 476 tbl->keys[i].start_index = 477 INVALID_ARRAY_INDEX; 478 tbl->key_num--; 479 } else 480 /* update start_index of the key */ 481 tbl->keys[i].start_index = j; 482 483 if (k == nb_out) 484 return k; 485 } else 486 /* 487 * left packets of this key won't be 488 * timeout, so go to check other keys. 489 */ 490 break; 491 } 492 } 493 return k; 494 } 495 496 uint32_t 497 gro_tcp4_tbl_pkt_count(void *tbl) 498 { 499 struct gro_tcp4_tbl *gro_tbl = tbl; 500 501 if (gro_tbl) 502 return gro_tbl->item_num; 503 504 return 0; 505 } 506