1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 #include <assert.h> 27 #include <unistd.h> 28 #include <sys/mman.h> 29 #include <errno.h> 30 31 #include <rte_common.h> 32 #include <rte_byteorder.h> 33 #include <rte_log.h> 34 #include <rte_memory.h> 35 #include <rte_memcpy.h> 36 #include <rte_memzone.h> 37 #include <rte_config.h> 38 #include <rte_eal.h> 39 #include <rte_pci.h> 40 #include <rte_mbuf.h> 41 #include <rte_memory.h> 42 #include <rte_lcore.h> 43 #include <rte_launch.h> 44 #include <rte_ethdev.h> 45 #include <rte_debug.h> 46 #include <rte_common.h> 47 #include <rte_ether.h> 48 #include <rte_malloc.h> 49 #include <rte_cycles.h> 50 #include <rte_timer.h> 51 #include <rte_thash.h> 52 #include <rte_ip.h> 53 #include <rte_tcp.h> 54 #include <rte_udp.h> 55 #include <rte_eth_bond.h> 56 57 #include "ff_dpdk_if.h" 58 #include "ff_dpdk_pcap.h" 59 #include "ff_dpdk_kni.h" 60 #include "ff_config.h" 61 #include "ff_veth.h" 62 #include "ff_host_interface.h" 63 #include "ff_msg.h" 64 #include "ff_api.h" 65 #include "ff_memory.h" 66 67 #ifdef FF_KNI 68 #define KNI_MBUF_MAX 2048 69 #define KNI_QUEUE_SIZE 2048 70 71 int enable_kni; 72 static int kni_accept; 73 static int knictl_action = FF_KNICTL_ACTION_DEFAULT; 74 #endif 75 76 static int numa_on; 77 78 static unsigned idle_sleep; 79 static unsigned pkt_tx_delay; 80 81 static struct rte_timer freebsd_clock; 82 83 // Mellanox Linux's driver key 84 static uint8_t default_rsskey_40bytes[40] = { 85 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 86 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 87 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 88 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 89 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 90 }; 91 92 static uint8_t default_rsskey_52bytes[52] = { 93 0x44, 0x39, 0x79, 0x6b, 0xb5, 0x4c, 0x50, 0x23, 94 0xb6, 0x75, 0xea, 0x5b, 0x12, 0x4f, 0x9f, 0x30, 95 0xb8, 0xa2, 0xc0, 0x3d, 0xdf, 0xdc, 0x4d, 0x02, 96 0xa0, 0x8c, 0x9b, 0x33, 0x4a, 0xf6, 0x4a, 0x4c, 97 0x05, 0xc6, 0xfa, 0x34, 0x39, 0x58, 0xd8, 0x55, 98 0x7d, 0x99, 0x58, 0x3a, 0xe1, 0x38, 0xc9, 0x2e, 99 0x81, 0x15, 0x03, 0x66 100 }; 101 102 static uint8_t symmetric_rsskey[52] = { 103 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 104 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 105 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 106 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 107 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 108 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 109 0x6d, 0x5a, 0x6d, 0x5a 110 }; 111 112 static int rsskey_len = sizeof(default_rsskey_40bytes); 113 static uint8_t *rsskey = default_rsskey_40bytes; 114 115 struct lcore_conf lcore_conf; 116 117 struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 118 119 static struct rte_ring **dispatch_ring[RTE_MAX_ETHPORTS]; 120 static dispatch_func_t packet_dispatcher; 121 122 static uint16_t rss_reta_size[RTE_MAX_ETHPORTS]; 123 124 #define BOND_DRIVER_NAME "net_bonding" 125 126 static inline int send_single_packet(struct rte_mbuf *m, uint8_t port); 127 128 struct ff_msg_ring { 129 char ring_name[FF_MSG_NUM][RTE_RING_NAMESIZE]; 130 /* ring[0] for lcore recv msg, other send */ 131 /* ring[1] for lcore send msg, other read */ 132 struct rte_ring *ring[FF_MSG_NUM]; 133 } __rte_cache_aligned; 134 135 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 136 static struct rte_mempool *message_pool; 137 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 138 139 static struct ff_top_args ff_top_status; 140 static struct ff_traffic_args ff_traffic; 141 extern void ff_hardclock(void); 142 143 static void 144 ff_hardclock_job(__rte_unused struct rte_timer *timer, 145 __rte_unused void *arg) { 146 ff_hardclock(); 147 ff_update_current_ts(); 148 } 149 150 struct ff_dpdk_if_context * 151 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 152 { 153 struct ff_dpdk_if_context *ctx; 154 155 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 156 if (ctx == NULL) 157 return NULL; 158 159 ctx->sc = sc; 160 ctx->ifp = ifp; 161 ctx->port_id = cfg->port_id; 162 ctx->hw_features = cfg->hw_features; 163 164 return ctx; 165 } 166 167 void 168 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 169 { 170 free(ctx); 171 } 172 173 static void 174 check_all_ports_link_status(void) 175 { 176 #define CHECK_INTERVAL 100 /* 100ms */ 177 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 178 179 uint16_t portid; 180 uint8_t count, all_ports_up, print_flag = 0; 181 struct rte_eth_link link; 182 183 printf("\nChecking link status"); 184 fflush(stdout); 185 186 int i, nb_ports; 187 nb_ports = ff_global_cfg.dpdk.nb_ports; 188 for (count = 0; count <= MAX_CHECK_TIME; count++) { 189 all_ports_up = 1; 190 for (i = 0; i < nb_ports; i++) { 191 uint16_t portid = ff_global_cfg.dpdk.portid_list[i]; 192 memset(&link, 0, sizeof(link)); 193 rte_eth_link_get_nowait(portid, &link); 194 195 /* print link status if flag set */ 196 if (print_flag == 1) { 197 if (link.link_status) { 198 printf("Port %d Link Up - speed %u " 199 "Mbps - %s\n", (int)portid, 200 (unsigned)link.link_speed, 201 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 202 ("full-duplex") : ("half-duplex\n")); 203 } else { 204 printf("Port %d Link Down\n", (int)portid); 205 } 206 continue; 207 } 208 /* clear all_ports_up flag if any link down */ 209 if (link.link_status == 0) { 210 all_ports_up = 0; 211 break; 212 } 213 } 214 215 /* after finally printing all link status, get out */ 216 if (print_flag == 1) 217 break; 218 219 if (all_ports_up == 0) { 220 printf("."); 221 fflush(stdout); 222 rte_delay_ms(CHECK_INTERVAL); 223 } 224 225 /* set the print_flag if all ports up or timeout */ 226 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 227 print_flag = 1; 228 printf("done\n"); 229 } 230 } 231 } 232 233 static int 234 init_lcore_conf(void) 235 { 236 uint8_t nb_dev_ports = rte_eth_dev_count_avail(); 237 if (nb_dev_ports == 0) { 238 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 239 } 240 241 if (ff_global_cfg.dpdk.max_portid >= nb_dev_ports) { 242 rte_exit(EXIT_FAILURE, "this machine doesn't have port %d.\n", 243 ff_global_cfg.dpdk.max_portid); 244 } 245 246 lcore_conf.port_cfgs = ff_global_cfg.dpdk.port_cfgs; 247 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 248 249 uint16_t socket_id = 0; 250 if (numa_on) { 251 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 252 } 253 254 lcore_conf.socket_id = socket_id; 255 256 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[lcore_conf.proc_id]; 257 if (!rte_lcore_is_enabled(lcore_id)) { 258 rte_exit(EXIT_FAILURE, "lcore %u unavailable\n", lcore_id); 259 } 260 261 int j; 262 for (j = 0; j < ff_global_cfg.dpdk.nb_ports; ++j) { 263 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 264 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 265 266 int queueid = -1; 267 int i; 268 for (i = 0; i < pconf->nb_lcores; i++) { 269 if (pconf->lcore_list[i] == lcore_id) { 270 queueid = i; 271 } 272 } 273 if (queueid < 0) { 274 continue; 275 } 276 printf("lcore: %u, port: %u, queue: %u\n", lcore_id, port_id, queueid); 277 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 278 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 279 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = queueid; 280 lcore_conf.nb_rx_queue++; 281 282 lcore_conf.tx_queue_id[port_id] = queueid; 283 lcore_conf.tx_port_id[lcore_conf.nb_tx_port] = port_id; 284 lcore_conf.nb_tx_port++; 285 286 /* Enable pcap dump */ 287 if (ff_global_cfg.pcap.enable) { 288 ff_enable_pcap(ff_global_cfg.pcap.save_path, ff_global_cfg.pcap.snap_len); 289 } 290 291 lcore_conf.nb_queue_list[port_id] = pconf->nb_lcores; 292 } 293 294 if (lcore_conf.nb_rx_queue == 0) { 295 rte_exit(EXIT_FAILURE, "lcore %u has nothing to do\n", lcore_id); 296 } 297 298 return 0; 299 } 300 301 static int 302 init_mem_pool(void) 303 { 304 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 305 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 306 uint32_t nb_tx_queue = nb_lcores; 307 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 308 uint16_t max_portid = ff_global_cfg.dpdk.max_portid; 309 310 unsigned nb_mbuf = RTE_ALIGN_CEIL ( 311 (nb_rx_queue * (max_portid + 1) * 2 * RX_QUEUE_SIZE + 312 nb_ports * (max_portid + 1) * 2 * nb_lcores * MAX_PKT_BURST + 313 nb_ports * (max_portid + 1) * 2 * nb_tx_queue * TX_QUEUE_SIZE + 314 nb_lcores * MEMPOOL_CACHE_SIZE + 315 #ifdef FF_KNI 316 nb_ports * KNI_MBUF_MAX + 317 nb_ports * KNI_QUEUE_SIZE + 318 #endif 319 nb_lcores * nb_ports * DISPATCH_RING_SIZE), 320 (unsigned)8192); 321 322 unsigned socketid = 0; 323 uint16_t i, lcore_id; 324 char s[64]; 325 326 for (i = 0; i < ff_global_cfg.dpdk.nb_procs; i++) { 327 lcore_id = ff_global_cfg.dpdk.proc_lcore[i]; 328 if (numa_on) { 329 socketid = rte_lcore_to_socket_id(lcore_id); 330 } 331 332 if (socketid >= NB_SOCKETS) { 333 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 334 socketid, i, NB_SOCKETS); 335 } 336 337 if (pktmbuf_pool[socketid] != NULL) { 338 continue; 339 } 340 341 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 342 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 343 pktmbuf_pool[socketid] = 344 rte_pktmbuf_pool_create(s, nb_mbuf, 345 MEMPOOL_CACHE_SIZE, 0, 346 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 347 } else { 348 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 349 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 350 } 351 352 if (pktmbuf_pool[socketid] == NULL) { 353 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 354 } else { 355 printf("create mbuf pool on socket %d\n", socketid); 356 } 357 358 #ifdef FF_USE_PAGE_ARRAY 359 nb_mbuf = RTE_ALIGN_CEIL ( 360 nb_ports*nb_lcores*MAX_PKT_BURST + 361 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 362 nb_lcores*MEMPOOL_CACHE_SIZE, 363 (unsigned)4096); 364 ff_init_ref_pool(nb_mbuf, socketid); 365 #endif 366 } 367 368 return 0; 369 } 370 371 static struct rte_ring * 372 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 373 { 374 struct rte_ring *ring; 375 376 if (name == NULL) { 377 rte_exit(EXIT_FAILURE, "create ring failed, no name!\n"); 378 } 379 380 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 381 ring = rte_ring_create(name, count, socket_id, flags); 382 } else { 383 ring = rte_ring_lookup(name); 384 } 385 386 if (ring == NULL) { 387 rte_exit(EXIT_FAILURE, "create ring:%s failed!\n", name); 388 } 389 390 return ring; 391 } 392 393 static int 394 init_dispatch_ring(void) 395 { 396 int j; 397 char name_buf[RTE_RING_NAMESIZE]; 398 int queueid; 399 400 unsigned socketid = lcore_conf.socket_id; 401 402 /* Create ring according to ports actually being used. */ 403 int nb_ports = ff_global_cfg.dpdk.nb_ports; 404 for (j = 0; j < nb_ports; j++) { 405 uint16_t portid = ff_global_cfg.dpdk.portid_list[j]; 406 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[portid]; 407 int nb_queues = pconf->nb_lcores; 408 if (dispatch_ring[portid] == NULL) { 409 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_p%d", portid); 410 411 dispatch_ring[portid] = rte_zmalloc(name_buf, 412 sizeof(struct rte_ring *) * nb_queues, 413 RTE_CACHE_LINE_SIZE); 414 if (dispatch_ring[portid] == NULL) { 415 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 416 "failed\n", name_buf); 417 } 418 } 419 420 for(queueid = 0; queueid < nb_queues; ++queueid) { 421 snprintf(name_buf, RTE_RING_NAMESIZE, "dispatch_ring_p%d_q%d", 422 portid, queueid); 423 dispatch_ring[portid][queueid] = create_ring(name_buf, 424 DISPATCH_RING_SIZE, socketid, RING_F_SC_DEQ); 425 426 if (dispatch_ring[portid][queueid] == NULL) 427 rte_panic("create ring:%s failed!\n", name_buf); 428 429 printf("create ring:%s success, %u ring entries are now free!\n", 430 name_buf, rte_ring_free_count(dispatch_ring[portid][queueid])); 431 } 432 } 433 434 return 0; 435 } 436 437 static void 438 ff_msg_init(struct rte_mempool *mp, 439 __attribute__((unused)) void *opaque_arg, 440 void *obj, __attribute__((unused)) unsigned i) 441 { 442 struct ff_msg *msg = (struct ff_msg *)obj; 443 msg->msg_type = FF_UNKNOWN; 444 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 445 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 446 msg->original_buf = NULL; 447 msg->original_buf_len = 0; 448 } 449 450 static int 451 init_msg_ring(void) 452 { 453 uint16_t i, j; 454 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 455 unsigned socketid = lcore_conf.socket_id; 456 457 /* Create message buffer pool */ 458 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 459 message_pool = rte_mempool_create(FF_MSG_POOL, 460 MSG_RING_SIZE * 2 * nb_procs, 461 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 462 NULL, NULL, ff_msg_init, NULL, 463 socketid, 0); 464 } else { 465 message_pool = rte_mempool_lookup(FF_MSG_POOL); 466 } 467 468 if (message_pool == NULL) { 469 rte_panic("Create msg mempool failed\n"); 470 } 471 472 for(i = 0; i < nb_procs; ++i) { 473 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 474 "%s%u", FF_MSG_RING_IN, i); 475 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 476 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 477 if (msg_ring[i].ring[0] == NULL) 478 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 479 480 for (j = FF_SYSCTL; j < FF_MSG_NUM; j++) { 481 snprintf(msg_ring[i].ring_name[j], RTE_RING_NAMESIZE, 482 "%s%u_%u", FF_MSG_RING_OUT, i, j); 483 msg_ring[i].ring[j] = create_ring(msg_ring[i].ring_name[j], 484 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 485 if (msg_ring[i].ring[j] == NULL) 486 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[j]); 487 } 488 } 489 490 return 0; 491 } 492 493 #ifdef FF_KNI 494 495 static enum FF_KNICTL_CMD get_kni_action(const char *c){ 496 if (!c) 497 return FF_KNICTL_ACTION_DEFAULT; 498 if (0 == strcasecmp(c, "alltokni")){ 499 return FF_KNICTL_ACTION_ALL_TO_KNI; 500 } else if (0 == strcasecmp(c, "alltoff")){ 501 return FF_KNICTL_ACTION_ALL_TO_FF; 502 } else if (0 == strcasecmp(c, "default")){ 503 return FF_KNICTL_ACTION_DEFAULT; 504 } else { 505 return FF_KNICTL_ACTION_DEFAULT; 506 } 507 } 508 509 static int 510 init_kni(void) 511 { 512 int nb_ports = rte_eth_dev_count_avail(); 513 kni_accept = 0; 514 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 515 kni_accept = 1; 516 517 knictl_action = get_kni_action(ff_global_cfg.kni.kni_action); 518 519 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 520 ff_global_cfg.kni.udp_port); 521 522 unsigned socket_id = lcore_conf.socket_id; 523 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 524 525 nb_ports = ff_global_cfg.dpdk.nb_ports; 526 int i, ret; 527 for (i = 0; i < nb_ports; i++) { 528 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 529 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 530 } 531 532 return 0; 533 } 534 #endif 535 536 //RSS reta update will failed when enable flow isolate 537 #ifndef FF_FLOW_ISOLATE 538 static void 539 set_rss_table(uint16_t port_id, uint16_t reta_size, uint16_t nb_queues) 540 { 541 if (reta_size == 0) { 542 return; 543 } 544 545 int reta_conf_size = RTE_MAX(1, reta_size / RTE_RETA_GROUP_SIZE); 546 struct rte_eth_rss_reta_entry64 reta_conf[reta_conf_size]; 547 548 /* config HW indirection table */ 549 unsigned i, j, hash=0; 550 for (i = 0; i < reta_conf_size; i++) { 551 reta_conf[i].mask = ~0ULL; 552 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 553 reta_conf[i].reta[j] = hash++ % nb_queues; 554 } 555 } 556 557 if (rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size)) { 558 rte_exit(EXIT_FAILURE, "port[%d], failed to update rss table\n", 559 port_id); 560 } 561 } 562 #endif 563 564 static int 565 init_port_start(void) 566 { 567 int nb_ports = ff_global_cfg.dpdk.nb_ports; 568 unsigned socketid = 0; 569 struct rte_mempool *mbuf_pool; 570 uint16_t i, j; 571 572 for (i = 0; i < nb_ports; i++) { 573 uint16_t port_id, u_port_id = ff_global_cfg.dpdk.portid_list[i]; 574 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[u_port_id]; 575 uint16_t nb_queues = pconf->nb_lcores; 576 577 for (j=0; j<=pconf->nb_slaves; j++) { 578 if (j < pconf->nb_slaves) { 579 port_id = pconf->slave_portid_list[j]; 580 printf("To init %s's %d'st slave port[%d]\n", 581 ff_global_cfg.dpdk.bond_cfgs->name, 582 j, port_id); 583 } else { 584 port_id = u_port_id; 585 } 586 587 struct rte_eth_dev_info dev_info; 588 struct rte_eth_conf port_conf = {0}; 589 struct rte_eth_rxconf rxq_conf; 590 struct rte_eth_txconf txq_conf; 591 592 int ret = rte_eth_dev_info_get(port_id, &dev_info); 593 if (ret != 0) 594 rte_exit(EXIT_FAILURE, 595 "Error during getting device (port %u) info: %s\n", 596 port_id, strerror(-ret)); 597 598 if (nb_queues > dev_info.max_rx_queues) { 599 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 600 nb_queues, 601 dev_info.max_rx_queues); 602 } 603 604 if (nb_queues > dev_info.max_tx_queues) { 605 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 606 nb_queues, 607 dev_info.max_tx_queues); 608 } 609 610 struct rte_ether_addr addr; 611 rte_eth_macaddr_get(port_id, &addr); 612 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 613 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 614 (unsigned)port_id, 615 addr.addr_bytes[0], addr.addr_bytes[1], 616 addr.addr_bytes[2], addr.addr_bytes[3], 617 addr.addr_bytes[4], addr.addr_bytes[5]); 618 619 rte_memcpy(pconf->mac, 620 addr.addr_bytes, RTE_ETHER_ADDR_LEN); 621 622 /* Set RSS mode */ 623 uint64_t default_rss_hf = ETH_RSS_PROTO_MASK; 624 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 625 port_conf.rx_adv_conf.rss_conf.rss_hf = default_rss_hf; 626 if (dev_info.hash_key_size == 52) { 627 rsskey = default_rsskey_52bytes; 628 rsskey_len = 52; 629 } 630 if (ff_global_cfg.dpdk.symmetric_rss) { 631 printf("Use symmetric Receive-side Scaling(RSS) key\n"); 632 rsskey = symmetric_rsskey; 633 } 634 port_conf.rx_adv_conf.rss_conf.rss_key = rsskey; 635 port_conf.rx_adv_conf.rss_conf.rss_key_len = rsskey_len; 636 port_conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads; 637 if (port_conf.rx_adv_conf.rss_conf.rss_hf != 638 ETH_RSS_PROTO_MASK) { 639 printf("Port %u modified RSS hash function based on hardware support," 640 "requested:%#"PRIx64" configured:%#"PRIx64"\n", 641 port_id, default_rss_hf, 642 port_conf.rx_adv_conf.rss_conf.rss_hf); 643 } 644 645 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) { 646 port_conf.txmode.offloads |= 647 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 648 } 649 650 /* Set Rx VLAN stripping */ 651 if (ff_global_cfg.dpdk.vlan_strip) { 652 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 653 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; 654 } 655 } 656 657 /* Enable HW CRC stripping */ 658 port_conf.rxmode.offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; 659 660 /* FIXME: Enable TCP LRO ?*/ 661 #if 0 662 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 663 printf("LRO is supported\n"); 664 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_TCP_LRO; 665 pconf->hw_features.rx_lro = 1; 666 } 667 #endif 668 669 /* Set Rx checksum checking */ 670 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 671 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 672 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 673 printf("RX checksum offload supported\n"); 674 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_CHECKSUM; 675 pconf->hw_features.rx_csum = 1; 676 } 677 678 if (ff_global_cfg.dpdk.tx_csum_offoad_skip == 0) { 679 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 680 printf("TX ip checksum offload supported\n"); 681 port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; 682 pconf->hw_features.tx_csum_ip = 1; 683 } 684 685 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 686 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 687 printf("TX TCP&UDP checksum offload supported\n"); 688 port_conf.txmode.offloads |= DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM; 689 pconf->hw_features.tx_csum_l4 = 1; 690 } 691 } else { 692 printf("TX checksum offoad is disabled\n"); 693 } 694 695 if (ff_global_cfg.dpdk.tso) { 696 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 697 printf("TSO is supported\n"); 698 port_conf.txmode.offloads |= DEV_TX_OFFLOAD_TCP_TSO; 699 pconf->hw_features.tx_tso = 1; 700 } 701 } else { 702 printf("TSO is disabled\n"); 703 } 704 705 if (dev_info.reta_size) { 706 /* reta size must be power of 2 */ 707 assert((dev_info.reta_size & (dev_info.reta_size - 1)) == 0); 708 709 rss_reta_size[port_id] = dev_info.reta_size; 710 printf("port[%d]: rss table size: %d\n", port_id, 711 dev_info.reta_size); 712 } 713 714 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 715 continue; 716 } 717 718 ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &port_conf); 719 if (ret != 0) { 720 return ret; 721 } 722 723 static uint16_t nb_rxd = RX_QUEUE_SIZE; 724 static uint16_t nb_txd = TX_QUEUE_SIZE; 725 ret = rte_eth_dev_adjust_nb_rx_tx_desc(port_id, &nb_rxd, &nb_txd); 726 if (ret < 0) 727 printf("Could not adjust number of descriptors " 728 "for port%u (%d)\n", (unsigned)port_id, ret); 729 730 uint16_t q; 731 for (q = 0; q < nb_queues; q++) { 732 if (numa_on) { 733 uint16_t lcore_id = lcore_conf.port_cfgs[port_id].lcore_list[q]; 734 socketid = rte_lcore_to_socket_id(lcore_id); 735 } 736 mbuf_pool = pktmbuf_pool[socketid]; 737 738 txq_conf = dev_info.default_txconf; 739 txq_conf.offloads = port_conf.txmode.offloads; 740 ret = rte_eth_tx_queue_setup(port_id, q, nb_txd, 741 socketid, &txq_conf); 742 if (ret < 0) { 743 return ret; 744 } 745 746 rxq_conf = dev_info.default_rxconf; 747 rxq_conf.offloads = port_conf.rxmode.offloads; 748 ret = rte_eth_rx_queue_setup(port_id, q, nb_rxd, 749 socketid, &rxq_conf, mbuf_pool); 750 if (ret < 0) { 751 return ret; 752 } 753 } 754 755 756 if (strncmp(dev_info.driver_name, BOND_DRIVER_NAME, 757 strlen(dev_info.driver_name)) == 0) { 758 759 rte_eth_macaddr_get(port_id, &addr); 760 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 761 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 762 (unsigned)port_id, 763 addr.addr_bytes[0], addr.addr_bytes[1], 764 addr.addr_bytes[2], addr.addr_bytes[3], 765 addr.addr_bytes[4], addr.addr_bytes[5]); 766 767 rte_memcpy(pconf->mac, 768 addr.addr_bytes, RTE_ETHER_ADDR_LEN); 769 770 int mode, count, x; 771 uint16_t slaves[RTE_MAX_ETHPORTS], len = RTE_MAX_ETHPORTS; 772 773 mode = rte_eth_bond_mode_get(port_id); 774 printf("Port %u, bond mode:%d\n", port_id, mode); 775 776 count = rte_eth_bond_slaves_get(port_id, slaves, len); 777 printf("Port %u, %s's slave ports count:%d\n", port_id, 778 ff_global_cfg.dpdk.bond_cfgs->name, count); 779 for (x=0; x<count; x++) { 780 printf("Port %u, %s's slave port[%u]\n", port_id, 781 ff_global_cfg.dpdk.bond_cfgs->name, slaves[x]); 782 } 783 } 784 785 ret = rte_eth_dev_start(port_id); 786 if (ret < 0) { 787 return ret; 788 } 789 //RSS reta update will failed when enable flow isolate 790 #ifndef FF_FLOW_ISOLATE 791 if (nb_queues > 1) { 792 /* set HW rss hash function to Toeplitz. */ 793 if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) { 794 struct rte_eth_hash_filter_info info = {0}; 795 info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG; 796 info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; 797 798 if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH, 799 RTE_ETH_FILTER_SET, &info) < 0) { 800 rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n", 801 port_id); 802 } 803 } 804 805 set_rss_table(port_id, dev_info.reta_size, nb_queues); 806 } 807 #endif 808 809 /* Enable RX in promiscuous mode for the Ethernet device. */ 810 if (ff_global_cfg.dpdk.promiscuous) { 811 ret = rte_eth_promiscuous_enable(port_id); 812 if (ret == 0) { 813 printf("set port %u to promiscuous mode ok\n", port_id); 814 } else { 815 printf("set port %u to promiscuous mode error\n", port_id); 816 } 817 } 818 } 819 } 820 821 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 822 check_all_ports_link_status(); 823 } 824 825 return 0; 826 } 827 828 static int 829 init_clock(void) 830 { 831 rte_timer_subsystem_init(); 832 uint64_t hz = rte_get_timer_hz(); 833 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 834 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 835 836 rte_timer_init(&freebsd_clock); 837 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 838 rte_lcore_id(), &ff_hardclock_job, NULL); 839 840 ff_update_current_ts(); 841 842 return 0; 843 } 844 845 #ifdef FF_FLOW_ISOLATE 846 /** Print a message out of a flow error. */ 847 static int 848 port_flow_complain(struct rte_flow_error *error) 849 { 850 static const char *const errstrlist[] = { 851 [RTE_FLOW_ERROR_TYPE_NONE] = "no error", 852 [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified", 853 [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)", 854 [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field", 855 [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field", 856 [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field", 857 [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field", 858 [RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER] = "transfer field", 859 [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure", 860 [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length", 861 [RTE_FLOW_ERROR_TYPE_ITEM_SPEC] = "item specification", 862 [RTE_FLOW_ERROR_TYPE_ITEM_LAST] = "item specification range", 863 [RTE_FLOW_ERROR_TYPE_ITEM_MASK] = "item specification mask", 864 [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item", 865 [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions", 866 [RTE_FLOW_ERROR_TYPE_ACTION_CONF] = "action configuration", 867 [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action", 868 }; 869 const char *errstr; 870 char buf[32]; 871 int err = rte_errno; 872 873 if ((unsigned int)error->type >= RTE_DIM(errstrlist) || 874 !errstrlist[error->type]) 875 errstr = "unknown type"; 876 else 877 errstr = errstrlist[error->type]; 878 printf("Caught error type %d (%s): %s%s: %s\n", 879 error->type, errstr, 880 error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ", 881 error->cause), buf) : "", 882 error->message ? error->message : "(no stated reason)", 883 rte_strerror(err)); 884 return -err; 885 } 886 887 static int 888 port_flow_isolate(uint16_t port_id, int set) 889 { 890 struct rte_flow_error error; 891 892 /* Poisoning to make sure PMDs update it in case of error. */ 893 memset(&error, 0x66, sizeof(error)); 894 if (rte_flow_isolate(port_id, set, &error)) 895 return port_flow_complain(&error); 896 printf("Ingress traffic on port %u is %s to the defined flow rules\n", 897 port_id, 898 set ? "now restricted" : "not restricted anymore"); 899 return 0; 900 } 901 #endif 902 903 int 904 ff_dpdk_init(int argc, char **argv) 905 { 906 if (ff_global_cfg.dpdk.nb_procs < 1 || 907 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 908 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 909 ff_global_cfg.dpdk.proc_id < 0) { 910 printf("param num_procs[%d] or proc_id[%d] error!\n", 911 ff_global_cfg.dpdk.nb_procs, 912 ff_global_cfg.dpdk.proc_id); 913 exit(1); 914 } 915 916 int ret = rte_eal_init(argc, argv); 917 if (ret < 0) { 918 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 919 } 920 921 numa_on = ff_global_cfg.dpdk.numa_on; 922 923 idle_sleep = ff_global_cfg.dpdk.idle_sleep; 924 pkt_tx_delay = ff_global_cfg.dpdk.pkt_tx_delay > BURST_TX_DRAIN_US ? \ 925 BURST_TX_DRAIN_US : ff_global_cfg.dpdk.pkt_tx_delay; 926 927 init_lcore_conf(); 928 929 init_mem_pool(); 930 931 init_dispatch_ring(); 932 933 init_msg_ring(); 934 935 #ifdef FF_KNI 936 enable_kni = ff_global_cfg.kni.enable; 937 if (enable_kni) { 938 init_kni(); 939 } 940 #endif 941 942 #ifdef FF_USE_PAGE_ARRAY 943 ff_mmap_init(); 944 #endif 945 946 #ifdef FF_FLOW_ISOLATE 947 // run once in primary process 948 if (0 == lcore_conf.tx_queue_id[0]){ 949 ret = port_flow_isolate(0, 1); 950 if (ret < 0) 951 rte_exit(EXIT_FAILURE, "init_port_isolate failed\n"); 952 } 953 #endif 954 955 ret = init_port_start(); 956 if (ret < 0) { 957 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 958 } 959 960 init_clock(); 961 962 return 0; 963 } 964 965 static void 966 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 967 { 968 uint8_t rx_csum = ctx->hw_features.rx_csum; 969 if (rx_csum) { 970 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 971 rte_pktmbuf_free(pkt); 972 return; 973 } 974 } 975 976 void *data = rte_pktmbuf_mtod(pkt, void*); 977 uint16_t len = rte_pktmbuf_data_len(pkt); 978 979 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 980 if (hdr == NULL) { 981 rte_pktmbuf_free(pkt); 982 return; 983 } 984 985 if (pkt->ol_flags & PKT_RX_VLAN_STRIPPED) { 986 ff_mbuf_set_vlan_info(hdr, pkt->vlan_tci); 987 } 988 989 struct rte_mbuf *pn = pkt->next; 990 void *prev = hdr; 991 while(pn != NULL) { 992 data = rte_pktmbuf_mtod(pn, void*); 993 len = rte_pktmbuf_data_len(pn); 994 995 void *mb = ff_mbuf_get(prev, data, len); 996 if (mb == NULL) { 997 ff_mbuf_free(hdr); 998 rte_pktmbuf_free(pkt); 999 return; 1000 } 1001 pn = pn->next; 1002 prev = mb; 1003 } 1004 1005 ff_veth_process_packet(ctx->ifp, hdr); 1006 } 1007 1008 static enum FilterReturn 1009 protocol_filter(const void *data, uint16_t len) 1010 { 1011 if(len < RTE_ETHER_ADDR_LEN) 1012 return FILTER_UNKNOWN; 1013 1014 const struct rte_ether_hdr *hdr; 1015 const struct rte_vlan_hdr *vlanhdr; 1016 hdr = (const struct rte_ether_hdr *)data; 1017 uint16_t ether_type = rte_be_to_cpu_16(hdr->ether_type); 1018 data += RTE_ETHER_HDR_LEN; 1019 len -= RTE_ETHER_HDR_LEN; 1020 1021 if (ether_type == RTE_ETHER_TYPE_VLAN) { 1022 vlanhdr = (struct rte_vlan_hdr *)data; 1023 ether_type = rte_be_to_cpu_16(vlanhdr->eth_proto); 1024 data += sizeof(struct rte_vlan_hdr); 1025 len -= sizeof(struct rte_vlan_hdr); 1026 } 1027 1028 if(ether_type == RTE_ETHER_TYPE_ARP) 1029 return FILTER_ARP; 1030 1031 #ifdef INET6 1032 if (ether_type == RTE_ETHER_TYPE_IPV6) { 1033 return ff_kni_proto_filter(data, 1034 len, ether_type); 1035 } 1036 #endif 1037 1038 #ifndef FF_KNI 1039 return FILTER_UNKNOWN; 1040 #else 1041 if (!enable_kni) { 1042 return FILTER_UNKNOWN; 1043 } 1044 1045 if(ether_type != RTE_ETHER_TYPE_IPV4) 1046 return FILTER_UNKNOWN; 1047 1048 return ff_kni_proto_filter(data, 1049 len, ether_type); 1050 #endif 1051 } 1052 1053 static inline void 1054 pktmbuf_deep_attach(struct rte_mbuf *mi, const struct rte_mbuf *m) 1055 { 1056 struct rte_mbuf *md; 1057 void *src, *dst; 1058 1059 dst = rte_pktmbuf_mtod(mi, void *); 1060 src = rte_pktmbuf_mtod(m, void *); 1061 1062 mi->data_len = m->data_len; 1063 rte_memcpy(dst, src, m->data_len); 1064 1065 mi->port = m->port; 1066 mi->vlan_tci = m->vlan_tci; 1067 mi->vlan_tci_outer = m->vlan_tci_outer; 1068 mi->tx_offload = m->tx_offload; 1069 mi->hash = m->hash; 1070 mi->ol_flags = m->ol_flags; 1071 mi->packet_type = m->packet_type; 1072 } 1073 1074 /* copied from rte_pktmbuf_clone */ 1075 static inline struct rte_mbuf * 1076 pktmbuf_deep_clone(const struct rte_mbuf *md, 1077 struct rte_mempool *mp) 1078 { 1079 struct rte_mbuf *mc, *mi, **prev; 1080 uint32_t pktlen; 1081 uint8_t nseg; 1082 1083 if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL)) 1084 return NULL; 1085 1086 mi = mc; 1087 prev = &mi->next; 1088 pktlen = md->pkt_len; 1089 nseg = 0; 1090 1091 do { 1092 nseg++; 1093 pktmbuf_deep_attach(mi, md); 1094 *prev = mi; 1095 prev = &mi->next; 1096 } while ((md = md->next) != NULL && 1097 (mi = rte_pktmbuf_alloc(mp)) != NULL); 1098 1099 *prev = NULL; 1100 mc->nb_segs = nseg; 1101 mc->pkt_len = pktlen; 1102 1103 /* Allocation of new indirect segment failed */ 1104 if (unlikely (mi == NULL)) { 1105 rte_pktmbuf_free(mc); 1106 return NULL; 1107 } 1108 1109 __rte_mbuf_sanity_check(mc, 1); 1110 return mc; 1111 } 1112 1113 static inline void 1114 process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 1115 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 1116 { 1117 struct lcore_conf *qconf = &lcore_conf; 1118 uint16_t nb_queues = qconf->nb_queue_list[port_id]; 1119 1120 uint16_t i; 1121 for (i = 0; i < count; i++) { 1122 struct rte_mbuf *rtem = bufs[i]; 1123 1124 if (unlikely( ff_global_cfg.pcap.enable)) { 1125 if (!pkts_from_ring) { 1126 ff_dump_packets( ff_global_cfg.pcap.save_path, rtem, ff_global_cfg.pcap.snap_len, ff_global_cfg.pcap.save_len); 1127 } 1128 } 1129 1130 void *data = rte_pktmbuf_mtod(rtem, void*); 1131 uint16_t len = rte_pktmbuf_data_len(rtem); 1132 1133 if (!pkts_from_ring) { 1134 ff_traffic.rx_packets++; 1135 ff_traffic.rx_bytes += len; 1136 } 1137 1138 if (!pkts_from_ring && packet_dispatcher) { 1139 int ret = (*packet_dispatcher)(data, &len, queue_id, nb_queues); 1140 if (ret == FF_DISPATCH_RESPONSE) { 1141 rte_pktmbuf_pkt_len(rtem) = rte_pktmbuf_data_len(rtem) = len; 1142 1143 /* 1144 * We have not support vlan out strip 1145 */ 1146 if (rtem->vlan_tci) { 1147 data = rte_pktmbuf_prepend(rtem, sizeof(struct rte_vlan_hdr)); 1148 if (data != NULL) { 1149 memmove(data, data + sizeof(struct rte_vlan_hdr), RTE_ETHER_HDR_LEN); 1150 struct rte_ether_hdr *etherhdr = (struct rte_ether_hdr *)data; 1151 struct rte_vlan_hdr *vlanhdr = (struct rte_vlan_hdr *)(data + RTE_ETHER_HDR_LEN); 1152 vlanhdr->vlan_tci = rte_cpu_to_be_16(rtem->vlan_tci); 1153 vlanhdr->eth_proto = etherhdr->ether_type; 1154 etherhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN); 1155 } 1156 } 1157 send_single_packet(rtem, port_id); 1158 continue; 1159 } 1160 1161 if (ret == FF_DISPATCH_ERROR || ret >= nb_queues) { 1162 rte_pktmbuf_free(rtem); 1163 continue; 1164 } 1165 1166 if (ret != queue_id) { 1167 ret = rte_ring_enqueue(dispatch_ring[port_id][ret], rtem); 1168 if (ret < 0) 1169 rte_pktmbuf_free(rtem); 1170 1171 continue; 1172 } 1173 } 1174 1175 enum FilterReturn filter = protocol_filter(data, len); 1176 #ifdef INET6 1177 if (filter == FILTER_ARP || filter == FILTER_NDP) { 1178 #else 1179 if (filter == FILTER_ARP) { 1180 #endif 1181 struct rte_mempool *mbuf_pool; 1182 struct rte_mbuf *mbuf_clone; 1183 if (!pkts_from_ring) { 1184 uint16_t j; 1185 for(j = 0; j < nb_queues; ++j) { 1186 if(j == queue_id) 1187 continue; 1188 1189 unsigned socket_id = 0; 1190 if (numa_on) { 1191 uint16_t lcore_id = qconf->port_cfgs[port_id].lcore_list[j]; 1192 socket_id = rte_lcore_to_socket_id(lcore_id); 1193 } 1194 mbuf_pool = pktmbuf_pool[socket_id]; 1195 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1196 if(mbuf_clone) { 1197 int ret = rte_ring_enqueue(dispatch_ring[port_id][j], 1198 mbuf_clone); 1199 if (ret < 0) 1200 rte_pktmbuf_free(mbuf_clone); 1201 } 1202 } 1203 } 1204 1205 #ifdef FF_KNI 1206 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1207 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 1208 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1209 if(mbuf_clone) { 1210 ff_kni_enqueue(port_id, mbuf_clone); 1211 } 1212 } 1213 #endif 1214 ff_veth_input(ctx, rtem); 1215 #ifdef FF_KNI 1216 } else if (enable_kni) { 1217 if (knictl_action == FF_KNICTL_ACTION_ALL_TO_KNI){ 1218 ff_kni_enqueue(port_id, rtem); 1219 } else if (knictl_action == FF_KNICTL_ACTION_ALL_TO_FF){ 1220 ff_veth_input(ctx, rtem); 1221 } else if (knictl_action == FF_KNICTL_ACTION_DEFAULT){ 1222 if (enable_kni && 1223 ((filter == FILTER_KNI && kni_accept) || 1224 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 1225 ff_kni_enqueue(port_id, rtem); 1226 } else { 1227 ff_veth_input(ctx, rtem); 1228 } 1229 } else { 1230 ff_veth_input(ctx, rtem); 1231 } 1232 #endif 1233 } else { 1234 ff_veth_input(ctx, rtem); 1235 } 1236 } 1237 } 1238 1239 static inline int 1240 process_dispatch_ring(uint16_t port_id, uint16_t queue_id, 1241 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 1242 { 1243 /* read packet from ring buf and to process */ 1244 uint16_t nb_rb; 1245 nb_rb = rte_ring_dequeue_burst(dispatch_ring[port_id][queue_id], 1246 (void **)pkts_burst, MAX_PKT_BURST, NULL); 1247 1248 if(nb_rb > 0) { 1249 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 1250 } 1251 1252 return 0; 1253 } 1254 1255 static inline void 1256 handle_sysctl_msg(struct ff_msg *msg) 1257 { 1258 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 1259 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 1260 msg->sysctl.newlen); 1261 1262 if (ret < 0) { 1263 msg->result = errno; 1264 } else { 1265 msg->result = 0; 1266 } 1267 } 1268 1269 static inline void 1270 handle_ioctl_msg(struct ff_msg *msg) 1271 { 1272 int fd, ret; 1273 #ifdef INET6 1274 if (msg->msg_type == FF_IOCTL6) { 1275 fd = ff_socket(AF_INET6, SOCK_DGRAM, 0); 1276 } else 1277 #endif 1278 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 1279 1280 if (fd < 0) { 1281 ret = -1; 1282 goto done; 1283 } 1284 1285 ret = ff_ioctl_freebsd(fd, msg->ioctl.cmd, msg->ioctl.data); 1286 1287 ff_close(fd); 1288 1289 done: 1290 if (ret < 0) { 1291 msg->result = errno; 1292 } else { 1293 msg->result = 0; 1294 } 1295 } 1296 1297 static inline void 1298 handle_route_msg(struct ff_msg *msg) 1299 { 1300 int ret = ff_rtioctl(msg->route.fib, msg->route.data, 1301 &msg->route.len, msg->route.maxlen); 1302 if (ret < 0) { 1303 msg->result = errno; 1304 } else { 1305 msg->result = 0; 1306 } 1307 } 1308 1309 static inline void 1310 handle_top_msg(struct ff_msg *msg) 1311 { 1312 msg->top = ff_top_status; 1313 msg->result = 0; 1314 } 1315 1316 #ifdef FF_NETGRAPH 1317 static inline void 1318 handle_ngctl_msg(struct ff_msg *msg) 1319 { 1320 int ret = ff_ngctl(msg->ngctl.cmd, msg->ngctl.data); 1321 if (ret < 0) { 1322 msg->result = errno; 1323 } else { 1324 msg->result = 0; 1325 msg->ngctl.ret = ret; 1326 } 1327 } 1328 #endif 1329 1330 #ifdef FF_IPFW 1331 static inline void 1332 handle_ipfw_msg(struct ff_msg *msg) 1333 { 1334 int fd, ret; 1335 fd = ff_socket(AF_INET, SOCK_RAW, IPPROTO_RAW); 1336 if (fd < 0) { 1337 ret = -1; 1338 goto done; 1339 } 1340 1341 switch (msg->ipfw.cmd) { 1342 case FF_IPFW_GET: 1343 ret = ff_getsockopt_freebsd(fd, msg->ipfw.level, 1344 msg->ipfw.optname, msg->ipfw.optval, 1345 msg->ipfw.optlen); 1346 break; 1347 case FF_IPFW_SET: 1348 ret = ff_setsockopt_freebsd(fd, msg->ipfw.level, 1349 msg->ipfw.optname, msg->ipfw.optval, 1350 *(msg->ipfw.optlen)); 1351 break; 1352 default: 1353 ret = -1; 1354 errno = ENOTSUP; 1355 break; 1356 } 1357 1358 ff_close(fd); 1359 1360 done: 1361 if (ret < 0) { 1362 msg->result = errno; 1363 } else { 1364 msg->result = 0; 1365 } 1366 } 1367 #endif 1368 1369 static inline void 1370 handle_traffic_msg(struct ff_msg *msg) 1371 { 1372 msg->traffic = ff_traffic; 1373 msg->result = 0; 1374 } 1375 1376 #ifdef FF_KNI 1377 static inline void 1378 handle_knictl_msg(struct ff_msg *msg) 1379 { 1380 if (msg->knictl.kni_cmd == FF_KNICTL_CMD_SET){ 1381 switch (msg->knictl.kni_action){ 1382 case FF_KNICTL_ACTION_ALL_TO_FF: knictl_action = FF_KNICTL_ACTION_ALL_TO_FF; msg->result = 0; printf("new kni action: alltoff\n"); break; 1383 case FF_KNICTL_ACTION_ALL_TO_KNI: knictl_action = FF_KNICTL_ACTION_ALL_TO_KNI; msg->result = 0; printf("new kni action: alltokni\n"); break; 1384 case FF_KNICTL_ACTION_DEFAULT: knictl_action = FF_KNICTL_ACTION_DEFAULT; msg->result = 0; printf("new kni action: default\n"); break; 1385 default: msg->result = -1; 1386 } 1387 } 1388 else if (msg->knictl.kni_cmd == FF_KNICTL_CMD_GET){ 1389 msg->knictl.kni_action = knictl_action; 1390 } else { 1391 msg->result = -2; 1392 } 1393 } 1394 #endif 1395 1396 static inline void 1397 handle_default_msg(struct ff_msg *msg) 1398 { 1399 msg->result = ENOTSUP; 1400 } 1401 1402 static inline void 1403 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1404 { 1405 switch (msg->msg_type) { 1406 case FF_SYSCTL: 1407 handle_sysctl_msg(msg); 1408 break; 1409 case FF_IOCTL: 1410 #ifdef INET6 1411 case FF_IOCTL6: 1412 #endif 1413 handle_ioctl_msg(msg); 1414 break; 1415 case FF_ROUTE: 1416 handle_route_msg(msg); 1417 break; 1418 case FF_TOP: 1419 handle_top_msg(msg); 1420 break; 1421 #ifdef FF_NETGRAPH 1422 case FF_NGCTL: 1423 handle_ngctl_msg(msg); 1424 break; 1425 #endif 1426 #ifdef FF_IPFW 1427 case FF_IPFW_CTL: 1428 handle_ipfw_msg(msg); 1429 break; 1430 #endif 1431 case FF_TRAFFIC: 1432 handle_traffic_msg(msg); 1433 break; 1434 #ifdef FF_KNI 1435 case FF_KNICTL: 1436 handle_knictl_msg(msg); 1437 break; 1438 #endif 1439 default: 1440 handle_default_msg(msg); 1441 break; 1442 } 1443 rte_ring_enqueue(msg_ring[proc_id].ring[msg->msg_type], msg); 1444 } 1445 1446 static inline int 1447 process_msg_ring(uint16_t proc_id) 1448 { 1449 void *msg; 1450 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1451 1452 if (unlikely(ret == 0)) { 1453 handle_msg((struct ff_msg *)msg, proc_id); 1454 } 1455 1456 return 0; 1457 } 1458 1459 /* Send burst of packets on an output interface */ 1460 static inline int 1461 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1462 { 1463 struct rte_mbuf **m_table; 1464 int ret; 1465 uint16_t queueid; 1466 1467 queueid = qconf->tx_queue_id[port]; 1468 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1469 1470 if (unlikely(ff_global_cfg.pcap.enable)) { 1471 uint16_t i; 1472 for (i = 0; i < n; i++) { 1473 ff_dump_packets( ff_global_cfg.pcap.save_path, m_table[i], 1474 ff_global_cfg.pcap.snap_len, ff_global_cfg.pcap.save_len); 1475 } 1476 } 1477 1478 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1479 ff_traffic.tx_packets += ret; 1480 uint16_t i; 1481 for (i = 0; i < ret; i++) { 1482 ff_traffic.tx_bytes += rte_pktmbuf_pkt_len(m_table[i]); 1483 #ifdef FF_USE_PAGE_ARRAY 1484 if (qconf->tx_mbufs[port].bsd_m_table[i]) 1485 ff_enq_tx_bsdmbuf(port, qconf->tx_mbufs[port].bsd_m_table[i], m_table[i]->nb_segs); 1486 #endif 1487 } 1488 if (unlikely(ret < n)) { 1489 do { 1490 rte_pktmbuf_free(m_table[ret]); 1491 #ifdef FF_USE_PAGE_ARRAY 1492 if ( qconf->tx_mbufs[port].bsd_m_table[ret] ) 1493 ff_mbuf_free(qconf->tx_mbufs[port].bsd_m_table[ret]); 1494 #endif 1495 } while (++ret < n); 1496 } 1497 return 0; 1498 } 1499 1500 /* Enqueue a single packet, and send burst if queue is filled */ 1501 static inline int 1502 send_single_packet(struct rte_mbuf *m, uint8_t port) 1503 { 1504 uint16_t len; 1505 struct lcore_conf *qconf; 1506 1507 qconf = &lcore_conf; 1508 len = qconf->tx_mbufs[port].len; 1509 qconf->tx_mbufs[port].m_table[len] = m; 1510 len++; 1511 1512 /* enough pkts to be sent */ 1513 if (unlikely(len == MAX_PKT_BURST)) { 1514 send_burst(qconf, MAX_PKT_BURST, port); 1515 len = 0; 1516 } 1517 1518 qconf->tx_mbufs[port].len = len; 1519 return 0; 1520 } 1521 1522 int 1523 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1524 int total) 1525 { 1526 #ifdef FF_USE_PAGE_ARRAY 1527 struct lcore_conf *qconf = &lcore_conf; 1528 int len = 0; 1529 1530 len = ff_if_send_onepkt(ctx, m,total); 1531 if (unlikely(len == MAX_PKT_BURST)) { 1532 send_burst(qconf, MAX_PKT_BURST, ctx->port_id); 1533 len = 0; 1534 } 1535 qconf->tx_mbufs[ctx->port_id].len = len; 1536 return 0; 1537 #endif 1538 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1539 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1540 if (head == NULL) { 1541 ff_mbuf_free(m); 1542 return -1; 1543 } 1544 1545 head->pkt_len = total; 1546 head->nb_segs = 0; 1547 1548 int off = 0; 1549 struct rte_mbuf *cur = head, *prev = NULL; 1550 while(total > 0) { 1551 if (cur == NULL) { 1552 cur = rte_pktmbuf_alloc(mbuf_pool); 1553 if (cur == NULL) { 1554 rte_pktmbuf_free(head); 1555 ff_mbuf_free(m); 1556 return -1; 1557 } 1558 } 1559 1560 if (prev != NULL) { 1561 prev->next = cur; 1562 } 1563 head->nb_segs++; 1564 1565 prev = cur; 1566 void *data = rte_pktmbuf_mtod(cur, void*); 1567 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1568 int ret = ff_mbuf_copydata(m, data, off, len); 1569 if (ret < 0) { 1570 rte_pktmbuf_free(head); 1571 ff_mbuf_free(m); 1572 return -1; 1573 } 1574 1575 1576 cur->data_len = len; 1577 off += len; 1578 total -= len; 1579 cur = NULL; 1580 } 1581 1582 struct ff_tx_offload offload = {0}; 1583 ff_mbuf_tx_offload(m, &offload); 1584 1585 void *data = rte_pktmbuf_mtod(head, void*); 1586 1587 if (offload.ip_csum) { 1588 /* ipv6 not supported yet */ 1589 struct rte_ipv4_hdr *iph; 1590 int iph_len; 1591 iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN); 1592 iph_len = (iph->version_ihl & 0x0f) << 2; 1593 1594 head->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 1595 head->l2_len = RTE_ETHER_HDR_LEN; 1596 head->l3_len = iph_len; 1597 } 1598 1599 if (ctx->hw_features.tx_csum_l4) { 1600 struct rte_ipv4_hdr *iph; 1601 int iph_len; 1602 iph = (struct rte_ipv4_hdr *)(data + RTE_ETHER_HDR_LEN); 1603 iph_len = (iph->version_ihl & 0x0f) << 2; 1604 1605 if (offload.tcp_csum) { 1606 head->ol_flags |= PKT_TX_TCP_CKSUM; 1607 head->l2_len = RTE_ETHER_HDR_LEN; 1608 head->l3_len = iph_len; 1609 } 1610 1611 /* 1612 * TCP segmentation offload. 1613 * 1614 * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag 1615 * implies PKT_TX_TCP_CKSUM) 1616 * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 1617 * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and 1618 * write the IP checksum to 0 in the packet 1619 * - fill the mbuf offload information: l2_len, 1620 * l3_len, l4_len, tso_segsz 1621 * - calculate the pseudo header checksum without taking ip_len 1622 * in account, and set it in the TCP header. Refer to 1623 * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be 1624 * used as helpers. 1625 */ 1626 if (offload.tso_seg_size) { 1627 struct rte_tcp_hdr *tcph; 1628 int tcph_len; 1629 tcph = (struct rte_tcp_hdr *)((char *)iph + iph_len); 1630 tcph_len = (tcph->data_off & 0xf0) >> 2; 1631 tcph->cksum = rte_ipv4_phdr_cksum(iph, PKT_TX_TCP_SEG); 1632 1633 head->ol_flags |= PKT_TX_TCP_SEG; 1634 head->l4_len = tcph_len; 1635 head->tso_segsz = offload.tso_seg_size; 1636 } 1637 1638 if (offload.udp_csum) { 1639 head->ol_flags |= PKT_TX_UDP_CKSUM; 1640 head->l2_len = RTE_ETHER_HDR_LEN; 1641 head->l3_len = iph_len; 1642 } 1643 } 1644 1645 ff_mbuf_free(m); 1646 1647 return send_single_packet(head, ctx->port_id); 1648 } 1649 1650 static int 1651 main_loop(void *arg) 1652 { 1653 struct loop_routine *lr = (struct loop_routine *)arg; 1654 1655 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1656 uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc, idle_sleep_tsc; 1657 int i, j, nb_rx, idle; 1658 uint16_t port_id, queue_id; 1659 struct lcore_conf *qconf; 1660 uint64_t drain_tsc = 0; 1661 struct ff_dpdk_if_context *ctx; 1662 1663 if (pkt_tx_delay) { 1664 drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * pkt_tx_delay; 1665 } 1666 1667 prev_tsc = 0; 1668 usch_tsc = 0; 1669 1670 qconf = &lcore_conf; 1671 1672 while (1) { 1673 cur_tsc = rte_rdtsc(); 1674 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1675 rte_timer_manage(); 1676 } 1677 1678 idle = 1; 1679 sys_tsc = 0; 1680 usr_tsc = 0; 1681 1682 /* 1683 * TX burst queue drain 1684 */ 1685 diff_tsc = cur_tsc - prev_tsc; 1686 if (unlikely(diff_tsc >= drain_tsc)) { 1687 for (i = 0; i < qconf->nb_tx_port; i++) { 1688 port_id = qconf->tx_port_id[i]; 1689 if (qconf->tx_mbufs[port_id].len == 0) 1690 continue; 1691 1692 idle = 0; 1693 1694 send_burst(qconf, 1695 qconf->tx_mbufs[port_id].len, 1696 port_id); 1697 qconf->tx_mbufs[port_id].len = 0; 1698 } 1699 1700 prev_tsc = cur_tsc; 1701 } 1702 1703 /* 1704 * Read packet from RX queues 1705 */ 1706 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1707 port_id = qconf->rx_queue_list[i].port_id; 1708 queue_id = qconf->rx_queue_list[i].queue_id; 1709 ctx = veth_ctx[port_id]; 1710 1711 #ifdef FF_KNI 1712 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1713 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1714 } 1715 #endif 1716 1717 process_dispatch_ring(port_id, queue_id, pkts_burst, ctx); 1718 1719 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1720 MAX_PKT_BURST); 1721 if (nb_rx == 0) 1722 continue; 1723 1724 idle = 0; 1725 1726 /* Prefetch first packets */ 1727 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1728 rte_prefetch0(rte_pktmbuf_mtod( 1729 pkts_burst[j], void *)); 1730 } 1731 1732 /* Prefetch and handle already prefetched packets */ 1733 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1734 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1735 j + PREFETCH_OFFSET], void *)); 1736 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1737 } 1738 1739 /* Handle remaining prefetched packets */ 1740 for (; j < nb_rx; j++) { 1741 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1742 } 1743 } 1744 1745 process_msg_ring(qconf->proc_id); 1746 1747 div_tsc = rte_rdtsc(); 1748 1749 if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc >= drain_tsc))) { 1750 usch_tsc = cur_tsc; 1751 lr->loop(lr->arg); 1752 } 1753 1754 idle_sleep_tsc = rte_rdtsc(); 1755 if (likely(idle && idle_sleep)) { 1756 usleep(idle_sleep); 1757 end_tsc = rte_rdtsc(); 1758 } else { 1759 end_tsc = idle_sleep_tsc; 1760 } 1761 1762 if (usch_tsc == cur_tsc) { 1763 usr_tsc = idle_sleep_tsc - div_tsc; 1764 } 1765 1766 if (!idle) { 1767 sys_tsc = div_tsc - cur_tsc; 1768 ff_top_status.sys_tsc += sys_tsc; 1769 } 1770 1771 ff_top_status.usr_tsc += usr_tsc; 1772 ff_top_status.work_tsc += end_tsc - cur_tsc; 1773 ff_top_status.idle_tsc += end_tsc - cur_tsc - usr_tsc - sys_tsc; 1774 1775 ff_top_status.loops++; 1776 } 1777 1778 return 0; 1779 } 1780 1781 int 1782 ff_dpdk_if_up(void) { 1783 int i; 1784 struct lcore_conf *qconf = &lcore_conf; 1785 for (i = 0; i < qconf->nb_tx_port; i++) { 1786 uint16_t port_id = qconf->tx_port_id[i]; 1787 1788 struct ff_port_cfg *pconf = &qconf->port_cfgs[port_id]; 1789 veth_ctx[port_id] = ff_veth_attach(pconf); 1790 if (veth_ctx[port_id] == NULL) { 1791 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1792 } 1793 } 1794 1795 return 0; 1796 } 1797 1798 void 1799 ff_dpdk_run(loop_func_t loop, void *arg) { 1800 struct loop_routine *lr = rte_malloc(NULL, 1801 sizeof(struct loop_routine), 0); 1802 lr->loop = loop; 1803 lr->arg = arg; 1804 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1805 rte_eal_mp_wait_lcore(); 1806 rte_free(lr); 1807 } 1808 1809 void 1810 ff_dpdk_pktmbuf_free(void *m) 1811 { 1812 rte_pktmbuf_free((struct rte_mbuf *)m); 1813 } 1814 1815 static uint32_t 1816 toeplitz_hash(unsigned keylen, const uint8_t *key, 1817 unsigned datalen, const uint8_t *data) 1818 { 1819 uint32_t hash = 0, v; 1820 u_int i, b; 1821 1822 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1823 1824 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1825 for (i = 0; i < datalen; i++) { 1826 for (b = 0; b < 8; b++) { 1827 if (data[i] & (1<<(7-b))) 1828 hash ^= v; 1829 v <<= 1; 1830 if ((i + 4) < keylen && 1831 (key[i+4] & (1<<(7-b)))) 1832 v |= 1; 1833 } 1834 } 1835 return (hash); 1836 } 1837 1838 int 1839 ff_rss_check(void *softc, uint32_t saddr, uint32_t daddr, 1840 uint16_t sport, uint16_t dport) 1841 { 1842 struct lcore_conf *qconf = &lcore_conf; 1843 struct ff_dpdk_if_context *ctx = ff_veth_softc_to_hostc(softc); 1844 uint16_t nb_queues = qconf->nb_queue_list[ctx->port_id]; 1845 1846 if (nb_queues <= 1) { 1847 return 1; 1848 } 1849 1850 uint16_t reta_size = rss_reta_size[ctx->port_id]; 1851 uint16_t queueid = qconf->tx_queue_id[ctx->port_id]; 1852 1853 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1854 sizeof(dport)]; 1855 1856 unsigned datalen = 0; 1857 1858 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1859 datalen += sizeof(saddr); 1860 1861 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1862 datalen += sizeof(daddr); 1863 1864 bcopy(&sport, &data[datalen], sizeof(sport)); 1865 datalen += sizeof(sport); 1866 1867 bcopy(&dport, &data[datalen], sizeof(dport)); 1868 datalen += sizeof(dport); 1869 1870 uint32_t hash = 0; 1871 hash = toeplitz_hash(rsskey_len, rsskey, datalen, data); 1872 1873 return ((hash & (reta_size - 1)) % nb_queues) == queueid; 1874 } 1875 1876 void 1877 ff_regist_packet_dispatcher(dispatch_func_t func) 1878 { 1879 packet_dispatcher = func; 1880 } 1881 1882 uint64_t 1883 ff_get_tsc_ns() 1884 { 1885 uint64_t cur_tsc = rte_rdtsc(); 1886 uint64_t hz = rte_get_tsc_hz(); 1887 return ((double)cur_tsc/(double)hz) * NS_PER_S; 1888 } 1889 1890