1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <rte_common.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_memzone.h> 33 #include <rte_config.h> 34 #include <rte_eal.h> 35 #include <rte_pci.h> 36 #include <rte_mbuf.h> 37 #include <rte_memory.h> 38 #include <rte_lcore.h> 39 #include <rte_launch.h> 40 #include <rte_ethdev.h> 41 #include <rte_debug.h> 42 #include <rte_common.h> 43 #include <rte_ether.h> 44 #include <rte_malloc.h> 45 #include <rte_cycles.h> 46 #include <rte_timer.h> 47 #include <rte_thash.h> 48 49 #include "ff_dpdk_if.h" 50 #include "ff_dpdk_pcap.h" 51 #include "ff_dpdk_kni.h" 52 #include "ff_config.h" 53 #include "ff_veth.h" 54 #include "ff_host_interface.h" 55 56 #define MEMPOOL_CACHE_SIZE 256 57 58 #define ARP_RING_SIZE 2048 59 60 /* 61 * Configurable number of RX/TX ring descriptors 62 */ 63 #define RX_QUEUE_SIZE 512 64 #define TX_QUEUE_SIZE 256 65 66 #define MAX_PKT_BURST 32 67 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 68 69 /* 70 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 71 */ 72 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 73 74 #define NB_SOCKETS 8 75 76 /* Configure how many packets ahead to prefetch, when reading packets */ 77 #define PREFETCH_OFFSET 3 78 79 #define MAX_RX_QUEUE_PER_LCORE 16 80 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 81 #define MAX_RX_QUEUE_PER_PORT 128 82 83 #define BITS_PER_HEX 4 84 85 static int enable_kni; 86 static int kni_accept; 87 88 static struct rte_timer freebsd_clock; 89 90 // Mellanox Linux's driver key 91 static uint8_t default_rsskey_40bytes[40] = { 92 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 93 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 94 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 95 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 96 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 97 }; 98 99 static struct rte_eth_conf default_port_conf = { 100 .rxmode = { 101 .mq_mode = ETH_MQ_RX_RSS, 102 .max_rx_pkt_len = ETHER_MAX_LEN, 103 .split_hdr_size = 0, /**< hdr buf size */ 104 .header_split = 0, /**< Header Split disabled */ 105 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 106 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 107 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 108 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 109 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 110 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 111 .enable_lro = 0, /**< LRO disabled */ 112 }, 113 .rx_adv_conf = { 114 .rss_conf = { 115 .rss_key = default_rsskey_40bytes, 116 .rss_key_len = 40, 117 .rss_hf = ETH_RSS_PROTO_MASK, 118 }, 119 }, 120 .txmode = { 121 .mq_mode = ETH_MQ_TX_NONE, 122 }, 123 }; 124 125 struct mbuf_table { 126 uint16_t len; 127 struct rte_mbuf *m_table[MAX_PKT_BURST]; 128 }; 129 130 struct lcore_rx_queue { 131 uint8_t port_id; 132 uint8_t queue_id; 133 } __rte_cache_aligned; 134 135 struct lcore_conf { 136 uint16_t proc_id; 137 uint16_t nb_procs; 138 uint16_t socket_id; 139 uint16_t nb_rx_queue; 140 uint16_t *lcore_proc; 141 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 142 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 143 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 144 char *pcap[RTE_MAX_ETHPORTS]; 145 } __rte_cache_aligned; 146 147 static struct lcore_conf lcore_conf; 148 149 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 150 151 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 152 153 struct ff_dpdk_if_context { 154 void *sc; 155 void *ifp; 156 uint16_t port_id; 157 }; 158 159 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 160 161 extern void ff_hardclock(void); 162 163 static void 164 freebsd_hardclock_job(__rte_unused struct rte_timer *timer, 165 __rte_unused void *arg) { 166 ff_hardclock(); 167 } 168 169 struct ff_dpdk_if_context * 170 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 171 { 172 struct ff_dpdk_if_context *ctx; 173 174 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 175 if (ctx == NULL) 176 return NULL; 177 178 ctx->sc = sc; 179 ctx->ifp = ifp; 180 ctx->port_id = cfg->port_id; 181 182 return ctx; 183 } 184 185 void 186 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 187 { 188 free(ctx); 189 } 190 191 static void 192 check_all_ports_link_status(void) 193 { 194 #define CHECK_INTERVAL 100 /* 100ms */ 195 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 196 197 uint8_t portid, count, all_ports_up, print_flag = 0; 198 struct rte_eth_link link; 199 200 printf("\nChecking link status"); 201 fflush(stdout); 202 203 int i, nb_ports; 204 nb_ports = ff_global_cfg.dpdk.nb_ports; 205 for (count = 0; count <= MAX_CHECK_TIME; count++) { 206 all_ports_up = 1; 207 for (i = 0; i < nb_ports; i++) { 208 uint8_t portid = ff_global_cfg.dpdk.port_cfgs[i].port_id; 209 memset(&link, 0, sizeof(link)); 210 rte_eth_link_get_nowait(portid, &link); 211 212 /* print link status if flag set */ 213 if (print_flag == 1) { 214 if (link.link_status) { 215 printf("Port %d Link Up - speed %u " 216 "Mbps - %s\n", (int)portid, 217 (unsigned)link.link_speed, 218 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 219 ("full-duplex") : ("half-duplex\n")); 220 } else { 221 printf("Port %d Link Down\n", (int)portid); 222 } 223 continue; 224 } 225 /* clear all_ports_up flag if any link down */ 226 if (link.link_status == 0) { 227 all_ports_up = 0; 228 break; 229 } 230 } 231 232 /* after finally printing all link status, get out */ 233 if (print_flag == 1) 234 break; 235 236 if (all_ports_up == 0) { 237 printf("."); 238 fflush(stdout); 239 rte_delay_ms(CHECK_INTERVAL); 240 } 241 242 /* set the print_flag if all ports up or timeout */ 243 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 244 print_flag = 1; 245 printf("done\n"); 246 } 247 } 248 } 249 250 static int 251 xdigit2val(unsigned char c) 252 { 253 int val; 254 255 if (isdigit(c)) 256 val = c - '0'; 257 else if (isupper(c)) 258 val = c - 'A' + 10; 259 else 260 val = c - 'a' + 10; 261 return val; 262 } 263 264 static int 265 parse_lcore_mask(const char *coremask, uint16_t *lcore_proc, 266 uint16_t nb_procs) 267 { 268 int i, j, idx = 0; 269 unsigned count = 0; 270 char c; 271 int val; 272 273 if (coremask == NULL) 274 return -1; 275 276 /* Remove all blank characters ahead and after. 277 * Remove 0x/0X if exists. 278 */ 279 while (isblank(*coremask)) 280 coremask++; 281 if (coremask[0] == '0' && ((coremask[1] == 'x') 282 || (coremask[1] == 'X'))) 283 coremask += 2; 284 285 i = strlen(coremask); 286 while ((i > 0) && isblank(coremask[i - 1])) 287 i--; 288 289 if (i == 0) 290 return -1; 291 292 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE && count < nb_procs; i--) { 293 c = coremask[i]; 294 if (isxdigit(c) == 0) { 295 return -1; 296 } 297 val = xdigit2val(c); 298 for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE && count < nb_procs; 299 j++, idx++) { 300 if ((1 << j) & val) { 301 if (!lcore_config[idx].detected) { 302 RTE_LOG(ERR, EAL, "lcore %u unavailable\n", idx); 303 return -1; 304 } 305 lcore_proc[count] = idx; 306 count++; 307 } 308 } 309 } 310 311 for (; i >= 0; i--) 312 if (coremask[i] != '0') 313 return -1; 314 315 if (count < nb_procs) 316 return -1; 317 318 return 0; 319 } 320 321 static int 322 init_lcore_conf(void) 323 { 324 uint8_t nb_ports = rte_eth_dev_count(); 325 if (nb_ports == 0) { 326 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 327 } 328 329 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 330 lcore_conf.nb_procs = ff_global_cfg.dpdk.nb_procs; 331 lcore_conf.lcore_proc = rte_zmalloc(NULL, 332 sizeof(uint16_t)*lcore_conf.nb_procs, 0); 333 if (lcore_conf.lcore_proc == NULL) { 334 rte_exit(EXIT_FAILURE, "rte_zmalloc lcore_proc failed\n"); 335 } 336 337 int ret = parse_lcore_mask(ff_global_cfg.dpdk.lcore_mask, 338 lcore_conf.lcore_proc, lcore_conf.nb_procs); 339 if (ret < 0) { 340 rte_exit(EXIT_FAILURE, "parse_lcore_mask failed:%s\n", 341 ff_global_cfg.dpdk.lcore_mask); 342 } 343 344 uint16_t socket_id = 0; 345 if (ff_global_cfg.dpdk.numa_on) { 346 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 347 } 348 349 lcore_conf.socket_id = socket_id; 350 351 /* Currently, proc id 1:1 map to rx/tx queue id per port. */ 352 uint8_t port_id, enabled_ports = 0; 353 for (port_id = 0; port_id < nb_ports; port_id++) { 354 if (ff_global_cfg.dpdk.port_mask && 355 (ff_global_cfg.dpdk.port_mask & (1 << port_id)) == 0) { 356 printf("\nSkipping disabled port %d\n", port_id); 357 continue; 358 } 359 360 if (port_id >= ff_global_cfg.dpdk.nb_ports) { 361 printf("\nSkipping non-configured port %d\n", port_id); 362 break; 363 } 364 365 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 366 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 367 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = lcore_conf.proc_id; 368 lcore_conf.nb_rx_queue++; 369 370 lcore_conf.tx_queue_id[port_id] = lcore_conf.proc_id; 371 lcore_conf.pcap[port_id] = ff_global_cfg.dpdk.port_cfgs[enabled_ports].pcap; 372 373 ff_global_cfg.dpdk.port_cfgs[enabled_ports].port_id = port_id; 374 375 enabled_ports++; 376 } 377 378 ff_global_cfg.dpdk.nb_ports = enabled_ports; 379 380 return 0; 381 } 382 383 static int 384 init_mem_pool(void) 385 { 386 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 387 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 388 uint32_t nb_tx_queue = nb_lcores; 389 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 390 391 unsigned nb_mbuf = RTE_MAX ( 392 (nb_rx_queue*RX_QUEUE_SIZE + 393 nb_ports*nb_lcores*MAX_PKT_BURST + 394 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 395 nb_lcores*MEMPOOL_CACHE_SIZE), 396 (unsigned)8192); 397 398 unsigned socketid = 0; 399 uint16_t i, lcore_id; 400 char s[64]; 401 int numa_on = ff_global_cfg.dpdk.numa_on; 402 403 for (i = 0; i < lcore_conf.nb_procs; i++) { 404 lcore_id = lcore_conf.lcore_proc[i]; 405 if (numa_on) { 406 socketid = rte_lcore_to_socket_id(lcore_id); 407 } 408 409 if (socketid >= NB_SOCKETS) { 410 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 411 socketid, i, NB_SOCKETS); 412 } 413 414 if (pktmbuf_pool[socketid] != NULL) { 415 continue; 416 } 417 418 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 419 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 420 pktmbuf_pool[socketid] = 421 rte_pktmbuf_pool_create(s, nb_mbuf, 422 MEMPOOL_CACHE_SIZE, 0, 423 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 424 } else { 425 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 426 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 427 } 428 429 if (pktmbuf_pool[socketid] == NULL) { 430 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 431 } else { 432 printf("create mbuf pool on socket %d\n", socketid); 433 } 434 } 435 436 return 0; 437 } 438 439 static int 440 init_arp_ring(void) 441 { 442 int i, ret; 443 char name_buf[RTE_RING_NAMESIZE]; 444 int nb_procs = ff_global_cfg.dpdk.nb_procs; 445 int proc_id = ff_global_cfg.dpdk.proc_id; 446 447 /* Allocate arp ring ptr according to eth dev count. */ 448 int nb_ports = rte_eth_dev_count(); 449 for(i = 0; i < nb_procs; ++i) { 450 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 451 proc_id, i); 452 453 arp_ring[i] = rte_zmalloc(name_buf, 454 sizeof(struct rte_ring *) * nb_ports, 455 RTE_CACHE_LINE_SIZE); 456 if (arp_ring[i] == NULL) { 457 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 458 "failed\n", name_buf); 459 } 460 } 461 462 unsigned socketid = lcore_conf.socket_id; 463 464 /* Create ring according to ports actually being used. */ 465 nb_ports = ff_global_cfg.dpdk.nb_ports; 466 for (i = 0; i < nb_ports; i++) { 467 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 468 469 for(i = 0; i < nb_procs; ++i) { 470 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_%d_%d", i, port_id); 471 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 472 arp_ring[i][port_id] = rte_ring_create(name_buf, 473 ARP_RING_SIZE, socketid, 474 RING_F_SC_DEQ); 475 } else { 476 arp_ring[i][port_id] = rte_ring_lookup(name_buf); 477 } 478 479 if (arp_ring[i][port_id] == NULL) 480 rte_panic("create kni ring::%s failed!\n", name_buf); 481 482 if (rte_ring_lookup(name_buf) != arp_ring[i][port_id]) 483 rte_panic("lookup kni ring:%s failed!\n", name_buf); 484 485 printf("create arp ring:%s success, %u ring entries are now free!\n", 486 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 487 } 488 } 489 490 return 0; 491 } 492 493 static int 494 init_kni(void) 495 { 496 int nb_ports = rte_eth_dev_count(); 497 kni_accept = 0; 498 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 499 kni_accept = 1; 500 501 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 502 ff_global_cfg.kni.udp_port); 503 504 unsigned socket_id = lcore_conf.socket_id; 505 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 506 507 nb_ports = ff_global_cfg.dpdk.nb_ports; 508 int i, ret; 509 for (i = 0; i < nb_ports; i++) { 510 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 511 ff_kni_alloc(port_id, socket_id, mbuf_pool); 512 } 513 514 return 0; 515 } 516 517 static int 518 init_port_start(void) 519 { 520 int nb_ports = ff_global_cfg.dpdk.nb_ports; 521 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 522 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 523 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 524 uint16_t i; 525 526 for (i = 0; i < nb_ports; i++) { 527 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 528 529 struct rte_eth_dev_info dev_info; 530 rte_eth_dev_info_get(port_id, &dev_info); 531 532 if (nb_procs > dev_info.max_rx_queues) { 533 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 534 nb_procs, 535 dev_info.max_rx_queues); 536 } 537 538 if (nb_procs > dev_info.max_tx_queues) { 539 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 540 nb_procs, 541 dev_info.max_tx_queues); 542 } 543 544 struct ether_addr addr; 545 rte_eth_macaddr_get(port_id, &addr); 546 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 547 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 548 (unsigned)port_id, 549 addr.addr_bytes[0], addr.addr_bytes[1], 550 addr.addr_bytes[2], addr.addr_bytes[3], 551 addr.addr_bytes[4], addr.addr_bytes[5]); 552 553 rte_memcpy(ff_global_cfg.dpdk.port_cfgs[port_id].mac, 554 addr.addr_bytes, ETHER_ADDR_LEN); 555 556 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 557 return 0; 558 } 559 560 /* 561 * TODO: 562 * Set port conf according to dev's capability. 563 */ 564 struct rte_eth_conf port_conf = default_port_conf; 565 port_conf.rxmode.hw_vlan_strip = ff_global_cfg.dpdk.port_cfgs[port_id].vlanstrip; 566 567 /* Currently, proc id 1:1 map to queue id per port. */ 568 int ret = rte_eth_dev_configure(port_id, nb_procs, nb_procs, &port_conf); 569 if (ret != 0) { 570 return ret; 571 } 572 573 uint16_t q; 574 for (q = 0; q < nb_procs; q++) { 575 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 576 socketid, &dev_info.default_txconf); 577 if (ret < 0) { 578 return ret; 579 } 580 581 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 582 socketid, &dev_info.default_rxconf, mbuf_pool); 583 if (ret < 0) { 584 return ret; 585 } 586 } 587 588 ret = rte_eth_dev_start(port_id); 589 if (ret < 0) { 590 return ret; 591 } 592 593 /* Enable RX in promiscuous mode for the Ethernet device. */ 594 if (ff_global_cfg.dpdk.promiscuous) { 595 rte_eth_promiscuous_enable(port_id); 596 ret = rte_eth_promiscuous_get(port_id); 597 if (ret == 1) { 598 printf("set port %u to promiscuous mode ok\n", port_id); 599 } else { 600 printf("set port %u to promiscuous mode error\n", port_id); 601 } 602 } 603 604 /* Enable pcap dump */ 605 if (ff_global_cfg.dpdk.port_cfgs[port_id].pcap) { 606 ff_enable_pcap(ff_global_cfg.dpdk.port_cfgs[port_id].pcap); 607 } 608 } 609 610 return 0; 611 } 612 613 static int 614 init_freebsd_clock(void) 615 { 616 rte_timer_subsystem_init(); 617 uint64_t hz = rte_get_timer_hz(); 618 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 619 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 620 621 rte_timer_init(&freebsd_clock); 622 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 623 rte_lcore_id(), &freebsd_hardclock_job, NULL); 624 625 return 0; 626 } 627 628 int 629 ff_dpdk_init(int argc, char **argv) 630 { 631 if (ff_global_cfg.dpdk.nb_procs < 1 || 632 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 633 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 634 ff_global_cfg.dpdk.nb_procs < 0) { 635 printf("param num_procs[%d] or proc_id[%d] error!\n", 636 ff_global_cfg.dpdk.nb_procs, 637 ff_global_cfg.dpdk.proc_id); 638 exit(1); 639 } 640 641 int ret = rte_eal_init(argc, argv); 642 if (ret < 0) { 643 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 644 } 645 646 init_lcore_conf(); 647 648 init_mem_pool(); 649 650 init_arp_ring(); 651 652 enable_kni = ff_global_cfg.kni.enable; 653 if (enable_kni) { 654 init_kni(); 655 } 656 657 ret = init_port_start(); 658 if (ret < 0) { 659 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 660 } 661 662 check_all_ports_link_status(); 663 664 init_freebsd_clock(); 665 666 return 0; 667 } 668 669 static void 670 ff_veth_input(void *ifp, struct rte_mbuf *pkt) 671 { 672 void *data = rte_pktmbuf_mtod(pkt, void*); 673 uint16_t len = rte_pktmbuf_data_len(pkt); 674 675 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len); 676 if (hdr == NULL) { 677 rte_pktmbuf_free(pkt); 678 return; 679 } 680 681 pkt = pkt->next; 682 void *prev = hdr; 683 while(pkt != NULL) { 684 data = rte_pktmbuf_mtod(pkt, void*); 685 len = rte_pktmbuf_data_len(pkt); 686 687 void *mb = ff_mbuf_get(prev, data, len); 688 if (mb == NULL) { 689 ff_mbuf_free(hdr); 690 return; 691 } 692 pkt = pkt->next; 693 prev = mb; 694 } 695 696 ff_veth_process_packet(ifp, hdr); 697 } 698 699 static enum FilterReturn 700 protocol_filter(const void *data, uint16_t len) 701 { 702 if(len < sizeof(struct ether_hdr)) 703 return FILTER_UNKNOWN; 704 705 const struct ether_hdr *hdr; 706 hdr = (const struct ether_hdr *)data; 707 708 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 709 return FILTER_ARP; 710 711 if (!enable_kni) { 712 return FILTER_UNKNOWN; 713 } 714 715 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 716 return FILTER_UNKNOWN; 717 718 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 719 len - sizeof(struct ether_hdr)); 720 } 721 722 static inline void 723 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 724 uint16_t count, void *ifp, int pkts_from_ring) 725 { 726 struct lcore_conf *qconf = &lcore_conf; 727 728 uint16_t i; 729 for (i = 0; i < count; i++) { 730 struct rte_mbuf *rtem = bufs[i]; 731 732 if (unlikely(qconf->pcap[port_id] != NULL)) { 733 ff_dump_packets(qconf->pcap[port_id], rtem); 734 } 735 736 void *data = rte_pktmbuf_mtod(rtem, void*); 737 uint16_t len = rte_pktmbuf_data_len(rtem); 738 739 enum FilterReturn filter = protocol_filter(data, len); 740 if (filter == FILTER_ARP) { 741 struct rte_mempool *mbuf_pool; 742 struct rte_mbuf *mbuf_clone; 743 if (pkts_from_ring == 0) { 744 uint16_t i; 745 for(i = 0; i < qconf->nb_procs; ++i) { 746 if(i == queue_id) 747 continue; 748 749 mbuf_pool = pktmbuf_pool[rte_lcore_to_socket_id(qconf->lcore_proc[i])]; 750 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 751 if(mbuf_clone) { 752 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 753 if (ret < 0) 754 rte_pktmbuf_free(mbuf_clone); 755 } 756 } 757 } 758 759 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 760 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 761 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 762 if(mbuf_clone) { 763 ff_kni_enqueue(port_id, rtem); 764 } 765 } 766 767 ff_veth_input(ifp, rtem); 768 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 769 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 770 ff_kni_enqueue(port_id, rtem); 771 } else { 772 ff_veth_input(ifp, rtem); 773 } 774 } 775 } 776 777 static inline int 778 process_arp_ring(uint8_t port_id, uint16_t queue_id, 779 struct rte_mbuf **pkts_burst, void *ifp) 780 { 781 /* read packet from ring buf and to process */ 782 uint16_t nb_tx; 783 nb_tx = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 784 (void **)pkts_burst, MAX_PKT_BURST); 785 786 if(nb_tx > 0) { 787 process_packets(port_id, queue_id, pkts_burst, nb_tx, ifp, 1); 788 } 789 790 return 0; 791 } 792 793 /* Send burst of packets on an output interface */ 794 static inline int 795 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 796 { 797 struct rte_mbuf **m_table; 798 int ret; 799 uint16_t queueid; 800 801 queueid = qconf->tx_queue_id[port]; 802 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 803 804 if (unlikely(qconf->pcap[port] != NULL)) { 805 uint16_t i; 806 for (i = 0; i < n; i++) { 807 ff_dump_packets(qconf->pcap[port], m_table[i]); 808 } 809 } 810 811 ret = rte_eth_tx_burst(port, queueid, m_table, n); 812 if (unlikely(ret < n)) { 813 do { 814 rte_pktmbuf_free(m_table[ret]); 815 } while (++ret < n); 816 } 817 818 return 0; 819 } 820 821 /* Enqueue a single packet, and send burst if queue is filled */ 822 static inline int 823 send_single_packet(struct rte_mbuf *m, uint8_t port) 824 { 825 uint16_t len; 826 struct lcore_conf *qconf; 827 828 qconf = &lcore_conf; 829 len = qconf->tx_mbufs[port].len; 830 qconf->tx_mbufs[port].m_table[len] = m; 831 len++; 832 833 /* enough pkts to be sent */ 834 if (unlikely(len == MAX_PKT_BURST)) { 835 send_burst(qconf, MAX_PKT_BURST, port); 836 len = 0; 837 } 838 839 qconf->tx_mbufs[port].len = len; 840 return 0; 841 } 842 843 int 844 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 845 int total) 846 { 847 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 848 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 849 if (head == NULL) { 850 ff_mbuf_free(m); 851 return -1; 852 } 853 854 head->pkt_len = total; 855 856 int off = 0; 857 struct rte_mbuf *cur = head, *prev = NULL; 858 while(total > 0) { 859 if (cur == NULL) { 860 struct rte_mbuf *cur = rte_pktmbuf_alloc(mbuf_pool); 861 if (cur == NULL) { 862 rte_pktmbuf_free(head); 863 ff_mbuf_free(m); 864 return -1; 865 } 866 } 867 868 void *data = rte_pktmbuf_mtod(cur, void*); 869 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 870 int ret = ff_mbuf_copydata(m, data, off, len); 871 if (ret < 0) { 872 rte_pktmbuf_free(head); 873 ff_mbuf_free(m); 874 return -1; 875 } 876 877 if (prev == NULL) { 878 prev = cur; 879 } else { 880 prev->next = cur; 881 } 882 883 cur->data_len = len; 884 off += len; 885 total -= len; 886 head->nb_segs++; 887 } 888 889 /* 890 * FIXME: set offload flags according to mbuf.pkthdr; 891 */ 892 head->ol_flags = 0; 893 head->vlan_tci = 0; 894 895 ff_mbuf_free(m); 896 897 return send_single_packet(head, ctx->port_id); 898 } 899 900 static int 901 main_loop(void *arg) 902 { 903 struct loop_routine *lr = (struct loop_routine *)arg; 904 905 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 906 unsigned lcore_id; 907 uint64_t prev_tsc, diff_tsc, cur_tsc; 908 int i, j, nb_rx; 909 uint8_t port_id, queue_id; 910 struct lcore_conf *qconf; 911 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 912 US_PER_S * BURST_TX_DRAIN_US; 913 void *ifp; 914 915 prev_tsc = 0; 916 917 lcore_id = rte_lcore_id(); 918 qconf = &lcore_conf; 919 920 if (qconf->nb_rx_queue == 0) { 921 printf("lcore %u has nothing to do\n", lcore_id); 922 return 0; 923 } 924 925 while (1) { 926 cur_tsc = rte_rdtsc(); 927 if (unlikely(freebsd_clock.expire < cur_tsc)) { 928 rte_timer_manage(); 929 } 930 931 /* 932 * TX burst queue drain 933 */ 934 diff_tsc = cur_tsc - prev_tsc; 935 if (unlikely(diff_tsc > drain_tsc)) { 936 /* 937 * This could be optimized (use queueid instead of 938 * portid), but it is not called so often 939 */ 940 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 941 if (qconf->tx_mbufs[port_id].len == 0) 942 continue; 943 send_burst(qconf, 944 qconf->tx_mbufs[port_id].len, 945 port_id); 946 qconf->tx_mbufs[port_id].len = 0; 947 } 948 949 prev_tsc = cur_tsc; 950 } 951 952 /* 953 * Read packet from RX queues 954 */ 955 for (i = 0; i < qconf->nb_rx_queue; ++i) { 956 port_id = qconf->rx_queue_list[i].port_id; 957 queue_id = qconf->rx_queue_list[i].queue_id; 958 ifp = veth_ctx[port_id]->ifp; 959 960 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 961 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 962 } 963 964 process_arp_ring(port_id, queue_id, pkts_burst, ifp); 965 966 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 967 MAX_PKT_BURST); 968 if (nb_rx == 0) 969 continue; 970 971 /* Prefetch first packets */ 972 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 973 rte_prefetch0(rte_pktmbuf_mtod( 974 pkts_burst[j], void *)); 975 } 976 977 /* Prefetch and handle already prefetched packets */ 978 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 979 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 980 j + PREFETCH_OFFSET], void *)); 981 process_packets(port_id, queue_id, &pkts_burst[j], 1, ifp, 0); 982 } 983 984 /* Handle remaining prefetched packets */ 985 for (; j < nb_rx; j++) { 986 process_packets(port_id, queue_id, &pkts_burst[j], 1, ifp, 0); 987 } 988 } 989 990 if (likely(lr->loop != NULL)) { 991 lr->loop(lr->arg); 992 } 993 } 994 } 995 996 int 997 ff_dpdk_if_up(void) { 998 int nb_ports = ff_global_cfg.dpdk.nb_ports; 999 int i; 1000 for (i = 0; i < nb_ports; i++) { 1001 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 1002 veth_ctx[port_id] = ff_veth_attach(ff_global_cfg.dpdk.port_cfgs + i); 1003 if (veth_ctx[port_id] == NULL) { 1004 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1005 } 1006 } 1007 1008 return 0; 1009 } 1010 1011 void 1012 ff_dpdk_run(loop_func_t loop, void *arg) { 1013 struct loop_routine *lr = malloc(sizeof(struct loop_routine)); 1014 lr->loop = loop; 1015 lr->arg = arg; 1016 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1017 rte_eal_mp_wait_lcore(); 1018 free(lr); 1019 } 1020 1021 void 1022 ff_dpdk_pktmbuf_free(void *m) 1023 { 1024 rte_pktmbuf_free((struct rte_mbuf *)m); 1025 } 1026 1027 static uint32_t 1028 toeplitz_hash(unsigned keylen, const uint8_t *key, 1029 unsigned datalen, const uint8_t *data) 1030 { 1031 uint32_t hash = 0, v; 1032 u_int i, b; 1033 1034 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1035 1036 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1037 for (i = 0; i < datalen; i++) { 1038 for (b = 0; b < 8; b++) { 1039 if (data[i] & (1<<(7-b))) 1040 hash ^= v; 1041 v <<= 1; 1042 if ((i + 4) < keylen && 1043 (key[i+4] & (1<<(7-b)))) 1044 v |= 1; 1045 } 1046 } 1047 return (hash); 1048 } 1049 1050 int 1051 ff_rss_check(uint32_t saddr, uint32_t daddr, uint16_t sport, uint16_t dport) 1052 { 1053 struct lcore_conf *qconf = &lcore_conf; 1054 1055 if (qconf->nb_procs == 1) { 1056 return 1; 1057 } 1058 1059 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1060 sizeof(dport)]; 1061 1062 unsigned datalen = 0; 1063 1064 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1065 datalen += sizeof(saddr); 1066 1067 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1068 datalen += sizeof(daddr); 1069 1070 bcopy(&sport, &data[datalen], sizeof(sport)); 1071 datalen += sizeof(sport); 1072 1073 bcopy(&dport, &data[datalen], sizeof(dport)); 1074 datalen += sizeof(dport); 1075 1076 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), default_rsskey_40bytes, datalen, data); 1077 1078 return (hash % qconf->nb_procs) == qconf->proc_id; 1079 } 1080 1081 1082