1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <rte_common.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_memzone.h> 33 #include <rte_config.h> 34 #include <rte_eal.h> 35 #include <rte_pci.h> 36 #include <rte_mbuf.h> 37 #include <rte_memory.h> 38 #include <rte_lcore.h> 39 #include <rte_launch.h> 40 #include <rte_ethdev.h> 41 #include <rte_debug.h> 42 #include <rte_common.h> 43 #include <rte_ether.h> 44 #include <rte_malloc.h> 45 #include <rte_cycles.h> 46 #include <rte_timer.h> 47 #include <rte_thash.h> 48 49 #include "ff_dpdk_if.h" 50 #include "ff_dpdk_pcap.h" 51 #include "ff_dpdk_kni.h" 52 #include "ff_config.h" 53 #include "ff_veth.h" 54 #include "ff_host_interface.h" 55 56 #define MEMPOOL_CACHE_SIZE 256 57 58 #define ARP_RING_SIZE 2048 59 60 /* 61 * Configurable number of RX/TX ring descriptors 62 */ 63 #define RX_QUEUE_SIZE 512 64 #define TX_QUEUE_SIZE 256 65 66 #define MAX_PKT_BURST 32 67 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 68 69 /* 70 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 71 */ 72 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 73 74 #define NB_SOCKETS 8 75 76 /* Configure how many packets ahead to prefetch, when reading packets */ 77 #define PREFETCH_OFFSET 3 78 79 #define MAX_RX_QUEUE_PER_LCORE 16 80 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 81 #define MAX_RX_QUEUE_PER_PORT 128 82 83 #define BITS_PER_HEX 4 84 85 static int enable_kni = 0; 86 87 static struct rte_timer freebsd_clock; 88 89 // Mellanox Linux's driver key 90 static uint8_t default_rsskey_40bytes[40] = { 91 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 92 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 93 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 94 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 95 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 96 }; 97 98 static struct rte_eth_conf default_port_conf = { 99 .rxmode = { 100 .mq_mode = ETH_MQ_RX_RSS, 101 .max_rx_pkt_len = ETHER_MAX_LEN, 102 .split_hdr_size = 0, /**< hdr buf size */ 103 .header_split = 0, /**< Header Split disabled */ 104 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 105 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 106 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 107 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 108 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 109 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 110 .enable_lro = 0, /**< LRO disabled */ 111 }, 112 .rx_adv_conf = { 113 .rss_conf = { 114 .rss_key = default_rsskey_40bytes, 115 .rss_key_len = 40, 116 .rss_hf = ETH_RSS_PROTO_MASK, 117 }, 118 }, 119 .txmode = { 120 .mq_mode = ETH_MQ_TX_NONE, 121 }, 122 }; 123 124 struct mbuf_table { 125 uint16_t len; 126 struct rte_mbuf *m_table[MAX_PKT_BURST]; 127 }; 128 129 struct lcore_rx_queue { 130 uint8_t port_id; 131 uint8_t queue_id; 132 } __rte_cache_aligned; 133 134 struct lcore_conf { 135 uint16_t proc_id; 136 uint16_t nb_procs; 137 uint16_t socket_id; 138 uint16_t nb_rx_queue; 139 uint16_t *lcore_proc; 140 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 141 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 142 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 143 char *pcap[RTE_MAX_ETHPORTS]; 144 } __rte_cache_aligned; 145 146 static struct lcore_conf lcore_conf; 147 148 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 149 150 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 151 152 struct ff_dpdk_if_context { 153 void *sc; 154 void *ifp; 155 uint16_t port_id; 156 }; 157 158 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 159 160 extern void ff_hardclock(void); 161 162 static void 163 freebsd_hardclock_job(__rte_unused struct rte_timer *timer, 164 __rte_unused void *arg) { 165 ff_hardclock(); 166 } 167 168 struct ff_dpdk_if_context * 169 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 170 { 171 struct ff_dpdk_if_context *ctx; 172 173 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 174 if (ctx == NULL) 175 return NULL; 176 177 ctx->sc = sc; 178 ctx->ifp = ifp; 179 ctx->port_id = cfg->port_id; 180 181 return ctx; 182 } 183 184 void 185 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 186 { 187 free(ctx); 188 } 189 190 static void 191 check_all_ports_link_status(void) 192 { 193 #define CHECK_INTERVAL 100 /* 100ms */ 194 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 195 196 uint8_t portid, count, all_ports_up, print_flag = 0; 197 struct rte_eth_link link; 198 199 printf("\nChecking link status"); 200 fflush(stdout); 201 202 int i, nb_ports; 203 nb_ports = ff_global_cfg.dpdk.nb_ports; 204 for (count = 0; count <= MAX_CHECK_TIME; count++) { 205 all_ports_up = 1; 206 for (i = 0; i < nb_ports; i++) { 207 uint8_t portid = ff_global_cfg.dpdk.port_cfgs[i].port_id; 208 memset(&link, 0, sizeof(link)); 209 rte_eth_link_get_nowait(portid, &link); 210 211 /* print link status if flag set */ 212 if (print_flag == 1) { 213 if (link.link_status) { 214 printf("Port %d Link Up - speed %u " 215 "Mbps - %s\n", (int)portid, 216 (unsigned)link.link_speed, 217 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 218 ("full-duplex") : ("half-duplex\n")); 219 } else { 220 printf("Port %d Link Down\n", (int)portid); 221 } 222 continue; 223 } 224 /* clear all_ports_up flag if any link down */ 225 if (link.link_status == 0) { 226 all_ports_up = 0; 227 break; 228 } 229 } 230 231 /* after finally printing all link status, get out */ 232 if (print_flag == 1) 233 break; 234 235 if (all_ports_up == 0) { 236 printf("."); 237 fflush(stdout); 238 rte_delay_ms(CHECK_INTERVAL); 239 } 240 241 /* set the print_flag if all ports up or timeout */ 242 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 243 print_flag = 1; 244 printf("done\n"); 245 } 246 } 247 } 248 249 static int 250 xdigit2val(unsigned char c) 251 { 252 int val; 253 254 if (isdigit(c)) 255 val = c - '0'; 256 else if (isupper(c)) 257 val = c - 'A' + 10; 258 else 259 val = c - 'a' + 10; 260 return val; 261 } 262 263 static int 264 parse_lcore_mask(const char *coremask, uint16_t *lcore_proc, 265 uint16_t nb_procs) 266 { 267 int i, j, idx = 0; 268 unsigned count = 0; 269 char c; 270 int val; 271 272 if (coremask == NULL) 273 return -1; 274 275 /* Remove all blank characters ahead and after. 276 * Remove 0x/0X if exists. 277 */ 278 while (isblank(*coremask)) 279 coremask++; 280 if (coremask[0] == '0' && ((coremask[1] == 'x') 281 || (coremask[1] == 'X'))) 282 coremask += 2; 283 284 i = strlen(coremask); 285 while ((i > 0) && isblank(coremask[i - 1])) 286 i--; 287 288 if (i == 0) 289 return -1; 290 291 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE && count < nb_procs; i--) { 292 c = coremask[i]; 293 if (isxdigit(c) == 0) { 294 return -1; 295 } 296 val = xdigit2val(c); 297 for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE && count < nb_procs; 298 j++, idx++) { 299 if ((1 << j) & val) { 300 if (!lcore_config[idx].detected) { 301 RTE_LOG(ERR, EAL, "lcore %u unavailable\n", idx); 302 return -1; 303 } 304 lcore_proc[count] = idx; 305 count++; 306 } 307 } 308 } 309 310 for (; i >= 0; i--) 311 if (coremask[i] != '0') 312 return -1; 313 314 if (count < nb_procs) 315 return -1; 316 317 return 0; 318 } 319 320 static int 321 init_lcore_conf(void) 322 { 323 uint8_t nb_ports = rte_eth_dev_count(); 324 if (nb_ports == 0) { 325 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 326 } 327 328 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 329 lcore_conf.nb_procs = ff_global_cfg.dpdk.nb_procs; 330 lcore_conf.lcore_proc = rte_zmalloc(NULL, 331 sizeof(uint16_t)*lcore_conf.nb_procs, 0); 332 if (lcore_conf.lcore_proc == NULL) { 333 rte_exit(EXIT_FAILURE, "rte_zmalloc lcore_proc failed\n"); 334 } 335 336 int ret = parse_lcore_mask(ff_global_cfg.dpdk.lcore_mask, 337 lcore_conf.lcore_proc, lcore_conf.nb_procs); 338 if (ret < 0) { 339 rte_exit(EXIT_FAILURE, "parse_lcore_mask failed:%s\n", 340 ff_global_cfg.dpdk.lcore_mask); 341 } 342 343 uint16_t socket_id = 0; 344 if (ff_global_cfg.dpdk.numa_on) { 345 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 346 } 347 348 lcore_conf.socket_id = socket_id; 349 350 /* Currently, proc id 1:1 map to rx/tx queue id per port. */ 351 uint8_t port_id, enabled_ports = 0; 352 for (port_id = 0; port_id < nb_ports; port_id++) { 353 if (ff_global_cfg.dpdk.port_mask && 354 (ff_global_cfg.dpdk.port_mask & (1 << port_id)) == 0) { 355 printf("\nSkipping disabled port %d\n", port_id); 356 continue; 357 } 358 359 if (port_id >= ff_global_cfg.dpdk.nb_ports) { 360 printf("\nSkipping non-configured port %d\n", port_id); 361 break; 362 } 363 364 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 365 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 366 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = lcore_conf.proc_id; 367 lcore_conf.nb_rx_queue++; 368 369 lcore_conf.tx_queue_id[port_id] = lcore_conf.proc_id; 370 lcore_conf.pcap[port_id] = ff_global_cfg.dpdk.port_cfgs[enabled_ports].pcap; 371 372 ff_global_cfg.dpdk.port_cfgs[enabled_ports].port_id = port_id; 373 374 enabled_ports++; 375 } 376 377 ff_global_cfg.dpdk.nb_ports = enabled_ports; 378 379 return 0; 380 } 381 382 static int 383 init_mem_pool(void) 384 { 385 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 386 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 387 uint32_t nb_tx_queue = nb_lcores; 388 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 389 390 unsigned nb_mbuf = RTE_MAX ( 391 (nb_rx_queue*RX_QUEUE_SIZE + 392 nb_ports*nb_lcores*MAX_PKT_BURST + 393 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 394 nb_lcores*MEMPOOL_CACHE_SIZE), 395 (unsigned)8192); 396 397 unsigned socketid = 0; 398 uint16_t i, lcore_id; 399 char s[64]; 400 int numa_on = ff_global_cfg.dpdk.numa_on; 401 402 for (i = 0; i < lcore_conf.nb_procs; i++) { 403 lcore_id = lcore_conf.lcore_proc[i]; 404 if (numa_on) { 405 socketid = rte_lcore_to_socket_id(lcore_id); 406 } 407 408 if (socketid >= NB_SOCKETS) { 409 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 410 socketid, i, NB_SOCKETS); 411 } 412 413 if (pktmbuf_pool[socketid] != NULL) { 414 continue; 415 } 416 417 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 418 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 419 pktmbuf_pool[socketid] = 420 rte_pktmbuf_pool_create(s, nb_mbuf, 421 MEMPOOL_CACHE_SIZE, 0, 422 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 423 } else { 424 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 425 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 426 } 427 428 if (pktmbuf_pool[socketid] == NULL) { 429 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 430 } else { 431 printf("create mbuf pool on socket %d\n", socketid); 432 } 433 } 434 435 return 0; 436 } 437 438 static int 439 init_arp_ring(void) 440 { 441 int i, ret; 442 char name_buf[RTE_RING_NAMESIZE]; 443 int nb_procs = ff_global_cfg.dpdk.nb_procs; 444 int proc_id = ff_global_cfg.dpdk.proc_id; 445 446 /* Allocate arp ring ptr according to eth dev count. */ 447 int nb_ports = rte_eth_dev_count(); 448 for(i = 0; i < nb_procs; ++i) { 449 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 450 proc_id, i); 451 452 arp_ring[i] = rte_zmalloc(name_buf, 453 sizeof(struct rte_ring *) * nb_ports, 454 RTE_CACHE_LINE_SIZE); 455 if (arp_ring[i] == NULL) { 456 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 457 "failed\n", name_buf); 458 } 459 } 460 461 unsigned socketid = lcore_conf.socket_id; 462 463 /* Create ring according to ports actually being used. */ 464 nb_ports = ff_global_cfg.dpdk.nb_ports; 465 for (i = 0; i < nb_ports; i++) { 466 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 467 468 for(i = 0; i < nb_procs; ++i) { 469 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_%d_%d", i, port_id); 470 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 471 arp_ring[i][port_id] = rte_ring_create(name_buf, 472 ARP_RING_SIZE, socketid, 473 RING_F_SC_DEQ); 474 } else { 475 arp_ring[i][port_id] = rte_ring_lookup(name_buf); 476 } 477 478 if (arp_ring[i][port_id] == NULL) 479 rte_panic("create kni ring::%s failed!\n", name_buf); 480 481 if (rte_ring_lookup(name_buf) != arp_ring[i][port_id]) 482 rte_panic("lookup kni ring:%s failed!\n", name_buf); 483 484 printf("create arp ring:%s success, %u ring entries are now free!\n", 485 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 486 } 487 } 488 489 return 0; 490 } 491 492 static int 493 init_kni(void) 494 { 495 int nb_ports = rte_eth_dev_count(); 496 int accept = 0; 497 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 498 accept = 1; 499 500 ff_kni_init(nb_ports, 501 ff_global_cfg.kni.tcp_port, 502 ff_global_cfg.kni.udp_port, 503 accept); 504 505 unsigned socket_id = lcore_conf.socket_id; 506 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 507 508 nb_ports = ff_global_cfg.dpdk.nb_ports; 509 int i, ret; 510 for (i = 0; i < nb_ports; i++) { 511 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 512 ff_kni_alloc(port_id, socket_id, mbuf_pool); 513 } 514 515 return 0; 516 } 517 518 static int 519 init_port_start(void) 520 { 521 int nb_ports = ff_global_cfg.dpdk.nb_ports; 522 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 523 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 524 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 525 uint16_t i; 526 527 for (i = 0; i < nb_ports; i++) { 528 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 529 530 struct rte_eth_dev_info dev_info; 531 rte_eth_dev_info_get(port_id, &dev_info); 532 533 if (nb_procs > dev_info.max_rx_queues) { 534 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 535 nb_procs, 536 dev_info.max_rx_queues); 537 } 538 539 if (nb_procs > dev_info.max_tx_queues) { 540 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 541 nb_procs, 542 dev_info.max_tx_queues); 543 } 544 545 struct ether_addr addr; 546 rte_eth_macaddr_get(port_id, &addr); 547 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 548 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 549 (unsigned)port_id, 550 addr.addr_bytes[0], addr.addr_bytes[1], 551 addr.addr_bytes[2], addr.addr_bytes[3], 552 addr.addr_bytes[4], addr.addr_bytes[5]); 553 554 rte_memcpy(ff_global_cfg.dpdk.port_cfgs[port_id].mac, 555 addr.addr_bytes, ETHER_ADDR_LEN); 556 557 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 558 return 0; 559 } 560 561 /* 562 * TODO: 563 * Set port conf according to dev's capability. 564 */ 565 struct rte_eth_conf port_conf = default_port_conf; 566 port_conf.rxmode.hw_vlan_strip = ff_global_cfg.dpdk.port_cfgs[port_id].vlanstrip; 567 568 /* Currently, proc id 1:1 map to queue id per port. */ 569 int ret = rte_eth_dev_configure(port_id, nb_procs, nb_procs, &port_conf); 570 if (ret != 0) { 571 return ret; 572 } 573 574 uint16_t q; 575 for (q = 0; q < nb_procs; q++) { 576 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 577 socketid, &dev_info.default_txconf); 578 if (ret < 0) { 579 return ret; 580 } 581 582 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 583 socketid, &dev_info.default_rxconf, mbuf_pool); 584 if (ret < 0) { 585 return ret; 586 } 587 } 588 589 ret = rte_eth_dev_start(port_id); 590 if (ret < 0) { 591 return ret; 592 } 593 594 /* Enable RX in promiscuous mode for the Ethernet device. */ 595 if (ff_global_cfg.dpdk.promiscuous) { 596 rte_eth_promiscuous_enable(port_id); 597 ret = rte_eth_promiscuous_get(port_id); 598 if (ret == 1) { 599 printf("set port %u to promiscuous mode ok\n", port_id); 600 } else { 601 printf("set port %u to promiscuous mode error\n", port_id); 602 } 603 } 604 605 /* Enable pcap dump */ 606 if (ff_global_cfg.dpdk.port_cfgs[port_id].pcap) { 607 ff_enable_pcap(ff_global_cfg.dpdk.port_cfgs[port_id].pcap); 608 } 609 } 610 611 return 0; 612 } 613 614 static int 615 init_freebsd_clock(void) 616 { 617 rte_timer_subsystem_init(); 618 uint64_t hz = rte_get_timer_hz(); 619 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 620 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 621 622 rte_timer_init(&freebsd_clock); 623 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 624 rte_lcore_id(), &freebsd_hardclock_job, NULL); 625 626 return 0; 627 } 628 629 int 630 ff_dpdk_init(int argc, char **argv) 631 { 632 if (ff_global_cfg.dpdk.nb_procs < 1 || 633 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 634 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 635 ff_global_cfg.dpdk.nb_procs < 0) { 636 printf("param num_procs[%d] or proc_id[%d] error!\n", 637 ff_global_cfg.dpdk.nb_procs, 638 ff_global_cfg.dpdk.proc_id); 639 exit(1); 640 } 641 642 int ret = rte_eal_init(argc, argv); 643 if (ret < 0) { 644 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 645 } 646 647 init_lcore_conf(); 648 649 init_mem_pool(); 650 651 init_arp_ring(); 652 653 enable_kni = ff_global_cfg.kni.enable; 654 if (enable_kni) { 655 init_kni(); 656 } 657 658 ret = init_port_start(); 659 if (ret < 0) { 660 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 661 } 662 663 check_all_ports_link_status(); 664 665 init_freebsd_clock(); 666 667 return 0; 668 } 669 670 static void 671 ff_veth_input(void *ifp, struct rte_mbuf *pkt) 672 { 673 void *data = rte_pktmbuf_mtod(pkt, void*); 674 uint16_t len = rte_pktmbuf_data_len(pkt); 675 676 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len); 677 if (hdr == NULL) { 678 rte_pktmbuf_free(pkt); 679 return; 680 } 681 682 pkt = pkt->next; 683 void *prev = hdr; 684 while(pkt != NULL) { 685 data = rte_pktmbuf_mtod(pkt, void*); 686 len = rte_pktmbuf_data_len(pkt); 687 688 void *mb = ff_mbuf_get(prev, data, len); 689 if (mb == NULL) { 690 ff_mbuf_free(hdr); 691 return; 692 } 693 pkt = pkt->next; 694 prev = mb; 695 } 696 697 ff_veth_process_packet(ifp, hdr); 698 } 699 700 static enum FilterReturn 701 protocol_filter(const void *data, uint16_t len) 702 { 703 if(len < sizeof(struct ether_hdr)) 704 return FILTER_UNKNOWN; 705 706 const struct ether_hdr *hdr; 707 hdr = (const struct ether_hdr *)data; 708 709 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 710 return FILTER_ARP; 711 712 if (!enable_kni) { 713 return FILTER_UNKNOWN; 714 } 715 716 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 717 return FILTER_UNKNOWN; 718 719 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 720 len - sizeof(struct ether_hdr)); 721 } 722 723 static inline void 724 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 725 uint16_t count, void *ifp, int pkts_from_ring) 726 { 727 struct lcore_conf *qconf = &lcore_conf; 728 729 uint16_t i; 730 for (i = 0; i < count; i++) { 731 struct rte_mbuf *rtem = bufs[i]; 732 733 if (unlikely(qconf->pcap[port_id] != NULL)) { 734 ff_dump_packets(qconf->pcap[port_id], rtem); 735 } 736 737 void *data = rte_pktmbuf_mtod(rtem, void*); 738 uint16_t len = rte_pktmbuf_data_len(rtem); 739 740 enum FilterReturn filter = protocol_filter(data, len); 741 if (filter == FILTER_UNKNOWN) { 742 ff_veth_input(ifp, rtem); 743 } else if (filter == FILTER_KNI) { 744 ff_kni_enqueue(port_id, rtem); 745 } else { 746 struct rte_mempool *mbuf_pool; 747 struct rte_mbuf *mbuf_clone; 748 if (pkts_from_ring == 0) { 749 uint16_t i; 750 for(i = 0; i < qconf->nb_procs; ++i) { 751 if(i == queue_id) 752 continue; 753 754 mbuf_pool = pktmbuf_pool[rte_lcore_to_socket_id(qconf->lcore_proc[i])]; 755 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 756 if(mbuf_clone) { 757 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 758 if (ret < 0) 759 rte_pktmbuf_free(mbuf_clone); 760 } 761 } 762 } 763 764 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 765 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 766 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 767 if(mbuf_clone) { 768 ff_kni_enqueue(port_id, rtem); 769 } 770 } 771 772 ff_veth_input(ifp, rtem); 773 } 774 } 775 } 776 777 static inline int 778 process_arp_ring(uint8_t port_id, uint16_t queue_id, 779 struct rte_mbuf **pkts_burst, void *ifp) 780 { 781 /* read packet from ring buf and to process */ 782 uint16_t nb_tx; 783 nb_tx = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 784 (void **)pkts_burst, MAX_PKT_BURST); 785 786 if(nb_tx > 0) { 787 process_packets(port_id, queue_id, pkts_burst, nb_tx, ifp, 1); 788 } 789 790 return 0; 791 } 792 793 /* Send burst of packets on an output interface */ 794 static inline int 795 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 796 { 797 struct rte_mbuf **m_table; 798 int ret; 799 uint16_t queueid; 800 801 queueid = qconf->tx_queue_id[port]; 802 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 803 804 if (unlikely(qconf->pcap[port] != NULL)) { 805 uint16_t i; 806 for (i = 0; i < n; i++) { 807 ff_dump_packets(qconf->pcap[port], m_table[i]); 808 } 809 } 810 811 ret = rte_eth_tx_burst(port, queueid, m_table, n); 812 if (unlikely(ret < n)) { 813 do { 814 rte_pktmbuf_free(m_table[ret]); 815 } while (++ret < n); 816 } 817 818 return 0; 819 } 820 821 /* Enqueue a single packet, and send burst if queue is filled */ 822 static inline int 823 send_single_packet(struct rte_mbuf *m, uint8_t port) 824 { 825 uint16_t len; 826 struct lcore_conf *qconf; 827 828 qconf = &lcore_conf; 829 len = qconf->tx_mbufs[port].len; 830 qconf->tx_mbufs[port].m_table[len] = m; 831 len++; 832 833 /* enough pkts to be sent */ 834 if (unlikely(len == MAX_PKT_BURST)) { 835 send_burst(qconf, MAX_PKT_BURST, port); 836 len = 0; 837 } 838 839 qconf->tx_mbufs[port].len = len; 840 return 0; 841 } 842 843 int 844 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 845 int total) 846 { 847 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 848 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 849 if (head == NULL) { 850 ff_mbuf_free(m); 851 return -1; 852 } 853 854 head->pkt_len = total; 855 856 int off = 0; 857 struct rte_mbuf *cur = head, *prev = NULL; 858 while(total > 0) { 859 if (cur == NULL) { 860 struct rte_mbuf *cur = rte_pktmbuf_alloc(mbuf_pool); 861 if (cur == NULL) { 862 rte_pktmbuf_free(head); 863 ff_mbuf_free(m); 864 return -1; 865 } 866 } 867 868 void *data = rte_pktmbuf_mtod(cur, void*); 869 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 870 int ret = ff_mbuf_copydata(m, data, off, len); 871 if (ret < 0) { 872 rte_pktmbuf_free(head); 873 ff_mbuf_free(m); 874 return -1; 875 } 876 877 if (prev == NULL) { 878 prev = cur; 879 } else { 880 prev->next = cur; 881 } 882 883 cur->data_len = len; 884 off += len; 885 total -= len; 886 head->nb_segs++; 887 } 888 889 /* 890 * FIXME: set offload flags according to mbuf.pkthdr; 891 */ 892 head->ol_flags = 0; 893 head->vlan_tci = 0; 894 895 ff_mbuf_free(m); 896 897 return send_single_packet(head, ctx->port_id); 898 } 899 900 static int 901 main_loop(void *arg) 902 { 903 struct loop_routine *lr = (struct loop_routine *)arg; 904 905 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 906 unsigned lcore_id; 907 uint64_t prev_tsc, diff_tsc, cur_tsc; 908 int i, j, nb_rx; 909 uint8_t port_id, queue_id; 910 struct lcore_conf *qconf; 911 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 912 US_PER_S * BURST_TX_DRAIN_US; 913 void *ifp; 914 915 prev_tsc = 0; 916 917 lcore_id = rte_lcore_id(); 918 qconf = &lcore_conf; 919 920 if (qconf->nb_rx_queue == 0) { 921 printf("lcore %u has nothing to do\n", lcore_id); 922 return 0; 923 } 924 925 while (1) { 926 cur_tsc = rte_rdtsc(); 927 if (unlikely(freebsd_clock.expire < cur_tsc)) { 928 rte_timer_manage(); 929 } 930 931 /* 932 * TX burst queue drain 933 */ 934 diff_tsc = cur_tsc - prev_tsc; 935 if (unlikely(diff_tsc > drain_tsc)) { 936 /* 937 * This could be optimized (use queueid instead of 938 * portid), but it is not called so often 939 */ 940 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 941 if (qconf->tx_mbufs[port_id].len == 0) 942 continue; 943 send_burst(qconf, 944 qconf->tx_mbufs[port_id].len, 945 port_id); 946 qconf->tx_mbufs[port_id].len = 0; 947 } 948 949 prev_tsc = cur_tsc; 950 } 951 952 /* 953 * Read packet from RX queues 954 */ 955 for (i = 0; i < qconf->nb_rx_queue; ++i) { 956 port_id = qconf->rx_queue_list[i].port_id; 957 queue_id = qconf->rx_queue_list[i].queue_id; 958 ifp = veth_ctx[port_id]->ifp; 959 960 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 961 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 962 } 963 964 process_arp_ring(port_id, queue_id, pkts_burst, ifp); 965 966 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 967 MAX_PKT_BURST); 968 if (nb_rx == 0) 969 continue; 970 971 /* Prefetch first packets */ 972 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 973 rte_prefetch0(rte_pktmbuf_mtod( 974 pkts_burst[j], void *)); 975 } 976 977 /* Prefetch and handle already prefetched packets */ 978 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 979 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 980 j + PREFETCH_OFFSET], void *)); 981 process_packets(port_id, queue_id, &pkts_burst[j], 1, ifp, 0); 982 } 983 984 /* Handle remaining prefetched packets */ 985 for (; j < nb_rx; j++) { 986 process_packets(port_id, queue_id, &pkts_burst[j], 1, ifp, 0); 987 } 988 } 989 990 if (likely(lr->loop != NULL)) { 991 lr->loop(lr->arg); 992 } 993 } 994 } 995 996 int 997 ff_dpdk_if_up(void) { 998 int nb_ports = ff_global_cfg.dpdk.nb_ports; 999 int i; 1000 for (i = 0; i < nb_ports; i++) { 1001 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 1002 veth_ctx[port_id] = ff_veth_attach(ff_global_cfg.dpdk.port_cfgs + i); 1003 if (veth_ctx[port_id] == NULL) { 1004 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1005 } 1006 } 1007 1008 return 0; 1009 } 1010 1011 void 1012 ff_dpdk_run(loop_func_t loop, void *arg) { 1013 struct loop_routine *lr = malloc(sizeof(struct loop_routine)); 1014 lr->loop = loop; 1015 lr->arg = arg; 1016 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1017 rte_eal_mp_wait_lcore(); 1018 free(lr); 1019 } 1020 1021 void 1022 ff_dpdk_pktmbuf_free(void *m) 1023 { 1024 rte_pktmbuf_free((struct rte_mbuf *)m); 1025 } 1026 1027 static uint32_t 1028 toeplitz_hash(unsigned keylen, const uint8_t *key, 1029 unsigned datalen, const uint8_t *data) 1030 { 1031 uint32_t hash = 0, v; 1032 u_int i, b; 1033 1034 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1035 1036 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1037 for (i = 0; i < datalen; i++) { 1038 for (b = 0; b < 8; b++) { 1039 if (data[i] & (1<<(7-b))) 1040 hash ^= v; 1041 v <<= 1; 1042 if ((i + 4) < keylen && 1043 (key[i+4] & (1<<(7-b)))) 1044 v |= 1; 1045 } 1046 } 1047 return (hash); 1048 } 1049 1050 int 1051 ff_rss_check(uint32_t saddr, uint32_t daddr, uint16_t sport, uint16_t dport) 1052 { 1053 struct lcore_conf *qconf = &lcore_conf; 1054 1055 if (qconf->nb_procs == 1) { 1056 return 1; 1057 } 1058 1059 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1060 sizeof(dport)]; 1061 1062 unsigned datalen = 0; 1063 1064 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1065 datalen += sizeof(saddr); 1066 1067 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1068 datalen += sizeof(daddr); 1069 1070 bcopy(&sport, &data[datalen], sizeof(sport)); 1071 datalen += sizeof(sport); 1072 1073 bcopy(&dport, &data[datalen], sizeof(dport)); 1074 datalen += sizeof(dport); 1075 1076 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), default_rsskey_40bytes, datalen, data); 1077 1078 return (hash % qconf->nb_procs) == qconf->proc_id; 1079 } 1080 1081 1082