1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <rte_common.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_memzone.h> 33 #include <rte_config.h> 34 #include <rte_eal.h> 35 #include <rte_pci.h> 36 #include <rte_mbuf.h> 37 #include <rte_memory.h> 38 #include <rte_lcore.h> 39 #include <rte_launch.h> 40 #include <rte_ethdev.h> 41 #include <rte_debug.h> 42 #include <rte_common.h> 43 #include <rte_ether.h> 44 #include <rte_malloc.h> 45 #include <rte_cycles.h> 46 #include <rte_timer.h> 47 #include <rte_thash.h> 48 #include <rte_ip.h> 49 #include <rte_tcp.h> 50 #include <rte_udp.h> 51 52 #include "ff_dpdk_if.h" 53 #include "ff_dpdk_pcap.h" 54 #include "ff_dpdk_kni.h" 55 #include "ff_config.h" 56 #include "ff_veth.h" 57 #include "ff_host_interface.h" 58 #include "ff_msg.h" 59 #include "ff_api.h" 60 61 #define MEMPOOL_CACHE_SIZE 256 62 63 #define ARP_RING_SIZE 2048 64 65 #define MSG_RING_SIZE 32 66 67 /* 68 * Configurable number of RX/TX ring descriptors 69 */ 70 #define RX_QUEUE_SIZE 512 71 #define TX_QUEUE_SIZE 256 72 73 #define MAX_PKT_BURST 32 74 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 75 76 /* 77 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 78 */ 79 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 80 81 #define NB_SOCKETS 8 82 83 /* Configure how many packets ahead to prefetch, when reading packets */ 84 #define PREFETCH_OFFSET 3 85 86 #define MAX_RX_QUEUE_PER_LCORE 16 87 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 88 #define MAX_RX_QUEUE_PER_PORT 128 89 90 #define BITS_PER_HEX 4 91 92 #define KNI_MBUF_MAX 2048 93 #define KNI_QUEUE_SIZE 2048 94 95 static int enable_kni; 96 static int kni_accept; 97 98 static struct rte_timer freebsd_clock; 99 100 // Mellanox Linux's driver key 101 static uint8_t default_rsskey_40bytes[40] = { 102 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 103 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 104 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 105 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 106 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 107 }; 108 109 static struct rte_eth_conf default_port_conf = { 110 .rxmode = { 111 .mq_mode = ETH_MQ_RX_RSS, 112 .max_rx_pkt_len = ETHER_MAX_LEN, 113 .split_hdr_size = 0, /**< hdr buf size */ 114 .header_split = 0, /**< Header Split disabled */ 115 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 116 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 117 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 118 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 119 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 120 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 121 .enable_lro = 0, /**< LRO disabled */ 122 }, 123 .rx_adv_conf = { 124 .rss_conf = { 125 .rss_key = default_rsskey_40bytes, 126 .rss_key_len = 40, 127 .rss_hf = ETH_RSS_PROTO_MASK, 128 }, 129 }, 130 .txmode = { 131 .mq_mode = ETH_MQ_TX_NONE, 132 }, 133 }; 134 135 struct mbuf_table { 136 uint16_t len; 137 struct rte_mbuf *m_table[MAX_PKT_BURST]; 138 }; 139 140 struct lcore_rx_queue { 141 uint8_t port_id; 142 uint8_t queue_id; 143 } __rte_cache_aligned; 144 145 struct lcore_conf { 146 uint16_t proc_id; 147 uint16_t nb_procs; 148 uint16_t socket_id; 149 uint16_t nb_rx_queue; 150 uint16_t *lcore_proc; 151 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 152 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 153 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 154 char *pcap[RTE_MAX_ETHPORTS]; 155 } __rte_cache_aligned; 156 157 static struct lcore_conf lcore_conf; 158 159 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 160 161 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 162 163 struct ff_msg_ring { 164 char ring_name[2][RTE_RING_NAMESIZE]; 165 /* ring[0] for lcore recv msg, other send */ 166 /* ring[1] for lcore send msg, other read */ 167 struct rte_ring *ring[2]; 168 } __rte_cache_aligned; 169 170 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 171 static struct rte_mempool *message_pool; 172 173 struct ff_dpdk_if_context { 174 void *sc; 175 void *ifp; 176 uint16_t port_id; 177 struct ff_hw_features hw_features; 178 } __rte_cache_aligned; 179 180 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 181 182 extern void ff_hardclock(void); 183 184 static void 185 freebsd_hardclock_job(__rte_unused struct rte_timer *timer, 186 __rte_unused void *arg) { 187 ff_hardclock(); 188 } 189 190 struct ff_dpdk_if_context * 191 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 192 { 193 struct ff_dpdk_if_context *ctx; 194 195 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 196 if (ctx == NULL) 197 return NULL; 198 199 ctx->sc = sc; 200 ctx->ifp = ifp; 201 ctx->port_id = cfg->port_id; 202 ctx->hw_features = cfg->hw_features; 203 204 return ctx; 205 } 206 207 void 208 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 209 { 210 free(ctx); 211 } 212 213 static void 214 check_all_ports_link_status(void) 215 { 216 #define CHECK_INTERVAL 100 /* 100ms */ 217 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 218 219 uint8_t portid, count, all_ports_up, print_flag = 0; 220 struct rte_eth_link link; 221 222 printf("\nChecking link status"); 223 fflush(stdout); 224 225 int i, nb_ports; 226 nb_ports = ff_global_cfg.dpdk.nb_ports; 227 for (count = 0; count <= MAX_CHECK_TIME; count++) { 228 all_ports_up = 1; 229 for (i = 0; i < nb_ports; i++) { 230 uint8_t portid = ff_global_cfg.dpdk.port_cfgs[i].port_id; 231 memset(&link, 0, sizeof(link)); 232 rte_eth_link_get_nowait(portid, &link); 233 234 /* print link status if flag set */ 235 if (print_flag == 1) { 236 if (link.link_status) { 237 printf("Port %d Link Up - speed %u " 238 "Mbps - %s\n", (int)portid, 239 (unsigned)link.link_speed, 240 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 241 ("full-duplex") : ("half-duplex\n")); 242 } else { 243 printf("Port %d Link Down\n", (int)portid); 244 } 245 continue; 246 } 247 /* clear all_ports_up flag if any link down */ 248 if (link.link_status == 0) { 249 all_ports_up = 0; 250 break; 251 } 252 } 253 254 /* after finally printing all link status, get out */ 255 if (print_flag == 1) 256 break; 257 258 if (all_ports_up == 0) { 259 printf("."); 260 fflush(stdout); 261 rte_delay_ms(CHECK_INTERVAL); 262 } 263 264 /* set the print_flag if all ports up or timeout */ 265 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 266 print_flag = 1; 267 printf("done\n"); 268 } 269 } 270 } 271 272 static int 273 xdigit2val(unsigned char c) 274 { 275 int val; 276 277 if (isdigit(c)) 278 val = c - '0'; 279 else if (isupper(c)) 280 val = c - 'A' + 10; 281 else 282 val = c - 'a' + 10; 283 return val; 284 } 285 286 static int 287 parse_lcore_mask(const char *coremask, uint16_t *lcore_proc, 288 uint16_t nb_procs) 289 { 290 int i, j, idx = 0; 291 unsigned count = 0; 292 char c; 293 int val; 294 295 if (coremask == NULL) 296 return -1; 297 298 /* Remove all blank characters ahead and after. 299 * Remove 0x/0X if exists. 300 */ 301 while (isblank(*coremask)) 302 coremask++; 303 if (coremask[0] == '0' && ((coremask[1] == 'x') 304 || (coremask[1] == 'X'))) 305 coremask += 2; 306 307 i = strlen(coremask); 308 while ((i > 0) && isblank(coremask[i - 1])) 309 i--; 310 311 if (i == 0) 312 return -1; 313 314 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE && count < nb_procs; i--) { 315 c = coremask[i]; 316 if (isxdigit(c) == 0) { 317 return -1; 318 } 319 val = xdigit2val(c); 320 for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE && count < nb_procs; 321 j++, idx++) { 322 if ((1 << j) & val) { 323 if (!lcore_config[idx].detected) { 324 RTE_LOG(ERR, EAL, "lcore %u unavailable\n", idx); 325 return -1; 326 } 327 lcore_proc[count] = idx; 328 count++; 329 } 330 } 331 } 332 333 for (; i >= 0; i--) 334 if (coremask[i] != '0') 335 return -1; 336 337 if (count < nb_procs) 338 return -1; 339 340 return 0; 341 } 342 343 static int 344 init_lcore_conf(void) 345 { 346 uint8_t nb_ports = rte_eth_dev_count(); 347 if (nb_ports == 0) { 348 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 349 } 350 351 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 352 lcore_conf.nb_procs = ff_global_cfg.dpdk.nb_procs; 353 lcore_conf.lcore_proc = rte_zmalloc(NULL, 354 sizeof(uint16_t)*lcore_conf.nb_procs, 0); 355 if (lcore_conf.lcore_proc == NULL) { 356 rte_exit(EXIT_FAILURE, "rte_zmalloc lcore_proc failed\n"); 357 } 358 359 int ret = parse_lcore_mask(ff_global_cfg.dpdk.lcore_mask, 360 lcore_conf.lcore_proc, lcore_conf.nb_procs); 361 if (ret < 0) { 362 rte_exit(EXIT_FAILURE, "parse_lcore_mask failed:%s\n", 363 ff_global_cfg.dpdk.lcore_mask); 364 } 365 366 uint16_t socket_id = 0; 367 if (ff_global_cfg.dpdk.numa_on) { 368 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 369 } 370 371 lcore_conf.socket_id = socket_id; 372 373 /* Currently, proc id 1:1 map to rx/tx queue id per port. */ 374 uint8_t port_id, enabled_ports = 0; 375 for (port_id = 0; port_id < nb_ports; port_id++) { 376 if (ff_global_cfg.dpdk.port_mask && 377 (ff_global_cfg.dpdk.port_mask & (1 << port_id)) == 0) { 378 printf("\nSkipping disabled port %d\n", port_id); 379 continue; 380 } 381 382 if (port_id >= ff_global_cfg.dpdk.nb_ports) { 383 printf("\nSkipping non-configured port %d\n", port_id); 384 break; 385 } 386 387 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 388 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 389 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = lcore_conf.proc_id; 390 lcore_conf.nb_rx_queue++; 391 392 lcore_conf.tx_queue_id[port_id] = lcore_conf.proc_id; 393 lcore_conf.pcap[port_id] = ff_global_cfg.dpdk.port_cfgs[enabled_ports].pcap; 394 395 ff_global_cfg.dpdk.port_cfgs[enabled_ports].port_id = port_id; 396 397 enabled_ports++; 398 } 399 400 ff_global_cfg.dpdk.nb_ports = enabled_ports; 401 402 return 0; 403 } 404 405 static int 406 init_mem_pool(void) 407 { 408 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 409 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 410 uint32_t nb_tx_queue = nb_lcores; 411 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 412 413 unsigned nb_mbuf = RTE_MAX ( 414 (nb_rx_queue*RX_QUEUE_SIZE + 415 nb_ports*nb_lcores*MAX_PKT_BURST + 416 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 417 nb_lcores*MEMPOOL_CACHE_SIZE + 418 nb_ports*KNI_MBUF_MAX + 419 nb_ports*KNI_QUEUE_SIZE + 420 nb_lcores*nb_ports*ARP_RING_SIZE), 421 (unsigned)8192); 422 423 unsigned socketid = 0; 424 uint16_t i, lcore_id; 425 char s[64]; 426 int numa_on = ff_global_cfg.dpdk.numa_on; 427 428 for (i = 0; i < lcore_conf.nb_procs; i++) { 429 lcore_id = lcore_conf.lcore_proc[i]; 430 if (numa_on) { 431 socketid = rte_lcore_to_socket_id(lcore_id); 432 } 433 434 if (socketid >= NB_SOCKETS) { 435 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 436 socketid, i, NB_SOCKETS); 437 } 438 439 if (pktmbuf_pool[socketid] != NULL) { 440 continue; 441 } 442 443 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 444 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 445 pktmbuf_pool[socketid] = 446 rte_pktmbuf_pool_create(s, nb_mbuf, 447 MEMPOOL_CACHE_SIZE, 0, 448 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 449 } else { 450 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 451 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 452 } 453 454 if (pktmbuf_pool[socketid] == NULL) { 455 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 456 } else { 457 printf("create mbuf pool on socket %d\n", socketid); 458 } 459 } 460 461 return 0; 462 } 463 464 static struct rte_ring * 465 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 466 { 467 struct rte_ring *ring; 468 469 if (name == NULL) 470 return NULL; 471 472 /* If already create, just attached it */ 473 if (likely((ring = rte_ring_lookup(name)) != NULL)) 474 return ring; 475 476 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 477 return rte_ring_create(name, count, socket_id, flags); 478 } else { 479 return rte_ring_lookup(name); 480 } 481 } 482 483 static int 484 init_arp_ring(void) 485 { 486 int i, j, ret; 487 char name_buf[RTE_RING_NAMESIZE]; 488 int nb_procs = ff_global_cfg.dpdk.nb_procs; 489 int proc_id = ff_global_cfg.dpdk.proc_id; 490 491 /* Allocate arp ring ptr according to eth dev count. */ 492 int nb_ports = rte_eth_dev_count(); 493 for(i = 0; i < nb_procs; ++i) { 494 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 495 proc_id, i); 496 497 arp_ring[i] = rte_zmalloc(name_buf, 498 sizeof(struct rte_ring *) * nb_ports, 499 RTE_CACHE_LINE_SIZE); 500 if (arp_ring[i] == NULL) { 501 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 502 "failed\n", name_buf); 503 } 504 } 505 506 unsigned socketid = lcore_conf.socket_id; 507 508 /* Create ring according to ports actually being used. */ 509 nb_ports = ff_global_cfg.dpdk.nb_ports; 510 for (j = 0; j < nb_ports; j++) { 511 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[j].port_id; 512 513 for(i = 0; i < nb_procs; ++i) { 514 snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id); 515 arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE, 516 socketid, RING_F_SC_DEQ); 517 518 if (arp_ring[i][port_id] == NULL) 519 rte_panic("create ring:%s failed!\n", name_buf); 520 521 printf("create ring:%s success, %u ring entries are now free!\n", 522 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 523 } 524 } 525 526 return 0; 527 } 528 529 static void 530 ff_msg_init(struct rte_mempool *mp, 531 __attribute__((unused)) void *opaque_arg, 532 void *obj, __attribute__((unused)) unsigned i) 533 { 534 struct ff_msg *msg = (struct ff_msg *)obj; 535 msg->msg_type = FF_UNKNOWN; 536 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 537 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 538 } 539 540 static int 541 init_msg_ring(void) 542 { 543 uint16_t i; 544 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 545 unsigned socketid = lcore_conf.socket_id; 546 547 /* Create message buffer pool */ 548 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 549 message_pool = rte_mempool_create(FF_MSG_POOL, 550 MSG_RING_SIZE * 2 * nb_procs, 551 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 552 NULL, NULL, ff_msg_init, NULL, 553 socketid, 0); 554 } else { 555 message_pool = rte_mempool_lookup(FF_MSG_POOL); 556 } 557 558 if (message_pool == NULL) { 559 rte_panic("Create msg mempool failed\n"); 560 } 561 562 for(i = 0; i < nb_procs; ++i) { 563 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 564 "%s%u", FF_MSG_RING_IN, i); 565 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 566 "%s%u", FF_MSG_RING_OUT, i); 567 568 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 569 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 570 if (msg_ring[i].ring[0] == NULL) 571 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 572 573 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 574 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 575 if (msg_ring[i].ring[1] == NULL) 576 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 577 } 578 579 return 0; 580 } 581 582 static int 583 init_kni(void) 584 { 585 int nb_ports = rte_eth_dev_count(); 586 kni_accept = 0; 587 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 588 kni_accept = 1; 589 590 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 591 ff_global_cfg.kni.udp_port); 592 593 unsigned socket_id = lcore_conf.socket_id; 594 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 595 596 nb_ports = ff_global_cfg.dpdk.nb_ports; 597 int i, ret; 598 for (i = 0; i < nb_ports; i++) { 599 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 600 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 601 } 602 603 return 0; 604 } 605 606 static int 607 init_port_start(void) 608 { 609 int nb_ports = ff_global_cfg.dpdk.nb_ports; 610 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 611 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 612 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 613 uint16_t i; 614 615 for (i = 0; i < nb_ports; i++) { 616 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 617 618 struct rte_eth_dev_info dev_info; 619 rte_eth_dev_info_get(port_id, &dev_info); 620 621 if (nb_procs > dev_info.max_rx_queues) { 622 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 623 nb_procs, 624 dev_info.max_rx_queues); 625 } 626 627 if (nb_procs > dev_info.max_tx_queues) { 628 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 629 nb_procs, 630 dev_info.max_tx_queues); 631 } 632 633 struct ether_addr addr; 634 rte_eth_macaddr_get(port_id, &addr); 635 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 636 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 637 (unsigned)port_id, 638 addr.addr_bytes[0], addr.addr_bytes[1], 639 addr.addr_bytes[2], addr.addr_bytes[3], 640 addr.addr_bytes[4], addr.addr_bytes[5]); 641 642 rte_memcpy(ff_global_cfg.dpdk.port_cfgs[i].mac, 643 addr.addr_bytes, ETHER_ADDR_LEN); 644 645 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 646 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 647 ETH_TXQ_FLAGS_NOREFCOUNT; 648 649 /* Disable features that are not supported by port's HW */ 650 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 651 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 652 } 653 654 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 655 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 656 } 657 658 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 659 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 660 } 661 662 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 663 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 664 } 665 666 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 667 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 668 } 669 670 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 671 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 672 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 673 } 674 675 struct rte_eth_conf port_conf = {0}; 676 677 /* Set RSS mode */ 678 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 679 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 680 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 681 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 682 683 /* Set Rx VLAN stripping */ 684 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 685 port_conf.rxmode.hw_vlan_strip = 1; 686 } 687 688 /* Enable HW CRC stripping */ 689 port_conf.rxmode.hw_strip_crc = 1; 690 691 /* FIXME: Enable TCP LRO ?*/ 692 #if 0 693 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 694 printf("LRO is supported\n"); 695 port_conf.rxmode.enable_lro = 1; 696 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_lro = 1; 697 } 698 #endif 699 700 /* Set Rx checksum checking */ 701 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 702 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 703 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 704 printf("RX checksum offload supported\n"); 705 port_conf.rxmode.hw_ip_checksum = 1; 706 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_csum = 1; 707 } 708 709 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 710 printf("TX ip checksum offload supported\n"); 711 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_ip = 1; 712 } 713 714 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 715 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 716 printf("TX TCP&UDP checksum offload supported\n"); 717 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_l4 = 1; 718 } 719 720 if (ff_global_cfg.dpdk.tso) { 721 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 722 printf("TSO is supported\n"); 723 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_tso = 1; 724 } 725 } else { 726 printf("TSO is disabled\n"); 727 } 728 729 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 730 continue; 731 } 732 733 /* Currently, proc id 1:1 map to queue id per port. */ 734 int ret = rte_eth_dev_configure(port_id, nb_procs, nb_procs, &port_conf); 735 if (ret != 0) { 736 return ret; 737 } 738 739 uint16_t q; 740 for (q = 0; q < nb_procs; q++) { 741 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 742 socketid, &dev_info.default_txconf); 743 if (ret < 0) { 744 return ret; 745 } 746 747 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 748 socketid, &dev_info.default_rxconf, mbuf_pool); 749 if (ret < 0) { 750 return ret; 751 } 752 } 753 754 ret = rte_eth_dev_start(port_id); 755 if (ret < 0) { 756 return ret; 757 } 758 759 /* Enable RX in promiscuous mode for the Ethernet device. */ 760 if (ff_global_cfg.dpdk.promiscuous) { 761 rte_eth_promiscuous_enable(port_id); 762 ret = rte_eth_promiscuous_get(port_id); 763 if (ret == 1) { 764 printf("set port %u to promiscuous mode ok\n", port_id); 765 } else { 766 printf("set port %u to promiscuous mode error\n", port_id); 767 } 768 } 769 770 /* Enable pcap dump */ 771 if (ff_global_cfg.dpdk.port_cfgs[i].pcap) { 772 ff_enable_pcap(ff_global_cfg.dpdk.port_cfgs[i].pcap); 773 } 774 } 775 776 return 0; 777 } 778 779 static int 780 init_freebsd_clock(void) 781 { 782 rte_timer_subsystem_init(); 783 uint64_t hz = rte_get_timer_hz(); 784 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 785 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 786 787 rte_timer_init(&freebsd_clock); 788 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 789 rte_lcore_id(), &freebsd_hardclock_job, NULL); 790 791 return 0; 792 } 793 794 int 795 ff_dpdk_init(int argc, char **argv) 796 { 797 if (ff_global_cfg.dpdk.nb_procs < 1 || 798 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 799 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 800 ff_global_cfg.dpdk.proc_id < 0) { 801 printf("param num_procs[%d] or proc_id[%d] error!\n", 802 ff_global_cfg.dpdk.nb_procs, 803 ff_global_cfg.dpdk.proc_id); 804 exit(1); 805 } 806 807 int ret = rte_eal_init(argc, argv); 808 if (ret < 0) { 809 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 810 } 811 812 init_lcore_conf(); 813 814 init_mem_pool(); 815 816 init_arp_ring(); 817 818 init_msg_ring(); 819 820 enable_kni = ff_global_cfg.kni.enable; 821 if (enable_kni) { 822 init_kni(); 823 } 824 825 ret = init_port_start(); 826 if (ret < 0) { 827 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 828 } 829 830 check_all_ports_link_status(); 831 832 init_freebsd_clock(); 833 834 return 0; 835 } 836 837 static void 838 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 839 { 840 uint8_t rx_csum = ctx->hw_features.rx_csum; 841 if (rx_csum) { 842 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 843 return; 844 } 845 } 846 847 /* 848 * FIXME: should we save pkt->vlan_tci 849 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 850 */ 851 852 void *data = rte_pktmbuf_mtod(pkt, void*); 853 uint16_t len = rte_pktmbuf_data_len(pkt); 854 855 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 856 if (hdr == NULL) { 857 rte_pktmbuf_free(pkt); 858 return; 859 } 860 861 pkt = pkt->next; 862 void *prev = hdr; 863 while(pkt != NULL) { 864 data = rte_pktmbuf_mtod(pkt, void*); 865 len = rte_pktmbuf_data_len(pkt); 866 867 void *mb = ff_mbuf_get(prev, data, len); 868 if (mb == NULL) { 869 ff_mbuf_free(hdr); 870 return; 871 } 872 pkt = pkt->next; 873 prev = mb; 874 } 875 876 ff_veth_process_packet(ctx->ifp, hdr); 877 } 878 879 static enum FilterReturn 880 protocol_filter(const void *data, uint16_t len) 881 { 882 if(len < sizeof(struct ether_hdr)) 883 return FILTER_UNKNOWN; 884 885 const struct ether_hdr *hdr; 886 hdr = (const struct ether_hdr *)data; 887 888 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 889 return FILTER_ARP; 890 891 if (!enable_kni) { 892 return FILTER_UNKNOWN; 893 } 894 895 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 896 return FILTER_UNKNOWN; 897 898 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 899 len - sizeof(struct ether_hdr)); 900 } 901 902 static inline void 903 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 904 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 905 { 906 struct lcore_conf *qconf = &lcore_conf; 907 908 uint16_t i; 909 for (i = 0; i < count; i++) { 910 struct rte_mbuf *rtem = bufs[i]; 911 912 if (unlikely(qconf->pcap[port_id] != NULL)) { 913 ff_dump_packets(qconf->pcap[port_id], rtem); 914 } 915 916 void *data = rte_pktmbuf_mtod(rtem, void*); 917 uint16_t len = rte_pktmbuf_data_len(rtem); 918 919 enum FilterReturn filter = protocol_filter(data, len); 920 if (filter == FILTER_ARP) { 921 struct rte_mempool *mbuf_pool; 922 struct rte_mbuf *mbuf_clone; 923 if (pkts_from_ring == 0) { 924 uint16_t i; 925 for(i = 0; i < qconf->nb_procs; ++i) { 926 if(i == queue_id) 927 continue; 928 929 mbuf_pool = pktmbuf_pool[rte_lcore_to_socket_id(qconf->lcore_proc[i])]; 930 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 931 if(mbuf_clone) { 932 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 933 if (ret < 0) 934 rte_pktmbuf_free(mbuf_clone); 935 } 936 } 937 } 938 939 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 940 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 941 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 942 if(mbuf_clone) { 943 ff_kni_enqueue(port_id, rtem); 944 } 945 } 946 947 ff_veth_input(ctx, rtem); 948 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 949 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 950 ff_kni_enqueue(port_id, rtem); 951 } else { 952 ff_veth_input(ctx, rtem); 953 } 954 } 955 } 956 957 static inline int 958 process_arp_ring(uint8_t port_id, uint16_t queue_id, 959 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 960 { 961 /* read packet from ring buf and to process */ 962 uint16_t nb_rb; 963 nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 964 (void **)pkts_burst, MAX_PKT_BURST); 965 966 if(nb_rb > 0) { 967 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 968 } 969 970 return 0; 971 } 972 973 static inline void 974 handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id) 975 { 976 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 977 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 978 msg->sysctl.newlen); 979 980 if (ret < 0) { 981 msg->result = errno; 982 } else { 983 msg->result = 0; 984 } 985 986 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 987 } 988 989 static inline void 990 handle_ioctl_msg(struct ff_msg *msg, uint16_t proc_id) 991 { 992 int fd, ret; 993 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 994 if (fd < 0) { 995 ret = -1; 996 goto done; 997 } 998 999 ret = ff_ioctl(fd, msg->ioctl.cmd, msg->ioctl.data); 1000 1001 ff_close(fd); 1002 1003 done: 1004 if (ret < 0) { 1005 msg->result = errno; 1006 } else { 1007 msg->result = 0; 1008 } 1009 1010 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1011 } 1012 1013 static inline void 1014 handle_default_msg(struct ff_msg *msg, uint16_t proc_id) 1015 { 1016 msg->result = EINVAL; 1017 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1018 } 1019 1020 static inline void 1021 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1022 { 1023 switch (msg->msg_type) { 1024 case FF_SYSCTL: 1025 handle_sysctl_msg(msg, proc_id); 1026 break; 1027 case FF_IOCTL: 1028 handle_ioctl_msg(msg, proc_id); 1029 break; 1030 default: 1031 handle_default_msg(msg, proc_id); 1032 break; 1033 } 1034 } 1035 1036 static inline int 1037 process_msg_ring(uint16_t proc_id) 1038 { 1039 void *msg; 1040 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1041 1042 if (unlikely(ret == 0)) { 1043 handle_msg((struct ff_msg *)msg, proc_id); 1044 } 1045 1046 return 0; 1047 } 1048 1049 /* Send burst of packets on an output interface */ 1050 static inline int 1051 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1052 { 1053 struct rte_mbuf **m_table; 1054 int ret; 1055 uint16_t queueid; 1056 1057 queueid = qconf->tx_queue_id[port]; 1058 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1059 1060 if (unlikely(qconf->pcap[port] != NULL)) { 1061 uint16_t i; 1062 for (i = 0; i < n; i++) { 1063 ff_dump_packets(qconf->pcap[port], m_table[i]); 1064 } 1065 } 1066 1067 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1068 if (unlikely(ret < n)) { 1069 do { 1070 rte_pktmbuf_free(m_table[ret]); 1071 } while (++ret < n); 1072 } 1073 1074 return 0; 1075 } 1076 1077 /* Enqueue a single packet, and send burst if queue is filled */ 1078 static inline int 1079 send_single_packet(struct rte_mbuf *m, uint8_t port) 1080 { 1081 uint16_t len; 1082 struct lcore_conf *qconf; 1083 1084 qconf = &lcore_conf; 1085 len = qconf->tx_mbufs[port].len; 1086 qconf->tx_mbufs[port].m_table[len] = m; 1087 len++; 1088 1089 /* enough pkts to be sent */ 1090 if (unlikely(len == MAX_PKT_BURST)) { 1091 send_burst(qconf, MAX_PKT_BURST, port); 1092 len = 0; 1093 } 1094 1095 qconf->tx_mbufs[port].len = len; 1096 return 0; 1097 } 1098 1099 int 1100 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1101 int total) 1102 { 1103 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1104 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1105 if (head == NULL) { 1106 ff_mbuf_free(m); 1107 return -1; 1108 } 1109 1110 head->pkt_len = total; 1111 head->nb_segs = 0; 1112 1113 int off = 0; 1114 struct rte_mbuf *cur = head, *prev = NULL; 1115 while(total > 0) { 1116 if (cur == NULL) { 1117 cur = rte_pktmbuf_alloc(mbuf_pool); 1118 if (cur == NULL) { 1119 rte_pktmbuf_free(head); 1120 ff_mbuf_free(m); 1121 return -1; 1122 } 1123 } 1124 1125 void *data = rte_pktmbuf_mtod(cur, void*); 1126 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1127 int ret = ff_mbuf_copydata(m, data, off, len); 1128 if (ret < 0) { 1129 rte_pktmbuf_free(head); 1130 ff_mbuf_free(m); 1131 return -1; 1132 } 1133 1134 if (prev != NULL) { 1135 prev->next = cur; 1136 } 1137 prev = cur; 1138 1139 cur->data_len = len; 1140 off += len; 1141 total -= len; 1142 head->nb_segs++; 1143 cur = NULL; 1144 } 1145 1146 struct ff_tx_offload offload = {0}; 1147 ff_mbuf_tx_offload(m, &offload); 1148 1149 if (offload.ip_csum) { 1150 head->ol_flags |= PKT_TX_IP_CKSUM; 1151 head->l2_len = sizeof(struct ether_hdr); 1152 head->l3_len = sizeof(struct ipv4_hdr); 1153 } 1154 1155 if (ctx->hw_features.tx_csum_l4) { 1156 if (offload.tcp_csum) { 1157 head->ol_flags |= PKT_TX_TCP_CKSUM; 1158 head->l2_len = sizeof(struct ether_hdr); 1159 head->l3_len = sizeof(struct ipv4_hdr); 1160 } 1161 1162 if (offload.tso_seg_size) { 1163 head->ol_flags |= PKT_TX_TCP_SEG; 1164 head->l4_len = sizeof(struct tcp_hdr); 1165 head->tso_segsz = offload.tso_seg_size; 1166 } 1167 1168 if (offload.udp_csum) { 1169 head->ol_flags |= PKT_TX_UDP_CKSUM; 1170 head->l2_len = sizeof(struct ether_hdr); 1171 head->l3_len = sizeof(struct ipv4_hdr); 1172 } 1173 } 1174 1175 ff_mbuf_free(m); 1176 1177 return send_single_packet(head, ctx->port_id); 1178 } 1179 1180 static int 1181 main_loop(void *arg) 1182 { 1183 struct loop_routine *lr = (struct loop_routine *)arg; 1184 1185 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1186 unsigned lcore_id; 1187 uint64_t prev_tsc, diff_tsc, cur_tsc; 1188 int i, j, nb_rx; 1189 uint8_t port_id, queue_id; 1190 struct lcore_conf *qconf; 1191 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1192 US_PER_S * BURST_TX_DRAIN_US; 1193 struct ff_dpdk_if_context *ctx; 1194 1195 prev_tsc = 0; 1196 1197 lcore_id = rte_lcore_id(); 1198 qconf = &lcore_conf; 1199 1200 if (qconf->nb_rx_queue == 0) { 1201 printf("lcore %u has nothing to do\n", lcore_id); 1202 return 0; 1203 } 1204 1205 while (1) { 1206 cur_tsc = rte_rdtsc(); 1207 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1208 rte_timer_manage(); 1209 } 1210 1211 /* 1212 * TX burst queue drain 1213 */ 1214 diff_tsc = cur_tsc - prev_tsc; 1215 if (unlikely(diff_tsc > drain_tsc)) { 1216 /* 1217 * This could be optimized (use queueid instead of 1218 * portid), but it is not called so often 1219 */ 1220 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 1221 if (qconf->tx_mbufs[port_id].len == 0) 1222 continue; 1223 send_burst(qconf, 1224 qconf->tx_mbufs[port_id].len, 1225 port_id); 1226 qconf->tx_mbufs[port_id].len = 0; 1227 } 1228 1229 prev_tsc = cur_tsc; 1230 } 1231 1232 /* 1233 * Read packet from RX queues 1234 */ 1235 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1236 port_id = qconf->rx_queue_list[i].port_id; 1237 queue_id = qconf->rx_queue_list[i].queue_id; 1238 ctx = veth_ctx[port_id]; 1239 1240 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1241 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1242 } 1243 1244 process_arp_ring(port_id, queue_id, pkts_burst, ctx); 1245 1246 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1247 MAX_PKT_BURST); 1248 if (nb_rx == 0) 1249 continue; 1250 1251 /* Prefetch first packets */ 1252 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1253 rte_prefetch0(rte_pktmbuf_mtod( 1254 pkts_burst[j], void *)); 1255 } 1256 1257 /* Prefetch and handle already prefetched packets */ 1258 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1259 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1260 j + PREFETCH_OFFSET], void *)); 1261 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1262 } 1263 1264 /* Handle remaining prefetched packets */ 1265 for (; j < nb_rx; j++) { 1266 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1267 } 1268 } 1269 1270 process_msg_ring(qconf->proc_id); 1271 1272 if (likely(lr->loop != NULL)) { 1273 lr->loop(lr->arg); 1274 } 1275 } 1276 } 1277 1278 int 1279 ff_dpdk_if_up(void) { 1280 int nb_ports = ff_global_cfg.dpdk.nb_ports; 1281 int i; 1282 for (i = 0; i < nb_ports; i++) { 1283 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 1284 veth_ctx[port_id] = ff_veth_attach(ff_global_cfg.dpdk.port_cfgs + i); 1285 if (veth_ctx[port_id] == NULL) { 1286 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1287 } 1288 } 1289 1290 return 0; 1291 } 1292 1293 void 1294 ff_dpdk_run(loop_func_t loop, void *arg) { 1295 struct loop_routine *lr = malloc(sizeof(struct loop_routine)); 1296 lr->loop = loop; 1297 lr->arg = arg; 1298 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1299 rte_eal_mp_wait_lcore(); 1300 free(lr); 1301 } 1302 1303 void 1304 ff_dpdk_pktmbuf_free(void *m) 1305 { 1306 rte_pktmbuf_free((struct rte_mbuf *)m); 1307 } 1308 1309 static uint32_t 1310 toeplitz_hash(unsigned keylen, const uint8_t *key, 1311 unsigned datalen, const uint8_t *data) 1312 { 1313 uint32_t hash = 0, v; 1314 u_int i, b; 1315 1316 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1317 1318 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1319 for (i = 0; i < datalen; i++) { 1320 for (b = 0; b < 8; b++) { 1321 if (data[i] & (1<<(7-b))) 1322 hash ^= v; 1323 v <<= 1; 1324 if ((i + 4) < keylen && 1325 (key[i+4] & (1<<(7-b)))) 1326 v |= 1; 1327 } 1328 } 1329 return (hash); 1330 } 1331 1332 int 1333 ff_rss_check(uint32_t saddr, uint32_t daddr, uint16_t sport, uint16_t dport) 1334 { 1335 struct lcore_conf *qconf = &lcore_conf; 1336 1337 if (qconf->nb_procs == 1) { 1338 return 1; 1339 } 1340 1341 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1342 sizeof(dport)]; 1343 1344 unsigned datalen = 0; 1345 1346 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1347 datalen += sizeof(saddr); 1348 1349 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1350 datalen += sizeof(daddr); 1351 1352 bcopy(&sport, &data[datalen], sizeof(sport)); 1353 datalen += sizeof(sport); 1354 1355 bcopy(&dport, &data[datalen], sizeof(dport)); 1356 datalen += sizeof(dport); 1357 1358 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), default_rsskey_40bytes, datalen, data); 1359 1360 return (hash % qconf->nb_procs) == qconf->proc_id; 1361 } 1362 1363 1364