1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <rte_common.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_memzone.h> 33 #include <rte_config.h> 34 #include <rte_eal.h> 35 #include <rte_pci.h> 36 #include <rte_mbuf.h> 37 #include <rte_memory.h> 38 #include <rte_lcore.h> 39 #include <rte_launch.h> 40 #include <rte_ethdev.h> 41 #include <rte_debug.h> 42 #include <rte_common.h> 43 #include <rte_ether.h> 44 #include <rte_malloc.h> 45 #include <rte_cycles.h> 46 #include <rte_timer.h> 47 #include <rte_thash.h> 48 #include <rte_ip.h> 49 #include <rte_tcp.h> 50 #include <rte_udp.h> 51 52 #include "ff_dpdk_if.h" 53 #include "ff_dpdk_pcap.h" 54 #include "ff_dpdk_kni.h" 55 #include "ff_config.h" 56 #include "ff_veth.h" 57 #include "ff_host_interface.h" 58 #include "ff_msg.h" 59 #include "ff_api.h" 60 61 #define MEMPOOL_CACHE_SIZE 256 62 63 #define ARP_RING_SIZE 2048 64 65 #define MSG_RING_SIZE 32 66 67 /* 68 * Configurable number of RX/TX ring descriptors 69 */ 70 #define RX_QUEUE_SIZE 512 71 #define TX_QUEUE_SIZE 256 72 73 #define MAX_PKT_BURST 32 74 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 75 76 /* 77 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 78 */ 79 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 80 81 #define NB_SOCKETS 8 82 83 /* Configure how many packets ahead to prefetch, when reading packets */ 84 #define PREFETCH_OFFSET 3 85 86 #define MAX_RX_QUEUE_PER_LCORE 16 87 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 88 #define MAX_RX_QUEUE_PER_PORT 128 89 90 #define BITS_PER_HEX 4 91 92 #define KNI_MBUF_MAX 2048 93 #define KNI_QUEUE_SIZE 2048 94 95 static int enable_kni; 96 static int kni_accept; 97 98 static struct rte_timer freebsd_clock; 99 100 // Mellanox Linux's driver key 101 static uint8_t default_rsskey_40bytes[40] = { 102 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 103 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 104 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 105 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 106 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 107 }; 108 109 static struct rte_eth_conf default_port_conf = { 110 .rxmode = { 111 .mq_mode = ETH_MQ_RX_RSS, 112 .max_rx_pkt_len = ETHER_MAX_LEN, 113 .split_hdr_size = 0, /**< hdr buf size */ 114 .header_split = 0, /**< Header Split disabled */ 115 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 116 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 117 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 118 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 119 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 120 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 121 .enable_lro = 0, /**< LRO disabled */ 122 }, 123 .rx_adv_conf = { 124 .rss_conf = { 125 .rss_key = default_rsskey_40bytes, 126 .rss_key_len = 40, 127 .rss_hf = ETH_RSS_PROTO_MASK, 128 }, 129 }, 130 .txmode = { 131 .mq_mode = ETH_MQ_TX_NONE, 132 }, 133 }; 134 135 struct mbuf_table { 136 uint16_t len; 137 struct rte_mbuf *m_table[MAX_PKT_BURST]; 138 }; 139 140 struct lcore_rx_queue { 141 uint8_t port_id; 142 uint8_t queue_id; 143 } __rte_cache_aligned; 144 145 struct lcore_conf { 146 uint16_t proc_id; 147 uint16_t nb_procs; 148 uint16_t socket_id; 149 uint16_t nb_rx_queue; 150 uint16_t *lcore_proc; 151 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 152 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 153 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 154 char *pcap[RTE_MAX_ETHPORTS]; 155 } __rte_cache_aligned; 156 157 static struct lcore_conf lcore_conf; 158 159 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 160 161 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 162 163 struct ff_msg_ring { 164 char ring_name[2][RTE_RING_NAMESIZE]; 165 /* ring[0] for lcore recv msg, other send */ 166 /* ring[1] for lcore send msg, other read */ 167 struct rte_ring *ring[2]; 168 } __rte_cache_aligned; 169 170 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 171 static struct rte_mempool *message_pool; 172 173 struct ff_dpdk_if_context { 174 void *sc; 175 void *ifp; 176 uint16_t port_id; 177 struct ff_hw_features hw_features; 178 } __rte_cache_aligned; 179 180 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 181 182 extern void ff_hardclock(void); 183 184 static void 185 freebsd_hardclock_job(__rte_unused struct rte_timer *timer, 186 __rte_unused void *arg) { 187 ff_hardclock(); 188 } 189 190 struct ff_dpdk_if_context * 191 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 192 { 193 struct ff_dpdk_if_context *ctx; 194 195 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 196 if (ctx == NULL) 197 return NULL; 198 199 ctx->sc = sc; 200 ctx->ifp = ifp; 201 ctx->port_id = cfg->port_id; 202 ctx->hw_features = cfg->hw_features; 203 204 return ctx; 205 } 206 207 void 208 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 209 { 210 free(ctx); 211 } 212 213 static void 214 check_all_ports_link_status(void) 215 { 216 #define CHECK_INTERVAL 100 /* 100ms */ 217 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 218 219 uint8_t portid, count, all_ports_up, print_flag = 0; 220 struct rte_eth_link link; 221 222 printf("\nChecking link status"); 223 fflush(stdout); 224 225 int i, nb_ports; 226 nb_ports = ff_global_cfg.dpdk.nb_ports; 227 for (count = 0; count <= MAX_CHECK_TIME; count++) { 228 all_ports_up = 1; 229 for (i = 0; i < nb_ports; i++) { 230 uint8_t portid = ff_global_cfg.dpdk.port_cfgs[i].port_id; 231 memset(&link, 0, sizeof(link)); 232 rte_eth_link_get_nowait(portid, &link); 233 234 /* print link status if flag set */ 235 if (print_flag == 1) { 236 if (link.link_status) { 237 printf("Port %d Link Up - speed %u " 238 "Mbps - %s\n", (int)portid, 239 (unsigned)link.link_speed, 240 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 241 ("full-duplex") : ("half-duplex\n")); 242 } else { 243 printf("Port %d Link Down\n", (int)portid); 244 } 245 continue; 246 } 247 /* clear all_ports_up flag if any link down */ 248 if (link.link_status == 0) { 249 all_ports_up = 0; 250 break; 251 } 252 } 253 254 /* after finally printing all link status, get out */ 255 if (print_flag == 1) 256 break; 257 258 if (all_ports_up == 0) { 259 printf("."); 260 fflush(stdout); 261 rte_delay_ms(CHECK_INTERVAL); 262 } 263 264 /* set the print_flag if all ports up or timeout */ 265 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 266 print_flag = 1; 267 printf("done\n"); 268 } 269 } 270 } 271 272 static int 273 xdigit2val(unsigned char c) 274 { 275 int val; 276 277 if (isdigit(c)) 278 val = c - '0'; 279 else if (isupper(c)) 280 val = c - 'A' + 10; 281 else 282 val = c - 'a' + 10; 283 return val; 284 } 285 286 static int 287 parse_lcore_mask(const char *coremask, uint16_t *lcore_proc, 288 uint16_t nb_procs) 289 { 290 int i, j, idx = 0; 291 unsigned count = 0; 292 char c; 293 int val; 294 295 if (coremask == NULL) 296 return -1; 297 298 /* Remove all blank characters ahead and after. 299 * Remove 0x/0X if exists. 300 */ 301 while (isblank(*coremask)) 302 coremask++; 303 if (coremask[0] == '0' && ((coremask[1] == 'x') 304 || (coremask[1] == 'X'))) 305 coremask += 2; 306 307 i = strlen(coremask); 308 while ((i > 0) && isblank(coremask[i - 1])) 309 i--; 310 311 if (i == 0) 312 return -1; 313 314 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE && count < nb_procs; i--) { 315 c = coremask[i]; 316 if (isxdigit(c) == 0) { 317 return -1; 318 } 319 val = xdigit2val(c); 320 for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE && count < nb_procs; 321 j++, idx++) { 322 if ((1 << j) & val) { 323 if (!lcore_config[idx].detected) { 324 RTE_LOG(ERR, EAL, "lcore %u unavailable\n", idx); 325 return -1; 326 } 327 lcore_proc[count] = idx; 328 count++; 329 } 330 } 331 } 332 333 for (; i >= 0; i--) 334 if (coremask[i] != '0') 335 return -1; 336 337 if (count < nb_procs) 338 return -1; 339 340 return 0; 341 } 342 343 static int 344 init_lcore_conf(void) 345 { 346 uint8_t nb_ports = rte_eth_dev_count(); 347 if (nb_ports == 0) { 348 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 349 } 350 351 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 352 lcore_conf.nb_procs = ff_global_cfg.dpdk.nb_procs; 353 lcore_conf.lcore_proc = rte_zmalloc(NULL, 354 sizeof(uint16_t)*lcore_conf.nb_procs, 0); 355 if (lcore_conf.lcore_proc == NULL) { 356 rte_exit(EXIT_FAILURE, "rte_zmalloc lcore_proc failed\n"); 357 } 358 359 int ret = parse_lcore_mask(ff_global_cfg.dpdk.lcore_mask, 360 lcore_conf.lcore_proc, lcore_conf.nb_procs); 361 if (ret < 0) { 362 rte_exit(EXIT_FAILURE, "parse_lcore_mask failed:%s\n", 363 ff_global_cfg.dpdk.lcore_mask); 364 } 365 366 uint16_t socket_id = 0; 367 if (ff_global_cfg.dpdk.numa_on) { 368 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 369 } 370 371 lcore_conf.socket_id = socket_id; 372 373 /* Currently, proc id 1:1 map to rx/tx queue id per port. */ 374 uint8_t port_id, enabled_ports = 0; 375 for (port_id = 0; port_id < nb_ports; port_id++) { 376 if (ff_global_cfg.dpdk.port_mask && 377 (ff_global_cfg.dpdk.port_mask & (1 << port_id)) == 0) { 378 printf("\nSkipping disabled port %d\n", port_id); 379 continue; 380 } 381 382 if (port_id >= ff_global_cfg.dpdk.nb_ports) { 383 printf("\nSkipping non-configured port %d\n", port_id); 384 break; 385 } 386 387 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 388 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 389 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = lcore_conf.proc_id; 390 lcore_conf.nb_rx_queue++; 391 392 lcore_conf.tx_queue_id[port_id] = lcore_conf.proc_id; 393 lcore_conf.pcap[port_id] = ff_global_cfg.dpdk.port_cfgs[enabled_ports].pcap; 394 395 ff_global_cfg.dpdk.port_cfgs[enabled_ports].port_id = port_id; 396 397 enabled_ports++; 398 } 399 400 ff_global_cfg.dpdk.nb_ports = enabled_ports; 401 402 return 0; 403 } 404 405 static int 406 init_mem_pool(void) 407 { 408 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 409 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 410 uint32_t nb_tx_queue = nb_lcores; 411 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 412 413 unsigned nb_mbuf = RTE_MAX ( 414 (nb_rx_queue*RX_QUEUE_SIZE + 415 nb_ports*nb_lcores*MAX_PKT_BURST + 416 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 417 nb_lcores*MEMPOOL_CACHE_SIZE + 418 nb_ports*KNI_MBUF_MAX + 419 nb_ports*KNI_QUEUE_SIZE + 420 nb_lcores*nb_ports*ARP_RING_SIZE), 421 (unsigned)8192); 422 423 unsigned socketid = 0; 424 uint16_t i, lcore_id; 425 char s[64]; 426 int numa_on = ff_global_cfg.dpdk.numa_on; 427 428 for (i = 0; i < lcore_conf.nb_procs; i++) { 429 lcore_id = lcore_conf.lcore_proc[i]; 430 if (numa_on) { 431 socketid = rte_lcore_to_socket_id(lcore_id); 432 } 433 434 if (socketid >= NB_SOCKETS) { 435 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 436 socketid, i, NB_SOCKETS); 437 } 438 439 if (pktmbuf_pool[socketid] != NULL) { 440 continue; 441 } 442 443 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 444 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 445 pktmbuf_pool[socketid] = 446 rte_pktmbuf_pool_create(s, nb_mbuf, 447 MEMPOOL_CACHE_SIZE, 0, 448 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 449 } else { 450 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 451 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 452 } 453 454 if (pktmbuf_pool[socketid] == NULL) { 455 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 456 } else { 457 printf("create mbuf pool on socket %d\n", socketid); 458 } 459 } 460 461 return 0; 462 } 463 464 static struct rte_ring * 465 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 466 { 467 struct rte_ring *ring; 468 469 if (name == NULL) 470 return NULL; 471 472 /* If already create, just attached it */ 473 if (likely((ring = rte_ring_lookup(name)) != NULL)) 474 return ring; 475 476 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 477 return rte_ring_create(name, count, socket_id, flags); 478 } else { 479 return rte_ring_lookup(name); 480 } 481 } 482 483 static int 484 init_arp_ring(void) 485 { 486 int i, j, ret; 487 char name_buf[RTE_RING_NAMESIZE]; 488 int nb_procs = ff_global_cfg.dpdk.nb_procs; 489 int proc_id = ff_global_cfg.dpdk.proc_id; 490 491 /* Allocate arp ring ptr according to eth dev count. */ 492 int nb_ports = rte_eth_dev_count(); 493 for(i = 0; i < nb_procs; ++i) { 494 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 495 proc_id, i); 496 497 arp_ring[i] = rte_zmalloc(name_buf, 498 sizeof(struct rte_ring *) * nb_ports, 499 RTE_CACHE_LINE_SIZE); 500 if (arp_ring[i] == NULL) { 501 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 502 "failed\n", name_buf); 503 } 504 } 505 506 unsigned socketid = lcore_conf.socket_id; 507 508 /* Create ring according to ports actually being used. */ 509 nb_ports = ff_global_cfg.dpdk.nb_ports; 510 for (j = 0; j < nb_ports; j++) { 511 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[j].port_id; 512 513 for(i = 0; i < nb_procs; ++i) { 514 snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id); 515 arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE, 516 socketid, RING_F_SC_DEQ); 517 518 if (arp_ring[i][port_id] == NULL) 519 rte_panic("create ring:%s failed!\n", name_buf); 520 521 printf("create ring:%s success, %u ring entries are now free!\n", 522 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 523 } 524 } 525 526 return 0; 527 } 528 529 static void 530 ff_msg_init(struct rte_mempool *mp, 531 __attribute__((unused)) void *opaque_arg, 532 void *obj, __attribute__((unused)) unsigned i) 533 { 534 struct ff_msg *msg = (struct ff_msg *)obj; 535 msg->msg_type = FF_UNKNOWN; 536 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 537 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 538 } 539 540 static int 541 init_msg_ring(void) 542 { 543 uint16_t i; 544 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 545 unsigned socketid = lcore_conf.socket_id; 546 547 /* Create message buffer pool */ 548 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 549 message_pool = rte_mempool_create(FF_MSG_POOL, 550 MSG_RING_SIZE * 2 * nb_procs, 551 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 552 NULL, NULL, ff_msg_init, NULL, 553 socketid, 0); 554 } else { 555 message_pool = rte_mempool_lookup(FF_MSG_POOL); 556 } 557 558 if (message_pool == NULL) { 559 rte_panic("Create msg mempool failed\n"); 560 } 561 562 for(i = 0; i < nb_procs; ++i) { 563 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 564 "%s%u", FF_MSG_RING_IN, i); 565 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 566 "%s%u", FF_MSG_RING_OUT, i); 567 568 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 569 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 570 if (msg_ring[i].ring[0] == NULL) 571 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 572 573 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 574 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 575 if (msg_ring[i].ring[1] == NULL) 576 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 577 } 578 579 return 0; 580 } 581 582 static int 583 init_kni(void) 584 { 585 int nb_ports = rte_eth_dev_count(); 586 kni_accept = 0; 587 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 588 kni_accept = 1; 589 590 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 591 ff_global_cfg.kni.udp_port); 592 593 unsigned socket_id = lcore_conf.socket_id; 594 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 595 596 nb_ports = ff_global_cfg.dpdk.nb_ports; 597 int i, ret; 598 for (i = 0; i < nb_ports; i++) { 599 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 600 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 601 } 602 603 return 0; 604 } 605 606 static int 607 init_port_start(void) 608 { 609 int nb_ports = ff_global_cfg.dpdk.nb_ports; 610 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 611 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 612 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 613 uint16_t i; 614 615 for (i = 0; i < nb_ports; i++) { 616 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 617 618 struct rte_eth_dev_info dev_info; 619 rte_eth_dev_info_get(port_id, &dev_info); 620 621 if (nb_procs > dev_info.max_rx_queues) { 622 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 623 nb_procs, 624 dev_info.max_rx_queues); 625 } 626 627 if (nb_procs > dev_info.max_tx_queues) { 628 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 629 nb_procs, 630 dev_info.max_tx_queues); 631 } 632 633 struct ether_addr addr; 634 rte_eth_macaddr_get(port_id, &addr); 635 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 636 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 637 (unsigned)port_id, 638 addr.addr_bytes[0], addr.addr_bytes[1], 639 addr.addr_bytes[2], addr.addr_bytes[3], 640 addr.addr_bytes[4], addr.addr_bytes[5]); 641 642 rte_memcpy(ff_global_cfg.dpdk.port_cfgs[i].mac, 643 addr.addr_bytes, ETHER_ADDR_LEN); 644 645 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 646 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 647 ETH_TXQ_FLAGS_NOREFCOUNT; 648 649 /* Disable features that are not supported by port's HW */ 650 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 651 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 652 } 653 654 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 655 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 656 } 657 658 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 659 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 660 } 661 662 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 663 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 664 } 665 666 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 667 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 668 } 669 670 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 671 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 672 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 673 } 674 675 struct rte_eth_conf port_conf = {0}; 676 677 /* Set RSS mode */ 678 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 679 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 680 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 681 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 682 683 /* Set Rx VLAN stripping */ 684 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 685 port_conf.rxmode.hw_vlan_strip = 1; 686 } 687 688 /* Enable HW CRC stripping */ 689 port_conf.rxmode.hw_strip_crc = 1; 690 691 /* FIXME: Enable TCP LRO ?*/ 692 #if 0 693 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 694 printf("LRO is supported\n"); 695 port_conf.rxmode.enable_lro = 1; 696 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_lro = 1; 697 } 698 #endif 699 700 /* Set Rx checksum checking */ 701 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 702 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 703 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 704 printf("RX checksum offload supported\n"); 705 port_conf.rxmode.hw_ip_checksum = 1; 706 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_csum = 1; 707 } 708 709 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 710 printf("TX ip checksum offload supported\n"); 711 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_ip = 1; 712 } 713 714 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 715 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 716 printf("TX TCP&UDP checksum offload supported\n"); 717 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_l4 = 1; 718 } 719 720 if (ff_global_cfg.dpdk.tso) { 721 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 722 printf("TSO is supported\n"); 723 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_tso = 1; 724 } 725 } else { 726 printf("TSO is disabled\n"); 727 } 728 729 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 730 continue; 731 } 732 733 /* Currently, proc id 1:1 map to queue id per port. */ 734 int ret = rte_eth_dev_configure(port_id, nb_procs, nb_procs, &port_conf); 735 if (ret != 0) { 736 return ret; 737 } 738 739 uint16_t q; 740 for (q = 0; q < nb_procs; q++) { 741 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 742 socketid, &dev_info.default_txconf); 743 if (ret < 0) { 744 return ret; 745 } 746 747 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 748 socketid, &dev_info.default_rxconf, mbuf_pool); 749 if (ret < 0) { 750 return ret; 751 } 752 } 753 754 ret = rte_eth_dev_start(port_id); 755 if (ret < 0) { 756 return ret; 757 } 758 759 /* Enable RX in promiscuous mode for the Ethernet device. */ 760 if (ff_global_cfg.dpdk.promiscuous) { 761 rte_eth_promiscuous_enable(port_id); 762 ret = rte_eth_promiscuous_get(port_id); 763 if (ret == 1) { 764 printf("set port %u to promiscuous mode ok\n", port_id); 765 } else { 766 printf("set port %u to promiscuous mode error\n", port_id); 767 } 768 } 769 770 /* Enable pcap dump */ 771 if (ff_global_cfg.dpdk.port_cfgs[i].pcap) { 772 ff_enable_pcap(ff_global_cfg.dpdk.port_cfgs[i].pcap); 773 } 774 } 775 776 return 0; 777 } 778 779 static int 780 init_freebsd_clock(void) 781 { 782 rte_timer_subsystem_init(); 783 uint64_t hz = rte_get_timer_hz(); 784 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 785 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 786 787 rte_timer_init(&freebsd_clock); 788 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 789 rte_lcore_id(), &freebsd_hardclock_job, NULL); 790 791 return 0; 792 } 793 794 int 795 ff_dpdk_init(int argc, char **argv) 796 { 797 if (ff_global_cfg.dpdk.nb_procs < 1 || 798 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 799 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 800 ff_global_cfg.dpdk.proc_id < 0) { 801 printf("param num_procs[%d] or proc_id[%d] error!\n", 802 ff_global_cfg.dpdk.nb_procs, 803 ff_global_cfg.dpdk.proc_id); 804 exit(1); 805 } 806 807 int ret = rte_eal_init(argc, argv); 808 if (ret < 0) { 809 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 810 } 811 812 init_lcore_conf(); 813 814 init_mem_pool(); 815 816 init_arp_ring(); 817 818 init_msg_ring(); 819 820 enable_kni = ff_global_cfg.kni.enable; 821 if (enable_kni) { 822 init_kni(); 823 } 824 825 ret = init_port_start(); 826 if (ret < 0) { 827 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 828 } 829 830 check_all_ports_link_status(); 831 832 init_freebsd_clock(); 833 834 return 0; 835 } 836 837 static void 838 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 839 { 840 uint8_t rx_csum = ctx->hw_features.rx_csum; 841 if (rx_csum) { 842 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 843 return; 844 } 845 } 846 847 /* 848 * FIXME: should we save pkt->vlan_tci 849 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 850 */ 851 852 void *data = rte_pktmbuf_mtod(pkt, void*); 853 uint16_t len = rte_pktmbuf_data_len(pkt); 854 855 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 856 if (hdr == NULL) { 857 rte_pktmbuf_free(pkt); 858 return; 859 } 860 861 struct rte_mbuf *pn = pkt->next; 862 void *prev = hdr; 863 while(pn != NULL) { 864 data = rte_pktmbuf_mtod(pkt, void*); 865 len = rte_pktmbuf_data_len(pkt); 866 867 void *mb = ff_mbuf_get(prev, data, len); 868 if (mb == NULL) { 869 ff_mbuf_free(hdr); 870 rte_pktmbuf_free(pkt); 871 return; 872 } 873 pn = pn->next; 874 prev = mb; 875 } 876 877 ff_veth_process_packet(ctx->ifp, hdr); 878 } 879 880 static enum FilterReturn 881 protocol_filter(const void *data, uint16_t len) 882 { 883 if(len < sizeof(struct ether_hdr)) 884 return FILTER_UNKNOWN; 885 886 const struct ether_hdr *hdr; 887 hdr = (const struct ether_hdr *)data; 888 889 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 890 return FILTER_ARP; 891 892 if (!enable_kni) { 893 return FILTER_UNKNOWN; 894 } 895 896 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 897 return FILTER_UNKNOWN; 898 899 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 900 len - sizeof(struct ether_hdr)); 901 } 902 903 static inline void 904 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 905 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 906 { 907 struct lcore_conf *qconf = &lcore_conf; 908 909 uint16_t i; 910 for (i = 0; i < count; i++) { 911 struct rte_mbuf *rtem = bufs[i]; 912 913 if (unlikely(qconf->pcap[port_id] != NULL)) { 914 ff_dump_packets(qconf->pcap[port_id], rtem); 915 } 916 917 void *data = rte_pktmbuf_mtod(rtem, void*); 918 uint16_t len = rte_pktmbuf_data_len(rtem); 919 920 enum FilterReturn filter = protocol_filter(data, len); 921 if (filter == FILTER_ARP) { 922 struct rte_mempool *mbuf_pool; 923 struct rte_mbuf *mbuf_clone; 924 if (pkts_from_ring == 0) { 925 uint16_t i; 926 for(i = 0; i < qconf->nb_procs; ++i) { 927 if(i == queue_id) 928 continue; 929 930 mbuf_pool = pktmbuf_pool[rte_lcore_to_socket_id(qconf->lcore_proc[i])]; 931 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 932 if(mbuf_clone) { 933 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 934 if (ret < 0) 935 rte_pktmbuf_free(mbuf_clone); 936 } 937 } 938 } 939 940 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 941 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 942 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 943 if(mbuf_clone) { 944 ff_kni_enqueue(port_id, mbuf_clone); 945 } 946 } 947 948 ff_veth_input(ctx, rtem); 949 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 950 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 951 ff_kni_enqueue(port_id, rtem); 952 } else { 953 ff_veth_input(ctx, rtem); 954 } 955 } 956 } 957 958 static inline int 959 process_arp_ring(uint8_t port_id, uint16_t queue_id, 960 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 961 { 962 /* read packet from ring buf and to process */ 963 uint16_t nb_rb; 964 nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 965 (void **)pkts_burst, MAX_PKT_BURST); 966 967 if(nb_rb > 0) { 968 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 969 } 970 971 return 0; 972 } 973 974 static inline void 975 handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id) 976 { 977 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 978 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 979 msg->sysctl.newlen); 980 981 if (ret < 0) { 982 msg->result = errno; 983 } else { 984 msg->result = 0; 985 } 986 987 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 988 } 989 990 static inline void 991 handle_ioctl_msg(struct ff_msg *msg, uint16_t proc_id) 992 { 993 int fd, ret; 994 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 995 if (fd < 0) { 996 ret = -1; 997 goto done; 998 } 999 1000 ret = ff_ioctl(fd, msg->ioctl.cmd, msg->ioctl.data); 1001 1002 ff_close(fd); 1003 1004 done: 1005 if (ret < 0) { 1006 msg->result = errno; 1007 } else { 1008 msg->result = 0; 1009 } 1010 1011 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1012 } 1013 1014 static inline void 1015 handle_route_msg(struct ff_msg *msg, uint16_t proc_id) 1016 { 1017 msg->result = ff_rtioctl(msg->route.fib, msg->route.data, 1018 &msg->route.len, msg->route.maxlen); 1019 1020 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1021 } 1022 1023 static inline void 1024 handle_default_msg(struct ff_msg *msg, uint16_t proc_id) 1025 { 1026 msg->result = EINVAL; 1027 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1028 } 1029 1030 static inline void 1031 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1032 { 1033 switch (msg->msg_type) { 1034 case FF_SYSCTL: 1035 handle_sysctl_msg(msg, proc_id); 1036 break; 1037 case FF_IOCTL: 1038 handle_ioctl_msg(msg, proc_id); 1039 break; 1040 case FF_ROUTE: 1041 handle_route_msg(msg, proc_id); 1042 break; 1043 default: 1044 handle_default_msg(msg, proc_id); 1045 break; 1046 } 1047 } 1048 1049 static inline int 1050 process_msg_ring(uint16_t proc_id) 1051 { 1052 void *msg; 1053 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1054 1055 if (unlikely(ret == 0)) { 1056 handle_msg((struct ff_msg *)msg, proc_id); 1057 } 1058 1059 return 0; 1060 } 1061 1062 /* Send burst of packets on an output interface */ 1063 static inline int 1064 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1065 { 1066 struct rte_mbuf **m_table; 1067 int ret; 1068 uint16_t queueid; 1069 1070 queueid = qconf->tx_queue_id[port]; 1071 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1072 1073 if (unlikely(qconf->pcap[port] != NULL)) { 1074 uint16_t i; 1075 for (i = 0; i < n; i++) { 1076 ff_dump_packets(qconf->pcap[port], m_table[i]); 1077 } 1078 } 1079 1080 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1081 if (unlikely(ret < n)) { 1082 do { 1083 rte_pktmbuf_free(m_table[ret]); 1084 } while (++ret < n); 1085 } 1086 1087 return 0; 1088 } 1089 1090 /* Enqueue a single packet, and send burst if queue is filled */ 1091 static inline int 1092 send_single_packet(struct rte_mbuf *m, uint8_t port) 1093 { 1094 uint16_t len; 1095 struct lcore_conf *qconf; 1096 1097 qconf = &lcore_conf; 1098 len = qconf->tx_mbufs[port].len; 1099 qconf->tx_mbufs[port].m_table[len] = m; 1100 len++; 1101 1102 /* enough pkts to be sent */ 1103 if (unlikely(len == MAX_PKT_BURST)) { 1104 send_burst(qconf, MAX_PKT_BURST, port); 1105 len = 0; 1106 } 1107 1108 qconf->tx_mbufs[port].len = len; 1109 return 0; 1110 } 1111 1112 int 1113 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1114 int total) 1115 { 1116 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1117 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1118 if (head == NULL) { 1119 ff_mbuf_free(m); 1120 return -1; 1121 } 1122 1123 head->pkt_len = total; 1124 head->nb_segs = 0; 1125 1126 int off = 0; 1127 struct rte_mbuf *cur = head, *prev = NULL; 1128 while(total > 0) { 1129 if (cur == NULL) { 1130 cur = rte_pktmbuf_alloc(mbuf_pool); 1131 if (cur == NULL) { 1132 rte_pktmbuf_free(head); 1133 ff_mbuf_free(m); 1134 return -1; 1135 } 1136 } 1137 1138 void *data = rte_pktmbuf_mtod(cur, void*); 1139 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1140 int ret = ff_mbuf_copydata(m, data, off, len); 1141 if (ret < 0) { 1142 rte_pktmbuf_free(head); 1143 ff_mbuf_free(m); 1144 return -1; 1145 } 1146 1147 if (prev != NULL) { 1148 prev->next = cur; 1149 } 1150 prev = cur; 1151 1152 cur->data_len = len; 1153 off += len; 1154 total -= len; 1155 head->nb_segs++; 1156 cur = NULL; 1157 } 1158 1159 struct ff_tx_offload offload = {0}; 1160 ff_mbuf_tx_offload(m, &offload); 1161 1162 if (offload.ip_csum) { 1163 head->ol_flags |= PKT_TX_IP_CKSUM; 1164 head->l2_len = sizeof(struct ether_hdr); 1165 head->l3_len = sizeof(struct ipv4_hdr); 1166 } 1167 1168 if (ctx->hw_features.tx_csum_l4) { 1169 if (offload.tcp_csum) { 1170 head->ol_flags |= PKT_TX_TCP_CKSUM; 1171 head->l2_len = sizeof(struct ether_hdr); 1172 head->l3_len = sizeof(struct ipv4_hdr); 1173 } 1174 1175 if (offload.tso_seg_size) { 1176 head->ol_flags |= PKT_TX_TCP_SEG; 1177 head->l4_len = sizeof(struct tcp_hdr); 1178 head->tso_segsz = offload.tso_seg_size; 1179 } 1180 1181 if (offload.udp_csum) { 1182 head->ol_flags |= PKT_TX_UDP_CKSUM; 1183 head->l2_len = sizeof(struct ether_hdr); 1184 head->l3_len = sizeof(struct ipv4_hdr); 1185 } 1186 } 1187 1188 ff_mbuf_free(m); 1189 1190 return send_single_packet(head, ctx->port_id); 1191 } 1192 1193 static int 1194 main_loop(void *arg) 1195 { 1196 struct loop_routine *lr = (struct loop_routine *)arg; 1197 1198 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1199 unsigned lcore_id; 1200 uint64_t prev_tsc, diff_tsc, cur_tsc; 1201 int i, j, nb_rx; 1202 uint8_t port_id, queue_id; 1203 struct lcore_conf *qconf; 1204 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1205 US_PER_S * BURST_TX_DRAIN_US; 1206 struct ff_dpdk_if_context *ctx; 1207 1208 prev_tsc = 0; 1209 1210 lcore_id = rte_lcore_id(); 1211 qconf = &lcore_conf; 1212 1213 if (qconf->nb_rx_queue == 0) { 1214 printf("lcore %u has nothing to do\n", lcore_id); 1215 return 0; 1216 } 1217 1218 while (1) { 1219 cur_tsc = rte_rdtsc(); 1220 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1221 rte_timer_manage(); 1222 } 1223 1224 /* 1225 * TX burst queue drain 1226 */ 1227 diff_tsc = cur_tsc - prev_tsc; 1228 if (unlikely(diff_tsc > drain_tsc)) { 1229 /* 1230 * This could be optimized (use queueid instead of 1231 * portid), but it is not called so often 1232 */ 1233 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 1234 if (qconf->tx_mbufs[port_id].len == 0) 1235 continue; 1236 send_burst(qconf, 1237 qconf->tx_mbufs[port_id].len, 1238 port_id); 1239 qconf->tx_mbufs[port_id].len = 0; 1240 } 1241 1242 prev_tsc = cur_tsc; 1243 } 1244 1245 /* 1246 * Read packet from RX queues 1247 */ 1248 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1249 port_id = qconf->rx_queue_list[i].port_id; 1250 queue_id = qconf->rx_queue_list[i].queue_id; 1251 ctx = veth_ctx[port_id]; 1252 1253 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1254 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1255 } 1256 1257 process_arp_ring(port_id, queue_id, pkts_burst, ctx); 1258 1259 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1260 MAX_PKT_BURST); 1261 if (nb_rx == 0) 1262 continue; 1263 1264 /* Prefetch first packets */ 1265 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1266 rte_prefetch0(rte_pktmbuf_mtod( 1267 pkts_burst[j], void *)); 1268 } 1269 1270 /* Prefetch and handle already prefetched packets */ 1271 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1272 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1273 j + PREFETCH_OFFSET], void *)); 1274 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1275 } 1276 1277 /* Handle remaining prefetched packets */ 1278 for (; j < nb_rx; j++) { 1279 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1280 } 1281 } 1282 1283 process_msg_ring(qconf->proc_id); 1284 1285 if (likely(lr->loop != NULL)) { 1286 lr->loop(lr->arg); 1287 } 1288 } 1289 } 1290 1291 int 1292 ff_dpdk_if_up(void) { 1293 int nb_ports = ff_global_cfg.dpdk.nb_ports; 1294 int i; 1295 for (i = 0; i < nb_ports; i++) { 1296 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 1297 veth_ctx[port_id] = ff_veth_attach(ff_global_cfg.dpdk.port_cfgs + i); 1298 if (veth_ctx[port_id] == NULL) { 1299 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1300 } 1301 } 1302 1303 return 0; 1304 } 1305 1306 void 1307 ff_dpdk_run(loop_func_t loop, void *arg) { 1308 struct loop_routine *lr = malloc(sizeof(struct loop_routine)); 1309 lr->loop = loop; 1310 lr->arg = arg; 1311 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1312 rte_eal_mp_wait_lcore(); 1313 free(lr); 1314 } 1315 1316 void 1317 ff_dpdk_pktmbuf_free(void *m) 1318 { 1319 rte_pktmbuf_free((struct rte_mbuf *)m); 1320 } 1321 1322 static uint32_t 1323 toeplitz_hash(unsigned keylen, const uint8_t *key, 1324 unsigned datalen, const uint8_t *data) 1325 { 1326 uint32_t hash = 0, v; 1327 u_int i, b; 1328 1329 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1330 1331 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1332 for (i = 0; i < datalen; i++) { 1333 for (b = 0; b < 8; b++) { 1334 if (data[i] & (1<<(7-b))) 1335 hash ^= v; 1336 v <<= 1; 1337 if ((i + 4) < keylen && 1338 (key[i+4] & (1<<(7-b)))) 1339 v |= 1; 1340 } 1341 } 1342 return (hash); 1343 } 1344 1345 int 1346 ff_rss_check(uint32_t saddr, uint32_t daddr, uint16_t sport, uint16_t dport) 1347 { 1348 struct lcore_conf *qconf = &lcore_conf; 1349 1350 if (qconf->nb_procs == 1) { 1351 return 1; 1352 } 1353 1354 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1355 sizeof(dport)]; 1356 1357 unsigned datalen = 0; 1358 1359 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1360 datalen += sizeof(saddr); 1361 1362 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1363 datalen += sizeof(daddr); 1364 1365 bcopy(&sport, &data[datalen], sizeof(sport)); 1366 datalen += sizeof(sport); 1367 1368 bcopy(&dport, &data[datalen], sizeof(dport)); 1369 datalen += sizeof(dport); 1370 1371 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), default_rsskey_40bytes, datalen, data); 1372 1373 return (hash % qconf->nb_procs) == qconf->proc_id; 1374 } 1375 1376 1377