1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 27 #include <rte_common.h> 28 #include <rte_byteorder.h> 29 #include <rte_log.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_memzone.h> 33 #include <rte_config.h> 34 #include <rte_eal.h> 35 #include <rte_pci.h> 36 #include <rte_mbuf.h> 37 #include <rte_memory.h> 38 #include <rte_lcore.h> 39 #include <rte_launch.h> 40 #include <rte_ethdev.h> 41 #include <rte_debug.h> 42 #include <rte_common.h> 43 #include <rte_ether.h> 44 #include <rte_malloc.h> 45 #include <rte_cycles.h> 46 #include <rte_timer.h> 47 #include <rte_thash.h> 48 #include <rte_ip.h> 49 #include <rte_tcp.h> 50 #include <rte_udp.h> 51 52 #include "ff_dpdk_if.h" 53 #include "ff_dpdk_pcap.h" 54 #include "ff_dpdk_kni.h" 55 #include "ff_config.h" 56 #include "ff_veth.h" 57 #include "ff_host_interface.h" 58 #include "ff_msg.h" 59 #include "ff_api.h" 60 61 #define MEMPOOL_CACHE_SIZE 256 62 63 #define ARP_RING_SIZE 2048 64 65 #define MSG_RING_SIZE 32 66 67 /* 68 * Configurable number of RX/TX ring descriptors 69 */ 70 #define RX_QUEUE_SIZE 512 71 #define TX_QUEUE_SIZE 256 72 73 #define MAX_PKT_BURST 32 74 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 75 76 /* 77 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 78 */ 79 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 80 81 #define NB_SOCKETS 8 82 83 /* Configure how many packets ahead to prefetch, when reading packets */ 84 #define PREFETCH_OFFSET 3 85 86 #define MAX_RX_QUEUE_PER_LCORE 16 87 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 88 #define MAX_RX_QUEUE_PER_PORT 128 89 90 #define BITS_PER_HEX 4 91 92 #define KNI_MBUF_MAX 2048 93 #define KNI_QUEUE_SIZE 2048 94 95 static int enable_kni; 96 static int kni_accept; 97 98 static struct rte_timer freebsd_clock; 99 100 // Mellanox Linux's driver key 101 static uint8_t default_rsskey_40bytes[40] = { 102 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 103 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 104 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 105 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 106 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 107 }; 108 109 static struct rte_eth_conf default_port_conf = { 110 .rxmode = { 111 .mq_mode = ETH_MQ_RX_RSS, 112 .max_rx_pkt_len = ETHER_MAX_LEN, 113 .split_hdr_size = 0, /**< hdr buf size */ 114 .header_split = 0, /**< Header Split disabled */ 115 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 116 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 117 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 118 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 119 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 120 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 121 .enable_lro = 0, /**< LRO disabled */ 122 }, 123 .rx_adv_conf = { 124 .rss_conf = { 125 .rss_key = default_rsskey_40bytes, 126 .rss_key_len = 40, 127 .rss_hf = ETH_RSS_PROTO_MASK, 128 }, 129 }, 130 .txmode = { 131 .mq_mode = ETH_MQ_TX_NONE, 132 }, 133 }; 134 135 struct mbuf_table { 136 uint16_t len; 137 struct rte_mbuf *m_table[MAX_PKT_BURST]; 138 }; 139 140 struct lcore_rx_queue { 141 uint8_t port_id; 142 uint8_t queue_id; 143 } __rte_cache_aligned; 144 145 struct lcore_conf { 146 uint16_t proc_id; 147 uint16_t nb_procs; 148 uint16_t socket_id; 149 uint16_t nb_rx_queue; 150 uint16_t *lcore_proc; 151 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 152 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 153 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 154 char *pcap[RTE_MAX_ETHPORTS]; 155 } __rte_cache_aligned; 156 157 static struct lcore_conf lcore_conf; 158 159 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 160 161 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 162 163 struct ff_msg_ring { 164 char ring_name[2][RTE_RING_NAMESIZE]; 165 /* ring[0] for lcore recv msg, other send */ 166 /* ring[1] for lcore send msg, other read */ 167 struct rte_ring *ring[2]; 168 } __rte_cache_aligned; 169 170 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 171 static struct rte_mempool *message_pool; 172 173 struct ff_dpdk_if_context { 174 void *sc; 175 void *ifp; 176 uint16_t port_id; 177 struct ff_hw_features hw_features; 178 } __rte_cache_aligned; 179 180 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 181 182 extern void ff_hardclock(void); 183 184 static void 185 ff_hardclock_job(__rte_unused struct rte_timer *timer, 186 __rte_unused void *arg) { 187 ff_hardclock(); 188 ff_update_current_ts(); 189 } 190 191 struct ff_dpdk_if_context * 192 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 193 { 194 struct ff_dpdk_if_context *ctx; 195 196 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 197 if (ctx == NULL) 198 return NULL; 199 200 ctx->sc = sc; 201 ctx->ifp = ifp; 202 ctx->port_id = cfg->port_id; 203 ctx->hw_features = cfg->hw_features; 204 205 return ctx; 206 } 207 208 void 209 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 210 { 211 free(ctx); 212 } 213 214 static void 215 check_all_ports_link_status(void) 216 { 217 #define CHECK_INTERVAL 100 /* 100ms */ 218 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 219 220 uint8_t portid, count, all_ports_up, print_flag = 0; 221 struct rte_eth_link link; 222 223 printf("\nChecking link status"); 224 fflush(stdout); 225 226 int i, nb_ports; 227 nb_ports = ff_global_cfg.dpdk.nb_ports; 228 for (count = 0; count <= MAX_CHECK_TIME; count++) { 229 all_ports_up = 1; 230 for (i = 0; i < nb_ports; i++) { 231 uint8_t portid = ff_global_cfg.dpdk.port_cfgs[i].port_id; 232 memset(&link, 0, sizeof(link)); 233 rte_eth_link_get_nowait(portid, &link); 234 235 /* print link status if flag set */ 236 if (print_flag == 1) { 237 if (link.link_status) { 238 printf("Port %d Link Up - speed %u " 239 "Mbps - %s\n", (int)portid, 240 (unsigned)link.link_speed, 241 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 242 ("full-duplex") : ("half-duplex\n")); 243 } else { 244 printf("Port %d Link Down\n", (int)portid); 245 } 246 continue; 247 } 248 /* clear all_ports_up flag if any link down */ 249 if (link.link_status == 0) { 250 all_ports_up = 0; 251 break; 252 } 253 } 254 255 /* after finally printing all link status, get out */ 256 if (print_flag == 1) 257 break; 258 259 if (all_ports_up == 0) { 260 printf("."); 261 fflush(stdout); 262 rte_delay_ms(CHECK_INTERVAL); 263 } 264 265 /* set the print_flag if all ports up or timeout */ 266 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 267 print_flag = 1; 268 printf("done\n"); 269 } 270 } 271 } 272 273 static int 274 xdigit2val(unsigned char c) 275 { 276 int val; 277 278 if (isdigit(c)) 279 val = c - '0'; 280 else if (isupper(c)) 281 val = c - 'A' + 10; 282 else 283 val = c - 'a' + 10; 284 return val; 285 } 286 287 static int 288 parse_lcore_mask(const char *coremask, uint16_t *lcore_proc, 289 uint16_t nb_procs) 290 { 291 int i, j, idx = 0; 292 unsigned count = 0; 293 char c; 294 int val; 295 296 if (coremask == NULL) 297 return -1; 298 299 /* Remove all blank characters ahead and after. 300 * Remove 0x/0X if exists. 301 */ 302 while (isblank(*coremask)) 303 coremask++; 304 if (coremask[0] == '0' && ((coremask[1] == 'x') 305 || (coremask[1] == 'X'))) 306 coremask += 2; 307 308 i = strlen(coremask); 309 while ((i > 0) && isblank(coremask[i - 1])) 310 i--; 311 312 if (i == 0) 313 return -1; 314 315 for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE && count < nb_procs; i--) { 316 c = coremask[i]; 317 if (isxdigit(c) == 0) { 318 return -1; 319 } 320 val = xdigit2val(c); 321 for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE && count < nb_procs; 322 j++, idx++) { 323 if ((1 << j) & val) { 324 if (!lcore_config[idx].detected) { 325 RTE_LOG(ERR, EAL, "lcore %u unavailable\n", idx); 326 return -1; 327 } 328 lcore_proc[count] = idx; 329 count++; 330 } 331 } 332 } 333 334 for (; i >= 0; i--) 335 if (coremask[i] != '0') 336 return -1; 337 338 if (count < nb_procs) 339 return -1; 340 341 return 0; 342 } 343 344 static int 345 init_lcore_conf(void) 346 { 347 uint8_t nb_ports = rte_eth_dev_count(); 348 if (nb_ports == 0) { 349 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 350 } 351 352 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 353 lcore_conf.nb_procs = ff_global_cfg.dpdk.nb_procs; 354 lcore_conf.lcore_proc = rte_zmalloc(NULL, 355 sizeof(uint16_t)*lcore_conf.nb_procs, 0); 356 if (lcore_conf.lcore_proc == NULL) { 357 rte_exit(EXIT_FAILURE, "rte_zmalloc lcore_proc failed\n"); 358 } 359 360 int ret = parse_lcore_mask(ff_global_cfg.dpdk.lcore_mask, 361 lcore_conf.lcore_proc, lcore_conf.nb_procs); 362 if (ret < 0) { 363 rte_exit(EXIT_FAILURE, "parse_lcore_mask failed:%s\n", 364 ff_global_cfg.dpdk.lcore_mask); 365 } 366 367 uint16_t socket_id = 0; 368 if (ff_global_cfg.dpdk.numa_on) { 369 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 370 } 371 372 lcore_conf.socket_id = socket_id; 373 374 /* Currently, proc id 1:1 map to rx/tx queue id per port. */ 375 uint8_t port_id, enabled_ports = 0; 376 for (port_id = 0; port_id < nb_ports; port_id++) { 377 if (ff_global_cfg.dpdk.port_mask && 378 (ff_global_cfg.dpdk.port_mask & (1 << port_id)) == 0) { 379 printf("\nSkipping disabled port %d\n", port_id); 380 continue; 381 } 382 383 if (port_id >= ff_global_cfg.dpdk.nb_ports) { 384 printf("\nSkipping non-configured port %d\n", port_id); 385 break; 386 } 387 388 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 389 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 390 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = lcore_conf.proc_id; 391 lcore_conf.nb_rx_queue++; 392 393 lcore_conf.tx_queue_id[port_id] = lcore_conf.proc_id; 394 lcore_conf.pcap[port_id] = ff_global_cfg.dpdk.port_cfgs[enabled_ports].pcap; 395 396 ff_global_cfg.dpdk.port_cfgs[enabled_ports].port_id = port_id; 397 398 enabled_ports++; 399 } 400 401 ff_global_cfg.dpdk.nb_ports = enabled_ports; 402 403 return 0; 404 } 405 406 static int 407 init_mem_pool(void) 408 { 409 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 410 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 411 uint32_t nb_tx_queue = nb_lcores; 412 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 413 414 unsigned nb_mbuf = RTE_MAX ( 415 (nb_rx_queue*RX_QUEUE_SIZE + 416 nb_ports*nb_lcores*MAX_PKT_BURST + 417 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 418 nb_lcores*MEMPOOL_CACHE_SIZE + 419 nb_ports*KNI_MBUF_MAX + 420 nb_ports*KNI_QUEUE_SIZE + 421 nb_lcores*nb_ports*ARP_RING_SIZE), 422 (unsigned)8192); 423 424 unsigned socketid = 0; 425 uint16_t i, lcore_id; 426 char s[64]; 427 int numa_on = ff_global_cfg.dpdk.numa_on; 428 429 for (i = 0; i < lcore_conf.nb_procs; i++) { 430 lcore_id = lcore_conf.lcore_proc[i]; 431 if (numa_on) { 432 socketid = rte_lcore_to_socket_id(lcore_id); 433 } 434 435 if (socketid >= NB_SOCKETS) { 436 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 437 socketid, i, NB_SOCKETS); 438 } 439 440 if (pktmbuf_pool[socketid] != NULL) { 441 continue; 442 } 443 444 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 445 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 446 pktmbuf_pool[socketid] = 447 rte_pktmbuf_pool_create(s, nb_mbuf, 448 MEMPOOL_CACHE_SIZE, 0, 449 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 450 } else { 451 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 452 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 453 } 454 455 if (pktmbuf_pool[socketid] == NULL) { 456 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 457 } else { 458 printf("create mbuf pool on socket %d\n", socketid); 459 } 460 } 461 462 return 0; 463 } 464 465 static struct rte_ring * 466 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 467 { 468 struct rte_ring *ring; 469 470 if (name == NULL) 471 return NULL; 472 473 /* If already create, just attached it */ 474 if (likely((ring = rte_ring_lookup(name)) != NULL)) 475 return ring; 476 477 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 478 return rte_ring_create(name, count, socket_id, flags); 479 } else { 480 return rte_ring_lookup(name); 481 } 482 } 483 484 static int 485 init_arp_ring(void) 486 { 487 int i, j, ret; 488 char name_buf[RTE_RING_NAMESIZE]; 489 int nb_procs = ff_global_cfg.dpdk.nb_procs; 490 int proc_id = ff_global_cfg.dpdk.proc_id; 491 492 /* Allocate arp ring ptr according to eth dev count. */ 493 int nb_ports = rte_eth_dev_count(); 494 for(i = 0; i < nb_procs; ++i) { 495 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 496 proc_id, i); 497 498 arp_ring[i] = rte_zmalloc(name_buf, 499 sizeof(struct rte_ring *) * nb_ports, 500 RTE_CACHE_LINE_SIZE); 501 if (arp_ring[i] == NULL) { 502 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 503 "failed\n", name_buf); 504 } 505 } 506 507 unsigned socketid = lcore_conf.socket_id; 508 509 /* Create ring according to ports actually being used. */ 510 nb_ports = ff_global_cfg.dpdk.nb_ports; 511 for (j = 0; j < nb_ports; j++) { 512 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[j].port_id; 513 514 for(i = 0; i < nb_procs; ++i) { 515 snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id); 516 arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE, 517 socketid, RING_F_SC_DEQ); 518 519 if (arp_ring[i][port_id] == NULL) 520 rte_panic("create ring:%s failed!\n", name_buf); 521 522 printf("create ring:%s success, %u ring entries are now free!\n", 523 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 524 } 525 } 526 527 return 0; 528 } 529 530 static void 531 ff_msg_init(struct rte_mempool *mp, 532 __attribute__((unused)) void *opaque_arg, 533 void *obj, __attribute__((unused)) unsigned i) 534 { 535 struct ff_msg *msg = (struct ff_msg *)obj; 536 msg->msg_type = FF_UNKNOWN; 537 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 538 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 539 } 540 541 static int 542 init_msg_ring(void) 543 { 544 uint16_t i; 545 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 546 unsigned socketid = lcore_conf.socket_id; 547 548 /* Create message buffer pool */ 549 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 550 message_pool = rte_mempool_create(FF_MSG_POOL, 551 MSG_RING_SIZE * 2 * nb_procs, 552 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 553 NULL, NULL, ff_msg_init, NULL, 554 socketid, 0); 555 } else { 556 message_pool = rte_mempool_lookup(FF_MSG_POOL); 557 } 558 559 if (message_pool == NULL) { 560 rte_panic("Create msg mempool failed\n"); 561 } 562 563 for(i = 0; i < nb_procs; ++i) { 564 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 565 "%s%u", FF_MSG_RING_IN, i); 566 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 567 "%s%u", FF_MSG_RING_OUT, i); 568 569 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 570 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 571 if (msg_ring[i].ring[0] == NULL) 572 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 573 574 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 575 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 576 if (msg_ring[i].ring[1] == NULL) 577 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 578 } 579 580 return 0; 581 } 582 583 static int 584 init_kni(void) 585 { 586 int nb_ports = rte_eth_dev_count(); 587 kni_accept = 0; 588 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 589 kni_accept = 1; 590 591 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 592 ff_global_cfg.kni.udp_port); 593 594 unsigned socket_id = lcore_conf.socket_id; 595 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 596 597 nb_ports = ff_global_cfg.dpdk.nb_ports; 598 int i, ret; 599 for (i = 0; i < nb_ports; i++) { 600 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 601 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 602 } 603 604 return 0; 605 } 606 607 static int 608 init_port_start(void) 609 { 610 int nb_ports = ff_global_cfg.dpdk.nb_ports; 611 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 612 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 613 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 614 uint16_t i; 615 616 for (i = 0; i < nb_ports; i++) { 617 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 618 619 struct rte_eth_dev_info dev_info; 620 rte_eth_dev_info_get(port_id, &dev_info); 621 622 if (nb_procs > dev_info.max_rx_queues) { 623 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 624 nb_procs, 625 dev_info.max_rx_queues); 626 } 627 628 if (nb_procs > dev_info.max_tx_queues) { 629 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 630 nb_procs, 631 dev_info.max_tx_queues); 632 } 633 634 struct ether_addr addr; 635 rte_eth_macaddr_get(port_id, &addr); 636 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 637 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 638 (unsigned)port_id, 639 addr.addr_bytes[0], addr.addr_bytes[1], 640 addr.addr_bytes[2], addr.addr_bytes[3], 641 addr.addr_bytes[4], addr.addr_bytes[5]); 642 643 rte_memcpy(ff_global_cfg.dpdk.port_cfgs[i].mac, 644 addr.addr_bytes, ETHER_ADDR_LEN); 645 646 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 647 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 648 ETH_TXQ_FLAGS_NOREFCOUNT; 649 650 /* Disable features that are not supported by port's HW */ 651 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 652 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 653 } 654 655 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 656 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 657 } 658 659 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 660 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 661 } 662 663 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 664 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 665 } 666 667 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 668 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 669 } 670 671 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 672 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 673 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 674 } 675 676 struct rte_eth_conf port_conf = {0}; 677 678 /* Set RSS mode */ 679 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 680 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 681 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 682 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 683 684 /* Set Rx VLAN stripping */ 685 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 686 port_conf.rxmode.hw_vlan_strip = 1; 687 } 688 689 /* Enable HW CRC stripping */ 690 port_conf.rxmode.hw_strip_crc = 1; 691 692 /* FIXME: Enable TCP LRO ?*/ 693 #if 0 694 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 695 printf("LRO is supported\n"); 696 port_conf.rxmode.enable_lro = 1; 697 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_lro = 1; 698 } 699 #endif 700 701 /* Set Rx checksum checking */ 702 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 703 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 704 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 705 printf("RX checksum offload supported\n"); 706 port_conf.rxmode.hw_ip_checksum = 1; 707 ff_global_cfg.dpdk.port_cfgs[i].hw_features.rx_csum = 1; 708 } 709 710 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 711 printf("TX ip checksum offload supported\n"); 712 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_ip = 1; 713 } 714 715 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 716 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 717 printf("TX TCP&UDP checksum offload supported\n"); 718 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_csum_l4 = 1; 719 } 720 721 if (ff_global_cfg.dpdk.tso) { 722 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 723 printf("TSO is supported\n"); 724 ff_global_cfg.dpdk.port_cfgs[i].hw_features.tx_tso = 1; 725 } 726 } else { 727 printf("TSO is disabled\n"); 728 } 729 730 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 731 continue; 732 } 733 734 /* Currently, proc id 1:1 map to queue id per port. */ 735 int ret = rte_eth_dev_configure(port_id, nb_procs, nb_procs, &port_conf); 736 if (ret != 0) { 737 return ret; 738 } 739 740 uint16_t q; 741 for (q = 0; q < nb_procs; q++) { 742 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 743 socketid, &dev_info.default_txconf); 744 if (ret < 0) { 745 return ret; 746 } 747 748 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 749 socketid, &dev_info.default_rxconf, mbuf_pool); 750 if (ret < 0) { 751 return ret; 752 } 753 } 754 755 ret = rte_eth_dev_start(port_id); 756 if (ret < 0) { 757 return ret; 758 } 759 760 /* Enable RX in promiscuous mode for the Ethernet device. */ 761 if (ff_global_cfg.dpdk.promiscuous) { 762 rte_eth_promiscuous_enable(port_id); 763 ret = rte_eth_promiscuous_get(port_id); 764 if (ret == 1) { 765 printf("set port %u to promiscuous mode ok\n", port_id); 766 } else { 767 printf("set port %u to promiscuous mode error\n", port_id); 768 } 769 } 770 771 /* Enable pcap dump */ 772 if (ff_global_cfg.dpdk.port_cfgs[i].pcap) { 773 ff_enable_pcap(ff_global_cfg.dpdk.port_cfgs[i].pcap); 774 } 775 } 776 777 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 778 check_all_ports_link_status(); 779 } 780 781 return 0; 782 } 783 784 static int 785 init_clock(void) 786 { 787 rte_timer_subsystem_init(); 788 uint64_t hz = rte_get_timer_hz(); 789 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 790 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 791 792 rte_timer_init(&freebsd_clock); 793 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 794 rte_lcore_id(), &ff_hardclock_job, NULL); 795 796 ff_update_current_ts(); 797 798 return 0; 799 } 800 801 int 802 ff_dpdk_init(int argc, char **argv) 803 { 804 if (ff_global_cfg.dpdk.nb_procs < 1 || 805 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 806 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 807 ff_global_cfg.dpdk.proc_id < 0) { 808 printf("param num_procs[%d] or proc_id[%d] error!\n", 809 ff_global_cfg.dpdk.nb_procs, 810 ff_global_cfg.dpdk.proc_id); 811 exit(1); 812 } 813 814 int ret = rte_eal_init(argc, argv); 815 if (ret < 0) { 816 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 817 } 818 819 init_lcore_conf(); 820 821 init_mem_pool(); 822 823 init_arp_ring(); 824 825 init_msg_ring(); 826 827 enable_kni = ff_global_cfg.kni.enable; 828 if (enable_kni) { 829 init_kni(); 830 } 831 832 ret = init_port_start(); 833 if (ret < 0) { 834 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 835 } 836 837 init_clock(); 838 839 return 0; 840 } 841 842 static void 843 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 844 { 845 uint8_t rx_csum = ctx->hw_features.rx_csum; 846 if (rx_csum) { 847 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 848 return; 849 } 850 } 851 852 /* 853 * FIXME: should we save pkt->vlan_tci 854 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 855 */ 856 857 void *data = rte_pktmbuf_mtod(pkt, void*); 858 uint16_t len = rte_pktmbuf_data_len(pkt); 859 860 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 861 if (hdr == NULL) { 862 rte_pktmbuf_free(pkt); 863 return; 864 } 865 866 struct rte_mbuf *pn = pkt->next; 867 void *prev = hdr; 868 while(pn != NULL) { 869 data = rte_pktmbuf_mtod(pkt, void*); 870 len = rte_pktmbuf_data_len(pkt); 871 872 void *mb = ff_mbuf_get(prev, data, len); 873 if (mb == NULL) { 874 ff_mbuf_free(hdr); 875 rte_pktmbuf_free(pkt); 876 return; 877 } 878 pn = pn->next; 879 prev = mb; 880 } 881 882 ff_veth_process_packet(ctx->ifp, hdr); 883 } 884 885 static enum FilterReturn 886 protocol_filter(const void *data, uint16_t len) 887 { 888 if(len < sizeof(struct ether_hdr)) 889 return FILTER_UNKNOWN; 890 891 const struct ether_hdr *hdr; 892 hdr = (const struct ether_hdr *)data; 893 894 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 895 return FILTER_ARP; 896 897 if (!enable_kni) { 898 return FILTER_UNKNOWN; 899 } 900 901 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 902 return FILTER_UNKNOWN; 903 904 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 905 len - sizeof(struct ether_hdr)); 906 } 907 908 static inline void 909 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 910 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 911 { 912 struct lcore_conf *qconf = &lcore_conf; 913 914 uint16_t i; 915 for (i = 0; i < count; i++) { 916 struct rte_mbuf *rtem = bufs[i]; 917 918 if (unlikely(qconf->pcap[port_id] != NULL)) { 919 ff_dump_packets(qconf->pcap[port_id], rtem); 920 } 921 922 void *data = rte_pktmbuf_mtod(rtem, void*); 923 uint16_t len = rte_pktmbuf_data_len(rtem); 924 925 enum FilterReturn filter = protocol_filter(data, len); 926 if (filter == FILTER_ARP) { 927 struct rte_mempool *mbuf_pool; 928 struct rte_mbuf *mbuf_clone; 929 if (pkts_from_ring == 0) { 930 uint16_t i; 931 for(i = 0; i < qconf->nb_procs; ++i) { 932 if(i == queue_id) 933 continue; 934 935 mbuf_pool = pktmbuf_pool[rte_lcore_to_socket_id(qconf->lcore_proc[i])]; 936 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 937 if(mbuf_clone) { 938 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 939 if (ret < 0) 940 rte_pktmbuf_free(mbuf_clone); 941 } 942 } 943 } 944 945 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 946 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 947 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 948 if(mbuf_clone) { 949 ff_kni_enqueue(port_id, mbuf_clone); 950 } 951 } 952 953 ff_veth_input(ctx, rtem); 954 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 955 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 956 ff_kni_enqueue(port_id, rtem); 957 } else { 958 ff_veth_input(ctx, rtem); 959 } 960 } 961 } 962 963 static inline int 964 process_arp_ring(uint8_t port_id, uint16_t queue_id, 965 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 966 { 967 /* read packet from ring buf and to process */ 968 uint16_t nb_rb; 969 nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 970 (void **)pkts_burst, MAX_PKT_BURST); 971 972 if(nb_rb > 0) { 973 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 974 } 975 976 return 0; 977 } 978 979 static inline void 980 handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id) 981 { 982 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 983 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 984 msg->sysctl.newlen); 985 986 if (ret < 0) { 987 msg->result = errno; 988 } else { 989 msg->result = 0; 990 } 991 992 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 993 } 994 995 static inline void 996 handle_ioctl_msg(struct ff_msg *msg, uint16_t proc_id) 997 { 998 int fd, ret; 999 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 1000 if (fd < 0) { 1001 ret = -1; 1002 goto done; 1003 } 1004 1005 ret = ff_ioctl(fd, msg->ioctl.cmd, msg->ioctl.data); 1006 1007 ff_close(fd); 1008 1009 done: 1010 if (ret < 0) { 1011 msg->result = errno; 1012 } else { 1013 msg->result = 0; 1014 } 1015 1016 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1017 } 1018 1019 static inline void 1020 handle_route_msg(struct ff_msg *msg, uint16_t proc_id) 1021 { 1022 msg->result = ff_rtioctl(msg->route.fib, msg->route.data, 1023 &msg->route.len, msg->route.maxlen); 1024 1025 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1026 } 1027 1028 static inline void 1029 handle_default_msg(struct ff_msg *msg, uint16_t proc_id) 1030 { 1031 msg->result = EINVAL; 1032 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1033 } 1034 1035 static inline void 1036 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1037 { 1038 switch (msg->msg_type) { 1039 case FF_SYSCTL: 1040 handle_sysctl_msg(msg, proc_id); 1041 break; 1042 case FF_IOCTL: 1043 handle_ioctl_msg(msg, proc_id); 1044 break; 1045 case FF_ROUTE: 1046 handle_route_msg(msg, proc_id); 1047 break; 1048 default: 1049 handle_default_msg(msg, proc_id); 1050 break; 1051 } 1052 } 1053 1054 static inline int 1055 process_msg_ring(uint16_t proc_id) 1056 { 1057 void *msg; 1058 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1059 1060 if (unlikely(ret == 0)) { 1061 handle_msg((struct ff_msg *)msg, proc_id); 1062 } 1063 1064 return 0; 1065 } 1066 1067 /* Send burst of packets on an output interface */ 1068 static inline int 1069 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1070 { 1071 struct rte_mbuf **m_table; 1072 int ret; 1073 uint16_t queueid; 1074 1075 queueid = qconf->tx_queue_id[port]; 1076 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1077 1078 if (unlikely(qconf->pcap[port] != NULL)) { 1079 uint16_t i; 1080 for (i = 0; i < n; i++) { 1081 ff_dump_packets(qconf->pcap[port], m_table[i]); 1082 } 1083 } 1084 1085 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1086 if (unlikely(ret < n)) { 1087 do { 1088 rte_pktmbuf_free(m_table[ret]); 1089 } while (++ret < n); 1090 } 1091 1092 return 0; 1093 } 1094 1095 /* Enqueue a single packet, and send burst if queue is filled */ 1096 static inline int 1097 send_single_packet(struct rte_mbuf *m, uint8_t port) 1098 { 1099 uint16_t len; 1100 struct lcore_conf *qconf; 1101 1102 qconf = &lcore_conf; 1103 len = qconf->tx_mbufs[port].len; 1104 qconf->tx_mbufs[port].m_table[len] = m; 1105 len++; 1106 1107 /* enough pkts to be sent */ 1108 if (unlikely(len == MAX_PKT_BURST)) { 1109 send_burst(qconf, MAX_PKT_BURST, port); 1110 len = 0; 1111 } 1112 1113 qconf->tx_mbufs[port].len = len; 1114 return 0; 1115 } 1116 1117 int 1118 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1119 int total) 1120 { 1121 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1122 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1123 if (head == NULL) { 1124 ff_mbuf_free(m); 1125 return -1; 1126 } 1127 1128 head->pkt_len = total; 1129 head->nb_segs = 0; 1130 1131 int off = 0; 1132 struct rte_mbuf *cur = head, *prev = NULL; 1133 while(total > 0) { 1134 if (cur == NULL) { 1135 cur = rte_pktmbuf_alloc(mbuf_pool); 1136 if (cur == NULL) { 1137 rte_pktmbuf_free(head); 1138 ff_mbuf_free(m); 1139 return -1; 1140 } 1141 } 1142 1143 void *data = rte_pktmbuf_mtod(cur, void*); 1144 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1145 int ret = ff_mbuf_copydata(m, data, off, len); 1146 if (ret < 0) { 1147 rte_pktmbuf_free(head); 1148 ff_mbuf_free(m); 1149 return -1; 1150 } 1151 1152 if (prev != NULL) { 1153 prev->next = cur; 1154 } 1155 prev = cur; 1156 1157 cur->data_len = len; 1158 off += len; 1159 total -= len; 1160 head->nb_segs++; 1161 cur = NULL; 1162 } 1163 1164 struct ff_tx_offload offload = {0}; 1165 ff_mbuf_tx_offload(m, &offload); 1166 1167 if (offload.ip_csum) { 1168 head->ol_flags |= PKT_TX_IP_CKSUM; 1169 head->l2_len = sizeof(struct ether_hdr); 1170 head->l3_len = sizeof(struct ipv4_hdr); 1171 } 1172 1173 if (ctx->hw_features.tx_csum_l4) { 1174 if (offload.tcp_csum) { 1175 head->ol_flags |= PKT_TX_TCP_CKSUM; 1176 head->l2_len = sizeof(struct ether_hdr); 1177 head->l3_len = sizeof(struct ipv4_hdr); 1178 } 1179 1180 if (offload.tso_seg_size) { 1181 head->ol_flags |= PKT_TX_TCP_SEG; 1182 head->l4_len = sizeof(struct tcp_hdr); 1183 head->tso_segsz = offload.tso_seg_size; 1184 } 1185 1186 if (offload.udp_csum) { 1187 head->ol_flags |= PKT_TX_UDP_CKSUM; 1188 head->l2_len = sizeof(struct ether_hdr); 1189 head->l3_len = sizeof(struct ipv4_hdr); 1190 } 1191 } 1192 1193 ff_mbuf_free(m); 1194 1195 return send_single_packet(head, ctx->port_id); 1196 } 1197 1198 static int 1199 main_loop(void *arg) 1200 { 1201 struct loop_routine *lr = (struct loop_routine *)arg; 1202 1203 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1204 unsigned lcore_id; 1205 uint64_t prev_tsc, diff_tsc, cur_tsc; 1206 int i, j, nb_rx; 1207 uint8_t port_id, queue_id; 1208 struct lcore_conf *qconf; 1209 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1210 US_PER_S * BURST_TX_DRAIN_US; 1211 struct ff_dpdk_if_context *ctx; 1212 1213 prev_tsc = 0; 1214 1215 lcore_id = rte_lcore_id(); 1216 qconf = &lcore_conf; 1217 1218 if (qconf->nb_rx_queue == 0) { 1219 printf("lcore %u has nothing to do\n", lcore_id); 1220 return 0; 1221 } 1222 1223 while (1) { 1224 cur_tsc = rte_rdtsc(); 1225 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1226 rte_timer_manage(); 1227 } 1228 1229 /* 1230 * TX burst queue drain 1231 */ 1232 diff_tsc = cur_tsc - prev_tsc; 1233 if (unlikely(diff_tsc > drain_tsc)) { 1234 /* 1235 * This could be optimized (use queueid instead of 1236 * portid), but it is not called so often 1237 */ 1238 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 1239 if (qconf->tx_mbufs[port_id].len == 0) 1240 continue; 1241 send_burst(qconf, 1242 qconf->tx_mbufs[port_id].len, 1243 port_id); 1244 qconf->tx_mbufs[port_id].len = 0; 1245 } 1246 1247 prev_tsc = cur_tsc; 1248 } 1249 1250 /* 1251 * Read packet from RX queues 1252 */ 1253 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1254 port_id = qconf->rx_queue_list[i].port_id; 1255 queue_id = qconf->rx_queue_list[i].queue_id; 1256 ctx = veth_ctx[port_id]; 1257 1258 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1259 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1260 } 1261 1262 process_arp_ring(port_id, queue_id, pkts_burst, ctx); 1263 1264 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1265 MAX_PKT_BURST); 1266 if (nb_rx == 0) 1267 continue; 1268 1269 /* Prefetch first packets */ 1270 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1271 rte_prefetch0(rte_pktmbuf_mtod( 1272 pkts_burst[j], void *)); 1273 } 1274 1275 /* Prefetch and handle already prefetched packets */ 1276 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1277 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1278 j + PREFETCH_OFFSET], void *)); 1279 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1280 } 1281 1282 /* Handle remaining prefetched packets */ 1283 for (; j < nb_rx; j++) { 1284 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1285 } 1286 } 1287 1288 process_msg_ring(qconf->proc_id); 1289 1290 if (likely(lr->loop != NULL)) { 1291 lr->loop(lr->arg); 1292 } 1293 } 1294 } 1295 1296 int 1297 ff_dpdk_if_up(void) { 1298 int nb_ports = ff_global_cfg.dpdk.nb_ports; 1299 int i; 1300 for (i = 0; i < nb_ports; i++) { 1301 uint8_t port_id = ff_global_cfg.dpdk.port_cfgs[i].port_id; 1302 veth_ctx[port_id] = ff_veth_attach(ff_global_cfg.dpdk.port_cfgs + i); 1303 if (veth_ctx[port_id] == NULL) { 1304 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1305 } 1306 } 1307 1308 return 0; 1309 } 1310 1311 void 1312 ff_dpdk_run(loop_func_t loop, void *arg) { 1313 struct loop_routine *lr = malloc(sizeof(struct loop_routine)); 1314 lr->loop = loop; 1315 lr->arg = arg; 1316 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1317 rte_eal_mp_wait_lcore(); 1318 free(lr); 1319 } 1320 1321 void 1322 ff_dpdk_pktmbuf_free(void *m) 1323 { 1324 rte_pktmbuf_free((struct rte_mbuf *)m); 1325 } 1326 1327 static uint32_t 1328 toeplitz_hash(unsigned keylen, const uint8_t *key, 1329 unsigned datalen, const uint8_t *data) 1330 { 1331 uint32_t hash = 0, v; 1332 u_int i, b; 1333 1334 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1335 1336 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1337 for (i = 0; i < datalen; i++) { 1338 for (b = 0; b < 8; b++) { 1339 if (data[i] & (1<<(7-b))) 1340 hash ^= v; 1341 v <<= 1; 1342 if ((i + 4) < keylen && 1343 (key[i+4] & (1<<(7-b)))) 1344 v |= 1; 1345 } 1346 } 1347 return (hash); 1348 } 1349 1350 int 1351 ff_rss_check(uint32_t saddr, uint32_t daddr, uint16_t sport, uint16_t dport) 1352 { 1353 struct lcore_conf *qconf = &lcore_conf; 1354 1355 if (qconf->nb_procs == 1) { 1356 return 1; 1357 } 1358 1359 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1360 sizeof(dport)]; 1361 1362 unsigned datalen = 0; 1363 1364 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1365 datalen += sizeof(saddr); 1366 1367 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1368 datalen += sizeof(daddr); 1369 1370 bcopy(&sport, &data[datalen], sizeof(sport)); 1371 datalen += sizeof(sport); 1372 1373 bcopy(&dport, &data[datalen], sizeof(dport)); 1374 datalen += sizeof(dport); 1375 1376 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), default_rsskey_40bytes, datalen, data); 1377 1378 return (hash % qconf->nb_procs) == qconf->proc_id; 1379 } 1380 1381 1382