1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 #include <assert.h> 27 28 #include <rte_common.h> 29 #include <rte_byteorder.h> 30 #include <rte_log.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_memzone.h> 34 #include <rte_config.h> 35 #include <rte_eal.h> 36 #include <rte_pci.h> 37 #include <rte_mbuf.h> 38 #include <rte_memory.h> 39 #include <rte_lcore.h> 40 #include <rte_launch.h> 41 #include <rte_ethdev.h> 42 #include <rte_debug.h> 43 #include <rte_common.h> 44 #include <rte_ether.h> 45 #include <rte_malloc.h> 46 #include <rte_cycles.h> 47 #include <rte_timer.h> 48 #include <rte_thash.h> 49 #include <rte_ip.h> 50 #include <rte_tcp.h> 51 #include <rte_udp.h> 52 53 #include "ff_dpdk_if.h" 54 #include "ff_dpdk_pcap.h" 55 #include "ff_dpdk_kni.h" 56 #include "ff_config.h" 57 #include "ff_veth.h" 58 #include "ff_host_interface.h" 59 #include "ff_msg.h" 60 #include "ff_api.h" 61 62 #define MEMPOOL_CACHE_SIZE 256 63 64 #define ARP_RING_SIZE 2048 65 66 #define MSG_RING_SIZE 32 67 68 /* 69 * Configurable number of RX/TX ring descriptors 70 */ 71 #define RX_QUEUE_SIZE 512 72 #define TX_QUEUE_SIZE 512 73 74 #define MAX_PKT_BURST 32 75 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 76 77 /* 78 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 79 */ 80 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 81 82 #define NB_SOCKETS 8 83 84 /* Configure how many packets ahead to prefetch, when reading packets */ 85 #define PREFETCH_OFFSET 3 86 87 #define MAX_RX_QUEUE_PER_LCORE 16 88 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 89 #define MAX_RX_QUEUE_PER_PORT 128 90 91 #define KNI_MBUF_MAX 2048 92 #define KNI_QUEUE_SIZE 2048 93 94 static int enable_kni; 95 static int kni_accept; 96 97 static int numa_on; 98 99 static struct rte_timer freebsd_clock; 100 101 // Mellanox Linux's driver key 102 static uint8_t default_rsskey_40bytes[40] = { 103 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 104 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 105 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 106 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 107 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 108 }; 109 110 static struct rte_eth_conf default_port_conf = { 111 .rxmode = { 112 .mq_mode = ETH_MQ_RX_RSS, 113 .max_rx_pkt_len = ETHER_MAX_LEN, 114 .split_hdr_size = 0, /**< hdr buf size */ 115 .header_split = 0, /**< Header Split disabled */ 116 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 117 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 118 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 119 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 120 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 121 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 122 .enable_lro = 0, /**< LRO disabled */ 123 }, 124 .rx_adv_conf = { 125 .rss_conf = { 126 .rss_key = default_rsskey_40bytes, 127 .rss_key_len = 40, 128 .rss_hf = ETH_RSS_PROTO_MASK, 129 }, 130 }, 131 .txmode = { 132 .mq_mode = ETH_MQ_TX_NONE, 133 }, 134 }; 135 136 struct mbuf_table { 137 uint16_t len; 138 struct rte_mbuf *m_table[MAX_PKT_BURST]; 139 }; 140 141 struct lcore_rx_queue { 142 uint8_t port_id; 143 uint8_t queue_id; 144 } __rte_cache_aligned; 145 146 struct lcore_conf { 147 uint16_t proc_id; 148 uint16_t socket_id; 149 uint16_t nb_queue_list[RTE_MAX_ETHPORTS]; 150 struct ff_port_cfg *port_cfgs; 151 152 uint16_t nb_rx_queue; 153 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 154 uint16_t nb_tx_port; 155 uint16_t tx_port_id[RTE_MAX_ETHPORTS]; 156 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 157 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 158 char *pcap[RTE_MAX_ETHPORTS]; 159 } __rte_cache_aligned; 160 161 static struct lcore_conf lcore_conf; 162 163 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 164 165 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 166 167 static uint16_t rss_reta_size[RTE_MAX_ETHPORTS]; 168 169 struct ff_msg_ring { 170 char ring_name[2][RTE_RING_NAMESIZE]; 171 /* ring[0] for lcore recv msg, other send */ 172 /* ring[1] for lcore send msg, other read */ 173 struct rte_ring *ring[2]; 174 } __rte_cache_aligned; 175 176 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 177 static struct rte_mempool *message_pool; 178 179 struct ff_dpdk_if_context { 180 void *sc; 181 void *ifp; 182 uint16_t port_id; 183 struct ff_hw_features hw_features; 184 } __rte_cache_aligned; 185 186 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 187 188 extern void ff_hardclock(void); 189 190 static void 191 ff_hardclock_job(__rte_unused struct rte_timer *timer, 192 __rte_unused void *arg) { 193 ff_hardclock(); 194 ff_update_current_ts(); 195 } 196 197 struct ff_dpdk_if_context * 198 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 199 { 200 struct ff_dpdk_if_context *ctx; 201 202 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 203 if (ctx == NULL) 204 return NULL; 205 206 ctx->sc = sc; 207 ctx->ifp = ifp; 208 ctx->port_id = cfg->port_id; 209 ctx->hw_features = cfg->hw_features; 210 211 return ctx; 212 } 213 214 void 215 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 216 { 217 free(ctx); 218 } 219 220 static void 221 check_all_ports_link_status(void) 222 { 223 #define CHECK_INTERVAL 100 /* 100ms */ 224 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 225 226 uint8_t portid, count, all_ports_up, print_flag = 0; 227 struct rte_eth_link link; 228 229 printf("\nChecking link status"); 230 fflush(stdout); 231 232 int i, nb_ports; 233 nb_ports = ff_global_cfg.dpdk.nb_ports; 234 for (count = 0; count <= MAX_CHECK_TIME; count++) { 235 all_ports_up = 1; 236 for (i = 0; i < nb_ports; i++) { 237 uint8_t portid = ff_global_cfg.dpdk.portid_list[i]; 238 memset(&link, 0, sizeof(link)); 239 rte_eth_link_get_nowait(portid, &link); 240 241 /* print link status if flag set */ 242 if (print_flag == 1) { 243 if (link.link_status) { 244 printf("Port %d Link Up - speed %u " 245 "Mbps - %s\n", (int)portid, 246 (unsigned)link.link_speed, 247 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 248 ("full-duplex") : ("half-duplex\n")); 249 } else { 250 printf("Port %d Link Down\n", (int)portid); 251 } 252 continue; 253 } 254 /* clear all_ports_up flag if any link down */ 255 if (link.link_status == 0) { 256 all_ports_up = 0; 257 break; 258 } 259 } 260 261 /* after finally printing all link status, get out */ 262 if (print_flag == 1) 263 break; 264 265 if (all_ports_up == 0) { 266 printf("."); 267 fflush(stdout); 268 rte_delay_ms(CHECK_INTERVAL); 269 } 270 271 /* set the print_flag if all ports up or timeout */ 272 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 273 print_flag = 1; 274 printf("done\n"); 275 } 276 } 277 } 278 279 static int 280 init_lcore_conf(void) 281 { 282 uint8_t nb_dev_ports = rte_eth_dev_count(); 283 if (nb_dev_ports == 0) { 284 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 285 } 286 287 if (ff_global_cfg.dpdk.max_portid >= nb_dev_ports) { 288 rte_exit(EXIT_FAILURE, "this machine doesn't have port %d.\n", 289 ff_global_cfg.dpdk.max_portid); 290 } 291 292 lcore_conf.port_cfgs = ff_global_cfg.dpdk.port_cfgs; 293 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 294 295 uint16_t proc_id; 296 for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { 297 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[proc_id]; 298 if (!lcore_config[lcore_id].detected) { 299 rte_exit(EXIT_FAILURE, "lcore %u unavailable\n", lcore_id); 300 } 301 } 302 303 uint16_t socket_id = 0; 304 if (numa_on) { 305 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 306 } 307 308 lcore_conf.socket_id = socket_id; 309 310 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[lcore_conf.proc_id]; 311 int j; 312 for (j = 0; j < ff_global_cfg.dpdk.nb_ports; ++j) { 313 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 314 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 315 316 int queueid = -1; 317 int i; 318 for (i = 0; i < pconf->nb_lcores; i++) { 319 if (pconf->lcore_list[i] == lcore_id) { 320 queueid = i; 321 } 322 } 323 if (queueid < 0) { 324 continue; 325 } 326 printf("lcore: %u, port: %u, queue: %u\n", lcore_id, port_id, queueid); 327 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 328 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 329 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = queueid; 330 lcore_conf.nb_rx_queue++; 331 332 lcore_conf.tx_queue_id[port_id] = queueid; 333 lcore_conf.tx_port_id[lcore_conf.nb_tx_port] = port_id; 334 lcore_conf.nb_tx_port++; 335 336 lcore_conf.pcap[port_id] = pconf->pcap; 337 lcore_conf.nb_queue_list[port_id] = pconf->nb_lcores; 338 } 339 340 return 0; 341 } 342 343 static int 344 init_mem_pool(void) 345 { 346 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 347 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 348 uint32_t nb_tx_queue = nb_lcores; 349 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 350 351 unsigned nb_mbuf = RTE_MAX ( 352 (nb_rx_queue*RX_QUEUE_SIZE + 353 nb_ports*nb_lcores*MAX_PKT_BURST + 354 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 355 nb_lcores*MEMPOOL_CACHE_SIZE + 356 nb_ports*KNI_MBUF_MAX + 357 nb_ports*KNI_QUEUE_SIZE + 358 nb_lcores*nb_ports*ARP_RING_SIZE), 359 (unsigned)8192); 360 361 unsigned socketid = 0; 362 uint16_t i, lcore_id; 363 char s[64]; 364 365 for (i = 0; i < ff_global_cfg.dpdk.nb_procs; i++) { 366 lcore_id = ff_global_cfg.dpdk.proc_lcore[i]; 367 if (numa_on) { 368 socketid = rte_lcore_to_socket_id(lcore_id); 369 } 370 371 if (socketid >= NB_SOCKETS) { 372 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 373 socketid, i, NB_SOCKETS); 374 } 375 376 if (pktmbuf_pool[socketid] != NULL) { 377 continue; 378 } 379 380 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 381 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 382 pktmbuf_pool[socketid] = 383 rte_pktmbuf_pool_create(s, nb_mbuf, 384 MEMPOOL_CACHE_SIZE, 0, 385 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 386 } else { 387 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 388 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 389 } 390 391 if (pktmbuf_pool[socketid] == NULL) { 392 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 393 } else { 394 printf("create mbuf pool on socket %d\n", socketid); 395 } 396 } 397 398 return 0; 399 } 400 401 static struct rte_ring * 402 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 403 { 404 struct rte_ring *ring; 405 406 if (name == NULL) 407 return NULL; 408 409 /* If already create, just attached it */ 410 if (likely((ring = rte_ring_lookup(name)) != NULL)) 411 return ring; 412 413 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 414 return rte_ring_create(name, count, socket_id, flags); 415 } else { 416 return rte_ring_lookup(name); 417 } 418 } 419 420 static int 421 init_arp_ring(void) 422 { 423 int i, j, ret; 424 char name_buf[RTE_RING_NAMESIZE]; 425 int nb_procs = ff_global_cfg.dpdk.nb_procs; 426 int proc_id = ff_global_cfg.dpdk.proc_id; 427 428 /* Allocate arp ring ptr according to eth dev count. */ 429 int nb_dev_ports = rte_eth_dev_count(); 430 for(i = 0; i < nb_procs; ++i) { 431 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 432 proc_id, i); 433 434 arp_ring[i] = rte_zmalloc(name_buf, 435 sizeof(struct rte_ring *) * nb_dev_ports, 436 RTE_CACHE_LINE_SIZE); 437 if (arp_ring[i] == NULL) { 438 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 439 "failed\n", name_buf); 440 } 441 } 442 443 unsigned socketid = lcore_conf.socket_id; 444 445 /* Create ring according to ports actually being used. */ 446 int nb_ports = ff_global_cfg.dpdk.nb_ports; 447 for (j = 0; j < nb_ports; j++) { 448 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 449 450 for(i = 0; i < nb_procs; ++i) { 451 snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id); 452 arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE, 453 socketid, RING_F_SC_DEQ); 454 455 if (arp_ring[i][port_id] == NULL) 456 rte_panic("create ring:%s failed!\n", name_buf); 457 458 printf("create ring:%s success, %u ring entries are now free!\n", 459 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 460 } 461 } 462 463 return 0; 464 } 465 466 static void 467 ff_msg_init(struct rte_mempool *mp, 468 __attribute__((unused)) void *opaque_arg, 469 void *obj, __attribute__((unused)) unsigned i) 470 { 471 struct ff_msg *msg = (struct ff_msg *)obj; 472 msg->msg_type = FF_UNKNOWN; 473 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 474 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 475 } 476 477 static int 478 init_msg_ring(void) 479 { 480 uint16_t i; 481 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 482 unsigned socketid = lcore_conf.socket_id; 483 484 /* Create message buffer pool */ 485 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 486 message_pool = rte_mempool_create(FF_MSG_POOL, 487 MSG_RING_SIZE * 2 * nb_procs, 488 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 489 NULL, NULL, ff_msg_init, NULL, 490 socketid, 0); 491 } else { 492 message_pool = rte_mempool_lookup(FF_MSG_POOL); 493 } 494 495 if (message_pool == NULL) { 496 rte_panic("Create msg mempool failed\n"); 497 } 498 499 for(i = 0; i < nb_procs; ++i) { 500 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 501 "%s%u", FF_MSG_RING_IN, i); 502 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 503 "%s%u", FF_MSG_RING_OUT, i); 504 505 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 506 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 507 if (msg_ring[i].ring[0] == NULL) 508 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 509 510 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 511 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 512 if (msg_ring[i].ring[1] == NULL) 513 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 514 } 515 516 return 0; 517 } 518 519 static int 520 init_kni(void) 521 { 522 int nb_ports = rte_eth_dev_count(); 523 kni_accept = 0; 524 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 525 kni_accept = 1; 526 527 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 528 ff_global_cfg.kni.udp_port); 529 530 unsigned socket_id = lcore_conf.socket_id; 531 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 532 533 nb_ports = ff_global_cfg.dpdk.nb_ports; 534 int i, ret; 535 for (i = 0; i < nb_ports; i++) { 536 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 537 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 538 } 539 540 return 0; 541 } 542 543 static void 544 set_rss_table(uint8_t port_id, uint16_t reta_size, uint16_t nb_queues) 545 { 546 if (reta_size == 0) { 547 return; 548 } 549 550 int reta_conf_size = RTE_MAX(1, reta_size / RTE_RETA_GROUP_SIZE); 551 struct rte_eth_rss_reta_entry64 reta_conf[reta_conf_size]; 552 553 /* config HW indirection table */ 554 unsigned i, j, hash=0; 555 for (i = 0; i < reta_conf_size; i++) { 556 reta_conf[i].mask = ~0ULL; 557 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 558 reta_conf[i].reta[j] = hash++ % nb_queues; 559 } 560 } 561 562 if (rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size)) { 563 rte_exit(EXIT_FAILURE, "port[%d], failed to update rss table\n", 564 port_id); 565 } 566 } 567 568 static int 569 init_port_start(void) 570 { 571 int nb_ports = ff_global_cfg.dpdk.nb_ports; 572 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 573 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 574 uint16_t i; 575 576 for (i = 0; i < nb_ports; i++) { 577 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 578 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 579 uint16_t nb_queues = pconf->nb_lcores; 580 581 struct rte_eth_dev_info dev_info; 582 rte_eth_dev_info_get(port_id, &dev_info); 583 584 if (nb_queues > dev_info.max_rx_queues) { 585 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 586 nb_queues, 587 dev_info.max_rx_queues); 588 } 589 590 if (nb_queues > dev_info.max_tx_queues) { 591 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 592 nb_queues, 593 dev_info.max_tx_queues); 594 } 595 596 struct ether_addr addr; 597 rte_eth_macaddr_get(port_id, &addr); 598 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 599 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 600 (unsigned)port_id, 601 addr.addr_bytes[0], addr.addr_bytes[1], 602 addr.addr_bytes[2], addr.addr_bytes[3], 603 addr.addr_bytes[4], addr.addr_bytes[5]); 604 605 rte_memcpy(pconf->mac, 606 addr.addr_bytes, ETHER_ADDR_LEN); 607 608 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 609 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 610 ETH_TXQ_FLAGS_NOREFCOUNT; 611 612 /* Disable features that are not supported by port's HW */ 613 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 614 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 615 } 616 617 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 618 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 619 } 620 621 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 622 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 623 } 624 625 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 626 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 627 } 628 629 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 630 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 631 } 632 633 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 634 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 635 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 636 } 637 638 struct rte_eth_conf port_conf = {0}; 639 640 /* Set RSS mode */ 641 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 642 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 643 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 644 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 645 646 /* Set Rx VLAN stripping */ 647 if (ff_global_cfg.dpdk.vlan_strip) { 648 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 649 port_conf.rxmode.hw_vlan_strip = 1; 650 } 651 } 652 653 /* Enable HW CRC stripping */ 654 port_conf.rxmode.hw_strip_crc = 1; 655 656 /* FIXME: Enable TCP LRO ?*/ 657 #if 0 658 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 659 printf("LRO is supported\n"); 660 port_conf.rxmode.enable_lro = 1; 661 pconf->hw_features.rx_lro = 1; 662 } 663 #endif 664 665 /* Set Rx checksum checking */ 666 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 667 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 668 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 669 printf("RX checksum offload supported\n"); 670 port_conf.rxmode.hw_ip_checksum = 1; 671 pconf->hw_features.rx_csum = 1; 672 } 673 674 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 675 printf("TX ip checksum offload supported\n"); 676 pconf->hw_features.tx_csum_ip = 1; 677 } 678 679 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 680 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 681 printf("TX TCP&UDP checksum offload supported\n"); 682 pconf->hw_features.tx_csum_l4 = 1; 683 } 684 685 if (ff_global_cfg.dpdk.tso) { 686 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 687 printf("TSO is supported\n"); 688 pconf->hw_features.tx_tso = 1; 689 } 690 } else { 691 printf("TSO is disabled\n"); 692 } 693 694 if (dev_info.reta_size) { 695 /* reta size must be power of 2 */ 696 assert((dev_info.reta_size & (dev_info.reta_size - 1)) == 0); 697 698 rss_reta_size[port_id] = dev_info.reta_size; 699 printf("port[%d]: rss table size: %d\n", port_id, 700 dev_info.reta_size); 701 } 702 703 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 704 continue; 705 } 706 707 int ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &port_conf); 708 if (ret != 0) { 709 return ret; 710 } 711 uint16_t q; 712 for (q = 0; q < nb_queues; q++) { 713 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 714 socketid, &dev_info.default_txconf); 715 if (ret < 0) { 716 return ret; 717 } 718 719 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 720 socketid, &dev_info.default_rxconf, mbuf_pool); 721 if (ret < 0) { 722 return ret; 723 } 724 } 725 726 ret = rte_eth_dev_start(port_id); 727 if (ret < 0) { 728 return ret; 729 } 730 731 if (nb_queues > 1) { 732 /* set HW rss hash function to Toeplitz. */ 733 if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) { 734 struct rte_eth_hash_filter_info info = {0}; 735 info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG; 736 info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; 737 738 if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH, 739 RTE_ETH_FILTER_SET, &info) < 0) { 740 rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n", 741 port_id); 742 } 743 } 744 745 set_rss_table(port_id, dev_info.reta_size, nb_queues); 746 } 747 748 /* Enable RX in promiscuous mode for the Ethernet device. */ 749 if (ff_global_cfg.dpdk.promiscuous) { 750 rte_eth_promiscuous_enable(port_id); 751 ret = rte_eth_promiscuous_get(port_id); 752 if (ret == 1) { 753 printf("set port %u to promiscuous mode ok\n", port_id); 754 } else { 755 printf("set port %u to promiscuous mode error\n", port_id); 756 } 757 } 758 759 /* Enable pcap dump */ 760 if (pconf->pcap) { 761 ff_enable_pcap(pconf->pcap); 762 } 763 } 764 765 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 766 check_all_ports_link_status(); 767 } 768 769 return 0; 770 } 771 772 static int 773 init_clock(void) 774 { 775 rte_timer_subsystem_init(); 776 uint64_t hz = rte_get_timer_hz(); 777 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 778 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 779 780 rte_timer_init(&freebsd_clock); 781 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 782 rte_lcore_id(), &ff_hardclock_job, NULL); 783 784 ff_update_current_ts(); 785 786 return 0; 787 } 788 789 int 790 ff_dpdk_init(int argc, char **argv) 791 { 792 if (ff_global_cfg.dpdk.nb_procs < 1 || 793 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 794 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 795 ff_global_cfg.dpdk.proc_id < 0) { 796 printf("param num_procs[%d] or proc_id[%d] error!\n", 797 ff_global_cfg.dpdk.nb_procs, 798 ff_global_cfg.dpdk.proc_id); 799 exit(1); 800 } 801 802 int ret = rte_eal_init(argc, argv); 803 if (ret < 0) { 804 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 805 } 806 807 numa_on = ff_global_cfg.dpdk.numa_on; 808 809 init_lcore_conf(); 810 811 init_mem_pool(); 812 813 init_arp_ring(); 814 815 init_msg_ring(); 816 817 enable_kni = ff_global_cfg.kni.enable; 818 if (enable_kni) { 819 init_kni(); 820 } 821 822 ret = init_port_start(); 823 if (ret < 0) { 824 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 825 } 826 827 init_clock(); 828 829 return 0; 830 } 831 832 static void 833 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 834 { 835 uint8_t rx_csum = ctx->hw_features.rx_csum; 836 if (rx_csum) { 837 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 838 return; 839 } 840 } 841 842 /* 843 * FIXME: should we save pkt->vlan_tci 844 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 845 */ 846 847 void *data = rte_pktmbuf_mtod(pkt, void*); 848 uint16_t len = rte_pktmbuf_data_len(pkt); 849 850 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 851 if (hdr == NULL) { 852 rte_pktmbuf_free(pkt); 853 return; 854 } 855 856 struct rte_mbuf *pn = pkt->next; 857 void *prev = hdr; 858 while(pn != NULL) { 859 data = rte_pktmbuf_mtod(pkt, void*); 860 len = rte_pktmbuf_data_len(pkt); 861 862 void *mb = ff_mbuf_get(prev, data, len); 863 if (mb == NULL) { 864 ff_mbuf_free(hdr); 865 rte_pktmbuf_free(pkt); 866 return; 867 } 868 pn = pn->next; 869 prev = mb; 870 } 871 872 ff_veth_process_packet(ctx->ifp, hdr); 873 } 874 875 static enum FilterReturn 876 protocol_filter(const void *data, uint16_t len) 877 { 878 if(len < sizeof(struct ether_hdr)) 879 return FILTER_UNKNOWN; 880 881 const struct ether_hdr *hdr; 882 hdr = (const struct ether_hdr *)data; 883 884 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 885 return FILTER_ARP; 886 887 if (!enable_kni) { 888 return FILTER_UNKNOWN; 889 } 890 891 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 892 return FILTER_UNKNOWN; 893 894 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 895 len - sizeof(struct ether_hdr)); 896 } 897 898 static inline void 899 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 900 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 901 { 902 struct lcore_conf *qconf = &lcore_conf; 903 904 uint16_t i; 905 for (i = 0; i < count; i++) { 906 struct rte_mbuf *rtem = bufs[i]; 907 908 if (unlikely(qconf->pcap[port_id] != NULL)) { 909 ff_dump_packets(qconf->pcap[port_id], rtem); 910 } 911 912 void *data = rte_pktmbuf_mtod(rtem, void*); 913 uint16_t len = rte_pktmbuf_data_len(rtem); 914 915 enum FilterReturn filter = protocol_filter(data, len); 916 if (filter == FILTER_ARP) { 917 struct rte_mempool *mbuf_pool; 918 struct rte_mbuf *mbuf_clone; 919 if (pkts_from_ring == 0) { 920 uint16_t i; 921 uint16_t nb_queues = qconf->nb_queue_list[port_id]; 922 for(i = 0; i < nb_queues; ++i) { 923 if(i == queue_id) 924 continue; 925 926 unsigned socket_id = 0; 927 if (numa_on) { 928 uint16_t lcore_id = qconf->port_cfgs[port_id].lcore_list[i]; 929 socket_id = rte_lcore_to_socket_id(lcore_id); 930 } 931 mbuf_pool = pktmbuf_pool[socket_id]; 932 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 933 if(mbuf_clone) { 934 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 935 if (ret < 0) 936 rte_pktmbuf_free(mbuf_clone); 937 } 938 } 939 } 940 941 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 942 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 943 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 944 if(mbuf_clone) { 945 ff_kni_enqueue(port_id, mbuf_clone); 946 } 947 } 948 949 ff_veth_input(ctx, rtem); 950 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 951 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 952 ff_kni_enqueue(port_id, rtem); 953 } else { 954 ff_veth_input(ctx, rtem); 955 } 956 } 957 } 958 959 static inline int 960 process_arp_ring(uint8_t port_id, uint16_t queue_id, 961 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 962 { 963 /* read packet from ring buf and to process */ 964 uint16_t nb_rb; 965 nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 966 (void **)pkts_burst, MAX_PKT_BURST); 967 968 if(nb_rb > 0) { 969 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 970 } 971 972 return 0; 973 } 974 975 static inline void 976 handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id) 977 { 978 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 979 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 980 msg->sysctl.newlen); 981 982 if (ret < 0) { 983 msg->result = errno; 984 } else { 985 msg->result = 0; 986 } 987 988 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 989 } 990 991 static inline void 992 handle_ioctl_msg(struct ff_msg *msg, uint16_t proc_id) 993 { 994 int fd, ret; 995 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 996 if (fd < 0) { 997 ret = -1; 998 goto done; 999 } 1000 1001 ret = ff_ioctl(fd, msg->ioctl.cmd, msg->ioctl.data); 1002 1003 ff_close(fd); 1004 1005 done: 1006 if (ret < 0) { 1007 msg->result = errno; 1008 } else { 1009 msg->result = 0; 1010 } 1011 1012 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1013 } 1014 1015 static inline void 1016 handle_route_msg(struct ff_msg *msg, uint16_t proc_id) 1017 { 1018 msg->result = ff_rtioctl(msg->route.fib, msg->route.data, 1019 &msg->route.len, msg->route.maxlen); 1020 1021 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1022 } 1023 1024 static struct ff_top_args ff_status; 1025 static inline void 1026 handle_top_msg(struct ff_msg *msg, uint16_t proc_id) 1027 { 1028 msg->top = ff_status; 1029 msg->result = 0; 1030 1031 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1032 } 1033 1034 static inline void 1035 handle_default_msg(struct ff_msg *msg, uint16_t proc_id) 1036 { 1037 msg->result = EINVAL; 1038 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1039 } 1040 1041 static inline void 1042 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1043 { 1044 switch (msg->msg_type) { 1045 case FF_SYSCTL: 1046 handle_sysctl_msg(msg, proc_id); 1047 break; 1048 case FF_IOCTL: 1049 handle_ioctl_msg(msg, proc_id); 1050 break; 1051 case FF_ROUTE: 1052 handle_route_msg(msg, proc_id); 1053 break; 1054 case FF_TOP: 1055 handle_top_msg(msg, proc_id); 1056 break; 1057 default: 1058 handle_default_msg(msg, proc_id); 1059 break; 1060 } 1061 } 1062 1063 static inline int 1064 process_msg_ring(uint16_t proc_id) 1065 { 1066 void *msg; 1067 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1068 1069 if (unlikely(ret == 0)) { 1070 handle_msg((struct ff_msg *)msg, proc_id); 1071 } 1072 1073 return 0; 1074 } 1075 1076 /* Send burst of packets on an output interface */ 1077 static inline int 1078 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1079 { 1080 struct rte_mbuf **m_table; 1081 int ret; 1082 uint16_t queueid; 1083 1084 queueid = qconf->tx_queue_id[port]; 1085 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1086 1087 if (unlikely(qconf->pcap[port] != NULL)) { 1088 uint16_t i; 1089 for (i = 0; i < n; i++) { 1090 ff_dump_packets(qconf->pcap[port], m_table[i]); 1091 } 1092 } 1093 1094 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1095 if (unlikely(ret < n)) { 1096 do { 1097 rte_pktmbuf_free(m_table[ret]); 1098 } while (++ret < n); 1099 } 1100 1101 return 0; 1102 } 1103 1104 /* Enqueue a single packet, and send burst if queue is filled */ 1105 static inline int 1106 send_single_packet(struct rte_mbuf *m, uint8_t port) 1107 { 1108 uint16_t len; 1109 struct lcore_conf *qconf; 1110 1111 qconf = &lcore_conf; 1112 len = qconf->tx_mbufs[port].len; 1113 qconf->tx_mbufs[port].m_table[len] = m; 1114 len++; 1115 1116 /* enough pkts to be sent */ 1117 if (unlikely(len == MAX_PKT_BURST)) { 1118 send_burst(qconf, MAX_PKT_BURST, port); 1119 len = 0; 1120 } 1121 1122 qconf->tx_mbufs[port].len = len; 1123 return 0; 1124 } 1125 1126 int 1127 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1128 int total) 1129 { 1130 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1131 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1132 if (head == NULL) { 1133 ff_mbuf_free(m); 1134 return -1; 1135 } 1136 1137 head->pkt_len = total; 1138 head->nb_segs = 0; 1139 1140 int off = 0; 1141 struct rte_mbuf *cur = head, *prev = NULL; 1142 while(total > 0) { 1143 if (cur == NULL) { 1144 cur = rte_pktmbuf_alloc(mbuf_pool); 1145 if (cur == NULL) { 1146 rte_pktmbuf_free(head); 1147 ff_mbuf_free(m); 1148 return -1; 1149 } 1150 } 1151 1152 void *data = rte_pktmbuf_mtod(cur, void*); 1153 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1154 int ret = ff_mbuf_copydata(m, data, off, len); 1155 if (ret < 0) { 1156 rte_pktmbuf_free(head); 1157 ff_mbuf_free(m); 1158 return -1; 1159 } 1160 1161 if (prev != NULL) { 1162 prev->next = cur; 1163 } 1164 prev = cur; 1165 1166 cur->data_len = len; 1167 off += len; 1168 total -= len; 1169 head->nb_segs++; 1170 cur = NULL; 1171 } 1172 1173 struct ff_tx_offload offload = {0}; 1174 ff_mbuf_tx_offload(m, &offload); 1175 1176 if (offload.ip_csum) { 1177 head->ol_flags |= PKT_TX_IP_CKSUM; 1178 head->l2_len = sizeof(struct ether_hdr); 1179 head->l3_len = sizeof(struct ipv4_hdr); 1180 } 1181 1182 if (ctx->hw_features.tx_csum_l4) { 1183 if (offload.tcp_csum) { 1184 head->ol_flags |= PKT_TX_TCP_CKSUM; 1185 head->l2_len = sizeof(struct ether_hdr); 1186 head->l3_len = sizeof(struct ipv4_hdr); 1187 } 1188 1189 if (offload.tso_seg_size) { 1190 head->ol_flags |= PKT_TX_TCP_SEG; 1191 head->l4_len = sizeof(struct tcp_hdr); 1192 head->tso_segsz = offload.tso_seg_size; 1193 } 1194 1195 if (offload.udp_csum) { 1196 head->ol_flags |= PKT_TX_UDP_CKSUM; 1197 head->l2_len = sizeof(struct ether_hdr); 1198 head->l3_len = sizeof(struct ipv4_hdr); 1199 } 1200 } 1201 1202 ff_mbuf_free(m); 1203 1204 return send_single_packet(head, ctx->port_id); 1205 } 1206 1207 static int 1208 main_loop(void *arg) 1209 { 1210 struct loop_routine *lr = (struct loop_routine *)arg; 1211 1212 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1213 unsigned lcore_id; 1214 uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc; 1215 int i, j, nb_rx, idle; 1216 uint8_t port_id, queue_id; 1217 struct lcore_conf *qconf; 1218 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1219 US_PER_S * BURST_TX_DRAIN_US; 1220 struct ff_dpdk_if_context *ctx; 1221 1222 prev_tsc = 0; 1223 usch_tsc = 0; 1224 1225 lcore_id = rte_lcore_id(); 1226 qconf = &lcore_conf; 1227 1228 if (qconf->nb_rx_queue == 0) { 1229 printf("lcore %u has nothing to do\n", lcore_id); 1230 return 0; 1231 } 1232 1233 while (1) { 1234 cur_tsc = rte_rdtsc(); 1235 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1236 rte_timer_manage(); 1237 } 1238 1239 idle = 1; 1240 sys_tsc = 0; 1241 usr_tsc = 0; 1242 1243 /* 1244 * TX burst queue drain 1245 */ 1246 diff_tsc = cur_tsc - prev_tsc; 1247 if (unlikely(diff_tsc > drain_tsc)) { 1248 for (i = 0; i < qconf->nb_tx_port; i++) { 1249 port_id = qconf->tx_port_id[i]; 1250 if (qconf->tx_mbufs[port_id].len == 0) 1251 continue; 1252 1253 idle = 0; 1254 1255 send_burst(qconf, 1256 qconf->tx_mbufs[port_id].len, 1257 port_id); 1258 qconf->tx_mbufs[port_id].len = 0; 1259 } 1260 1261 prev_tsc = cur_tsc; 1262 } 1263 1264 /* 1265 * Read packet from RX queues 1266 */ 1267 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1268 port_id = qconf->rx_queue_list[i].port_id; 1269 queue_id = qconf->rx_queue_list[i].queue_id; 1270 ctx = veth_ctx[port_id]; 1271 1272 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1273 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1274 } 1275 1276 process_arp_ring(port_id, queue_id, pkts_burst, ctx); 1277 1278 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1279 MAX_PKT_BURST); 1280 if (nb_rx == 0) 1281 continue; 1282 1283 idle = 0; 1284 1285 /* Prefetch first packets */ 1286 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1287 rte_prefetch0(rte_pktmbuf_mtod( 1288 pkts_burst[j], void *)); 1289 } 1290 1291 /* Prefetch and handle already prefetched packets */ 1292 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1293 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1294 j + PREFETCH_OFFSET], void *)); 1295 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1296 } 1297 1298 /* Handle remaining prefetched packets */ 1299 for (; j < nb_rx; j++) { 1300 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1301 } 1302 } 1303 1304 process_msg_ring(qconf->proc_id); 1305 1306 div_tsc = rte_rdtsc(); 1307 1308 if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc > drain_tsc))) { 1309 usch_tsc = cur_tsc; 1310 lr->loop(lr->arg); 1311 } 1312 1313 end_tsc = rte_rdtsc(); 1314 1315 if (usch_tsc == cur_tsc) { 1316 usr_tsc = end_tsc - div_tsc; 1317 } 1318 1319 if (!idle) { 1320 sys_tsc = div_tsc - cur_tsc; 1321 ff_status.sys_tsc += sys_tsc; 1322 } 1323 1324 ff_status.usr_tsc += usr_tsc; 1325 ff_status.work_tsc += end_tsc - cur_tsc; 1326 ff_status.idle_tsc += end_tsc - cur_tsc - usr_tsc - sys_tsc; 1327 1328 ff_status.loops++; 1329 } 1330 } 1331 1332 int 1333 ff_dpdk_if_up(void) { 1334 int i; 1335 struct lcore_conf *qconf = &lcore_conf; 1336 for (i = 0; i < qconf->nb_tx_port; i++) { 1337 uint16_t port_id = qconf->tx_port_id[i]; 1338 1339 struct ff_port_cfg *pconf = &qconf->port_cfgs[port_id]; 1340 veth_ctx[port_id] = ff_veth_attach(pconf); 1341 if (veth_ctx[port_id] == NULL) { 1342 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1343 } 1344 } 1345 1346 return 0; 1347 } 1348 1349 void 1350 ff_dpdk_run(loop_func_t loop, void *arg) { 1351 struct loop_routine *lr = rte_malloc(NULL, 1352 sizeof(struct loop_routine), 0); 1353 lr->loop = loop; 1354 lr->arg = arg; 1355 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1356 rte_eal_mp_wait_lcore(); 1357 rte_free(lr); 1358 } 1359 1360 void 1361 ff_dpdk_pktmbuf_free(void *m) 1362 { 1363 rte_pktmbuf_free((struct rte_mbuf *)m); 1364 } 1365 1366 static uint32_t 1367 toeplitz_hash(unsigned keylen, const uint8_t *key, 1368 unsigned datalen, const uint8_t *data) 1369 { 1370 uint32_t hash = 0, v; 1371 u_int i, b; 1372 1373 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1374 1375 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1376 for (i = 0; i < datalen; i++) { 1377 for (b = 0; b < 8; b++) { 1378 if (data[i] & (1<<(7-b))) 1379 hash ^= v; 1380 v <<= 1; 1381 if ((i + 4) < keylen && 1382 (key[i+4] & (1<<(7-b)))) 1383 v |= 1; 1384 } 1385 } 1386 return (hash); 1387 } 1388 1389 int 1390 ff_rss_check(void *softc, uint32_t saddr, uint32_t daddr, 1391 uint16_t sport, uint16_t dport) 1392 { 1393 struct lcore_conf *qconf = &lcore_conf; 1394 struct ff_dpdk_if_context *ctx = ff_veth_softc_to_hostc(softc); 1395 uint16_t nb_queues = qconf->nb_queue_list[ctx->port_id]; 1396 1397 if (nb_queues <= 1) { 1398 return 1; 1399 } 1400 1401 uint16_t reta_size = rss_reta_size[ctx->port_id]; 1402 uint16_t queueid = qconf->tx_queue_id[ctx->port_id]; 1403 1404 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1405 sizeof(dport)]; 1406 1407 unsigned datalen = 0; 1408 1409 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1410 datalen += sizeof(saddr); 1411 1412 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1413 datalen += sizeof(daddr); 1414 1415 bcopy(&sport, &data[datalen], sizeof(sport)); 1416 datalen += sizeof(sport); 1417 1418 bcopy(&dport, &data[datalen], sizeof(dport)); 1419 datalen += sizeof(dport); 1420 1421 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), 1422 default_rsskey_40bytes, datalen, data); 1423 1424 return ((hash & (reta_size - 1)) % nb_queues) == queueid; 1425 } 1426