1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 #include <assert.h> 27 28 #include <rte_common.h> 29 #include <rte_byteorder.h> 30 #include <rte_log.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_memzone.h> 34 #include <rte_config.h> 35 #include <rte_eal.h> 36 #include <rte_pci.h> 37 #include <rte_mbuf.h> 38 #include <rte_memory.h> 39 #include <rte_lcore.h> 40 #include <rte_launch.h> 41 #include <rte_ethdev.h> 42 #include <rte_debug.h> 43 #include <rte_common.h> 44 #include <rte_ether.h> 45 #include <rte_malloc.h> 46 #include <rte_cycles.h> 47 #include <rte_timer.h> 48 #include <rte_thash.h> 49 #include <rte_ip.h> 50 #include <rte_tcp.h> 51 #include <rte_udp.h> 52 53 #include "ff_dpdk_if.h" 54 #include "ff_dpdk_pcap.h" 55 #include "ff_dpdk_kni.h" 56 #include "ff_config.h" 57 #include "ff_veth.h" 58 #include "ff_host_interface.h" 59 #include "ff_msg.h" 60 #include "ff_api.h" 61 62 #define MEMPOOL_CACHE_SIZE 256 63 64 #define ARP_RING_SIZE 2048 65 66 #define MSG_RING_SIZE 32 67 68 /* 69 * Configurable number of RX/TX ring descriptors 70 */ 71 #define RX_QUEUE_SIZE 512 72 #define TX_QUEUE_SIZE 512 73 74 #define MAX_PKT_BURST 32 75 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 76 77 /* 78 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 79 */ 80 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 81 82 #define NB_SOCKETS 8 83 84 /* Configure how many packets ahead to prefetch, when reading packets */ 85 #define PREFETCH_OFFSET 3 86 87 #define MAX_RX_QUEUE_PER_LCORE 16 88 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 89 #define MAX_RX_QUEUE_PER_PORT 128 90 91 #define KNI_MBUF_MAX 2048 92 #define KNI_QUEUE_SIZE 2048 93 94 static int enable_kni; 95 static int kni_accept; 96 97 static int numa_on; 98 99 static struct rte_timer freebsd_clock; 100 101 // Mellanox Linux's driver key 102 static uint8_t default_rsskey_40bytes[40] = { 103 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 104 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 105 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 106 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 107 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 108 }; 109 110 static struct rte_eth_conf default_port_conf = { 111 .rxmode = { 112 .mq_mode = ETH_MQ_RX_RSS, 113 .max_rx_pkt_len = ETHER_MAX_LEN, 114 .split_hdr_size = 0, /**< hdr buf size */ 115 .header_split = 0, /**< Header Split disabled */ 116 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 117 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 118 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 119 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 120 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 121 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 122 .enable_lro = 0, /**< LRO disabled */ 123 }, 124 .rx_adv_conf = { 125 .rss_conf = { 126 .rss_key = default_rsskey_40bytes, 127 .rss_key_len = 40, 128 .rss_hf = ETH_RSS_PROTO_MASK, 129 }, 130 }, 131 .txmode = { 132 .mq_mode = ETH_MQ_TX_NONE, 133 }, 134 }; 135 136 struct mbuf_table { 137 uint16_t len; 138 struct rte_mbuf *m_table[MAX_PKT_BURST]; 139 }; 140 141 struct lcore_rx_queue { 142 uint8_t port_id; 143 uint8_t queue_id; 144 } __rte_cache_aligned; 145 146 struct lcore_conf { 147 uint16_t proc_id; 148 uint16_t socket_id; 149 int16_t nb_queue_list[RTE_MAX_ETHPORTS]; 150 struct ff_port_cfg *port_cfgs; 151 152 uint16_t nb_rx_queue; 153 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 154 int16_t tx_queue_id[RTE_MAX_ETHPORTS]; 155 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 156 char *pcap[RTE_MAX_ETHPORTS]; 157 } __rte_cache_aligned; 158 159 static struct lcore_conf lcore_conf; 160 161 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 162 163 static struct rte_ring **arp_ring[RTE_MAX_LCORE]; 164 165 static uint16_t rss_reta_size[RTE_MAX_ETHPORTS]; 166 167 struct ff_msg_ring { 168 char ring_name[2][RTE_RING_NAMESIZE]; 169 /* ring[0] for lcore recv msg, other send */ 170 /* ring[1] for lcore send msg, other read */ 171 struct rte_ring *ring[2]; 172 } __rte_cache_aligned; 173 174 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 175 static struct rte_mempool *message_pool; 176 177 struct ff_dpdk_if_context { 178 void *sc; 179 void *ifp; 180 uint16_t port_id; 181 struct ff_hw_features hw_features; 182 } __rte_cache_aligned; 183 184 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 185 186 extern void ff_hardclock(void); 187 188 static void 189 ff_hardclock_job(__rte_unused struct rte_timer *timer, 190 __rte_unused void *arg) { 191 ff_hardclock(); 192 ff_update_current_ts(); 193 } 194 195 struct ff_dpdk_if_context * 196 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 197 { 198 struct ff_dpdk_if_context *ctx; 199 200 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 201 if (ctx == NULL) 202 return NULL; 203 204 ctx->sc = sc; 205 ctx->ifp = ifp; 206 ctx->port_id = cfg->port_id; 207 ctx->hw_features = cfg->hw_features; 208 209 return ctx; 210 } 211 212 void 213 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 214 { 215 free(ctx); 216 } 217 218 static void 219 check_all_ports_link_status(void) 220 { 221 #define CHECK_INTERVAL 100 /* 100ms */ 222 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 223 224 uint8_t portid, count, all_ports_up, print_flag = 0; 225 struct rte_eth_link link; 226 227 printf("\nChecking link status"); 228 fflush(stdout); 229 230 int i, nb_ports; 231 nb_ports = ff_global_cfg.dpdk.nb_ports; 232 for (count = 0; count <= MAX_CHECK_TIME; count++) { 233 all_ports_up = 1; 234 for (i = 0; i < nb_ports; i++) { 235 uint8_t portid = ff_global_cfg.dpdk.portid_list[i]; 236 memset(&link, 0, sizeof(link)); 237 rte_eth_link_get_nowait(portid, &link); 238 239 /* print link status if flag set */ 240 if (print_flag == 1) { 241 if (link.link_status) { 242 printf("Port %d Link Up - speed %u " 243 "Mbps - %s\n", (int)portid, 244 (unsigned)link.link_speed, 245 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 246 ("full-duplex") : ("half-duplex\n")); 247 } else { 248 printf("Port %d Link Down\n", (int)portid); 249 } 250 continue; 251 } 252 /* clear all_ports_up flag if any link down */ 253 if (link.link_status == 0) { 254 all_ports_up = 0; 255 break; 256 } 257 } 258 259 /* after finally printing all link status, get out */ 260 if (print_flag == 1) 261 break; 262 263 if (all_ports_up == 0) { 264 printf("."); 265 fflush(stdout); 266 rte_delay_ms(CHECK_INTERVAL); 267 } 268 269 /* set the print_flag if all ports up or timeout */ 270 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 271 print_flag = 1; 272 printf("done\n"); 273 } 274 } 275 } 276 277 static int 278 init_lcore_conf(void) 279 { 280 /* 281 * set all elements in tx_queue_id to -1, so we can use this array to check 282 * if port is processed by this core. 283 */ 284 int k; 285 for (k = 0; k < RTE_MAX_ETHPORTS; ++k) { 286 lcore_conf.tx_queue_id[k] = -1; 287 lcore_conf.nb_queue_list[k] = -1; 288 } 289 lcore_conf.port_cfgs = ff_global_cfg.dpdk.port_cfgs; 290 291 uint8_t nb_dev_ports = rte_eth_dev_count(); 292 if (nb_dev_ports == 0) { 293 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 294 } 295 if (ff_global_cfg.dpdk.max_portid >= nb_dev_ports) { 296 rte_exit(EXIT_FAILURE, "this machine doesn't have port %d.\n", 297 ff_global_cfg.dpdk.max_portid); 298 } 299 300 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 301 302 uint16_t proc_id; 303 for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { 304 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[proc_id]; 305 if (!lcore_config[lcore_id].detected) { 306 rte_exit(EXIT_FAILURE, "lcore %u unavailable\n", lcore_id); 307 } 308 } 309 310 uint16_t socket_id = 0; 311 if (numa_on) { 312 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 313 } 314 315 lcore_conf.socket_id = socket_id; 316 317 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[lcore_conf.proc_id]; 318 int j; 319 for (j = 0; j < ff_global_cfg.dpdk.nb_ports; ++j) { 320 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 321 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 322 323 int queueid = -1; 324 int i; 325 for (i = 0; i < pconf->nb_lcores; i++) { 326 if (pconf->lcore_list[i] == lcore_id) { 327 queueid = i; 328 } 329 } 330 if (queueid < 0) { 331 continue; 332 } 333 printf("lcore: %u, port: %u, queue: %u\n", lcore_id, port_id, queueid); 334 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 335 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 336 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = queueid; 337 lcore_conf.nb_rx_queue++; 338 339 lcore_conf.tx_queue_id[port_id] = queueid; 340 341 lcore_conf.pcap[port_id] = pconf->pcap; 342 lcore_conf.nb_queue_list[port_id] = pconf->nb_lcores; 343 } 344 345 return 0; 346 } 347 348 static int 349 init_mem_pool(void) 350 { 351 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 352 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 353 uint32_t nb_tx_queue = nb_lcores; 354 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 355 356 unsigned nb_mbuf = RTE_MAX ( 357 (nb_rx_queue*RX_QUEUE_SIZE + 358 nb_ports*nb_lcores*MAX_PKT_BURST + 359 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 360 nb_lcores*MEMPOOL_CACHE_SIZE + 361 nb_ports*KNI_MBUF_MAX + 362 nb_ports*KNI_QUEUE_SIZE + 363 nb_lcores*nb_ports*ARP_RING_SIZE), 364 (unsigned)8192); 365 366 unsigned socketid = 0; 367 uint16_t i, lcore_id; 368 char s[64]; 369 370 for (i = 0; i < ff_global_cfg.dpdk.nb_procs; i++) { 371 lcore_id = ff_global_cfg.dpdk.proc_lcore[i]; 372 if (numa_on) { 373 socketid = rte_lcore_to_socket_id(lcore_id); 374 } 375 376 if (socketid >= NB_SOCKETS) { 377 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 378 socketid, i, NB_SOCKETS); 379 } 380 381 if (pktmbuf_pool[socketid] != NULL) { 382 continue; 383 } 384 385 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 386 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 387 pktmbuf_pool[socketid] = 388 rte_pktmbuf_pool_create(s, nb_mbuf, 389 MEMPOOL_CACHE_SIZE, 0, 390 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 391 } else { 392 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 393 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 394 } 395 396 if (pktmbuf_pool[socketid] == NULL) { 397 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 398 } else { 399 printf("create mbuf pool on socket %d\n", socketid); 400 } 401 } 402 403 return 0; 404 } 405 406 static struct rte_ring * 407 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 408 { 409 struct rte_ring *ring; 410 411 if (name == NULL) 412 return NULL; 413 414 /* If already create, just attached it */ 415 if (likely((ring = rte_ring_lookup(name)) != NULL)) 416 return ring; 417 418 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 419 return rte_ring_create(name, count, socket_id, flags); 420 } else { 421 return rte_ring_lookup(name); 422 } 423 } 424 425 static int 426 init_arp_ring(void) 427 { 428 int i, j, ret; 429 char name_buf[RTE_RING_NAMESIZE]; 430 int nb_procs = ff_global_cfg.dpdk.nb_procs; 431 int proc_id = ff_global_cfg.dpdk.proc_id; 432 433 /* Allocate arp ring ptr according to eth dev count. */ 434 int nb_dev_ports = rte_eth_dev_count(); 435 for(i = 0; i < nb_procs; ++i) { 436 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_%d_%d", 437 proc_id, i); 438 439 arp_ring[i] = rte_zmalloc(name_buf, 440 sizeof(struct rte_ring *) * nb_dev_ports, 441 RTE_CACHE_LINE_SIZE); 442 if (arp_ring[i] == NULL) { 443 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 444 "failed\n", name_buf); 445 } 446 } 447 448 unsigned socketid = lcore_conf.socket_id; 449 450 /* Create ring according to ports actually being used. */ 451 int nb_ports = ff_global_cfg.dpdk.nb_ports; 452 for (j = 0; j < nb_ports; j++) { 453 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 454 455 for(i = 0; i < nb_procs; ++i) { 456 snprintf(name_buf, RTE_RING_NAMESIZE, "arp_ring_%d_%d", i, port_id); 457 arp_ring[i][port_id] = create_ring(name_buf, ARP_RING_SIZE, 458 socketid, RING_F_SC_DEQ); 459 460 if (arp_ring[i][port_id] == NULL) 461 rte_panic("create ring:%s failed!\n", name_buf); 462 463 printf("create ring:%s success, %u ring entries are now free!\n", 464 name_buf, rte_ring_free_count(arp_ring[i][port_id])); 465 } 466 } 467 468 return 0; 469 } 470 471 static void 472 ff_msg_init(struct rte_mempool *mp, 473 __attribute__((unused)) void *opaque_arg, 474 void *obj, __attribute__((unused)) unsigned i) 475 { 476 struct ff_msg *msg = (struct ff_msg *)obj; 477 msg->msg_type = FF_UNKNOWN; 478 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 479 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 480 } 481 482 static int 483 init_msg_ring(void) 484 { 485 uint16_t i; 486 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 487 unsigned socketid = lcore_conf.socket_id; 488 489 /* Create message buffer pool */ 490 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 491 message_pool = rte_mempool_create(FF_MSG_POOL, 492 MSG_RING_SIZE * 2 * nb_procs, 493 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 494 NULL, NULL, ff_msg_init, NULL, 495 socketid, 0); 496 } else { 497 message_pool = rte_mempool_lookup(FF_MSG_POOL); 498 } 499 500 if (message_pool == NULL) { 501 rte_panic("Create msg mempool failed\n"); 502 } 503 504 for(i = 0; i < nb_procs; ++i) { 505 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 506 "%s%u", FF_MSG_RING_IN, i); 507 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 508 "%s%u", FF_MSG_RING_OUT, i); 509 510 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 511 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 512 if (msg_ring[i].ring[0] == NULL) 513 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 514 515 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 516 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 517 if (msg_ring[i].ring[1] == NULL) 518 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 519 } 520 521 return 0; 522 } 523 524 static int 525 init_kni(void) 526 { 527 int nb_ports = rte_eth_dev_count(); 528 kni_accept = 0; 529 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 530 kni_accept = 1; 531 532 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 533 ff_global_cfg.kni.udp_port); 534 535 unsigned socket_id = lcore_conf.socket_id; 536 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 537 538 nb_ports = ff_global_cfg.dpdk.nb_ports; 539 int i, ret; 540 for (i = 0; i < nb_ports; i++) { 541 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 542 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 543 } 544 545 return 0; 546 } 547 548 static void 549 set_rss_table(uint8_t port_id, uint16_t reta_size, uint16_t nb_queues) 550 { 551 if (reta_size == 0) { 552 return; 553 } 554 555 int reta_conf_size = RTE_MAX(1, reta_size / RTE_RETA_GROUP_SIZE); 556 struct rte_eth_rss_reta_entry64 reta_conf[reta_conf_size]; 557 558 /* config HW indirection table */ 559 unsigned i, j, hash=0; 560 for (i = 0; i < reta_conf_size; i++) { 561 reta_conf[i].mask = ~0ULL; 562 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 563 reta_conf[i].reta[j] = hash++ % nb_queues; 564 } 565 } 566 567 if (rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size)) { 568 rte_exit(EXIT_FAILURE, "port[%d], failed to update rss table\n", 569 port_id); 570 } 571 } 572 573 static int 574 init_port_start(void) 575 { 576 int nb_ports = ff_global_cfg.dpdk.nb_ports; 577 unsigned socketid = rte_lcore_to_socket_id(rte_lcore_id()); 578 struct rte_mempool *mbuf_pool = pktmbuf_pool[socketid]; 579 uint16_t i; 580 581 for (i = 0; i < nb_ports; i++) { 582 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 583 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 584 uint16_t nb_queues = pconf->nb_lcores; 585 586 struct rte_eth_dev_info dev_info; 587 rte_eth_dev_info_get(port_id, &dev_info); 588 589 if (nb_queues > dev_info.max_rx_queues) { 590 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 591 nb_queues, 592 dev_info.max_rx_queues); 593 } 594 595 if (nb_queues > dev_info.max_tx_queues) { 596 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 597 nb_queues, 598 dev_info.max_tx_queues); 599 } 600 601 struct ether_addr addr; 602 rte_eth_macaddr_get(port_id, &addr); 603 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 604 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 605 (unsigned)port_id, 606 addr.addr_bytes[0], addr.addr_bytes[1], 607 addr.addr_bytes[2], addr.addr_bytes[3], 608 addr.addr_bytes[4], addr.addr_bytes[5]); 609 610 rte_memcpy(pconf->mac, 611 addr.addr_bytes, ETHER_ADDR_LEN); 612 613 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 614 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 615 ETH_TXQ_FLAGS_NOREFCOUNT; 616 617 /* Disable features that are not supported by port's HW */ 618 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 619 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 620 } 621 622 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 623 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 624 } 625 626 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 627 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 628 } 629 630 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 631 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 632 } 633 634 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 635 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 636 } 637 638 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 639 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 640 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 641 } 642 643 struct rte_eth_conf port_conf = {0}; 644 645 /* Set RSS mode */ 646 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 647 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 648 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 649 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 650 651 /* Set Rx VLAN stripping */ 652 if (ff_global_cfg.dpdk.vlan_strip) { 653 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 654 port_conf.rxmode.hw_vlan_strip = 1; 655 } 656 } 657 658 /* Enable HW CRC stripping */ 659 port_conf.rxmode.hw_strip_crc = 1; 660 661 /* FIXME: Enable TCP LRO ?*/ 662 #if 0 663 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 664 printf("LRO is supported\n"); 665 port_conf.rxmode.enable_lro = 1; 666 pconf->hw_features.rx_lro = 1; 667 } 668 #endif 669 670 /* Set Rx checksum checking */ 671 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 672 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 673 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 674 printf("RX checksum offload supported\n"); 675 port_conf.rxmode.hw_ip_checksum = 1; 676 pconf->hw_features.rx_csum = 1; 677 } 678 679 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 680 printf("TX ip checksum offload supported\n"); 681 pconf->hw_features.tx_csum_ip = 1; 682 } 683 684 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 685 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 686 printf("TX TCP&UDP checksum offload supported\n"); 687 pconf->hw_features.tx_csum_l4 = 1; 688 } 689 690 if (ff_global_cfg.dpdk.tso) { 691 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 692 printf("TSO is supported\n"); 693 pconf->hw_features.tx_tso = 1; 694 } 695 } else { 696 printf("TSO is disabled\n"); 697 } 698 699 if (dev_info.reta_size) { 700 /* reta size must be power of 2 */ 701 assert((dev_info.reta_size & (dev_info.reta_size - 1)) == 0); 702 703 rss_reta_size[port_id] = dev_info.reta_size; 704 printf("port[%d]: rss table size: %d\n", port_id, 705 dev_info.reta_size); 706 } 707 708 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 709 continue; 710 } 711 712 int ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &port_conf); 713 if (ret != 0) { 714 return ret; 715 } 716 uint16_t q; 717 for (q = 0; q < nb_queues; q++) { 718 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 719 socketid, &dev_info.default_txconf); 720 if (ret < 0) { 721 return ret; 722 } 723 724 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 725 socketid, &dev_info.default_rxconf, mbuf_pool); 726 if (ret < 0) { 727 return ret; 728 } 729 } 730 731 ret = rte_eth_dev_start(port_id); 732 if (ret < 0) { 733 return ret; 734 } 735 736 if (nb_queues > 1) { 737 /* set HW rss hash function to Toeplitz. */ 738 if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) { 739 struct rte_eth_hash_filter_info info = {0}; 740 info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG; 741 info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; 742 743 if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH, 744 RTE_ETH_FILTER_SET, &info) < 0) { 745 rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n", 746 port_id); 747 } 748 } 749 750 set_rss_table(port_id, dev_info.reta_size, nb_queues); 751 } 752 753 /* Enable RX in promiscuous mode for the Ethernet device. */ 754 if (ff_global_cfg.dpdk.promiscuous) { 755 rte_eth_promiscuous_enable(port_id); 756 ret = rte_eth_promiscuous_get(port_id); 757 if (ret == 1) { 758 printf("set port %u to promiscuous mode ok\n", port_id); 759 } else { 760 printf("set port %u to promiscuous mode error\n", port_id); 761 } 762 } 763 764 /* Enable pcap dump */ 765 if (pconf->pcap) { 766 ff_enable_pcap(pconf->pcap); 767 } 768 } 769 770 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 771 check_all_ports_link_status(); 772 } 773 774 return 0; 775 } 776 777 static int 778 init_clock(void) 779 { 780 rte_timer_subsystem_init(); 781 uint64_t hz = rte_get_timer_hz(); 782 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 783 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 784 785 rte_timer_init(&freebsd_clock); 786 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 787 rte_lcore_id(), &ff_hardclock_job, NULL); 788 789 ff_update_current_ts(); 790 791 return 0; 792 } 793 794 int 795 ff_dpdk_init(int argc, char **argv) 796 { 797 if (ff_global_cfg.dpdk.nb_procs < 1 || 798 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 799 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 800 ff_global_cfg.dpdk.proc_id < 0) { 801 printf("param num_procs[%d] or proc_id[%d] error!\n", 802 ff_global_cfg.dpdk.nb_procs, 803 ff_global_cfg.dpdk.proc_id); 804 exit(1); 805 } 806 807 int ret = rte_eal_init(argc, argv); 808 if (ret < 0) { 809 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 810 } 811 812 numa_on = ff_global_cfg.dpdk.numa_on; 813 814 init_lcore_conf(); 815 816 init_mem_pool(); 817 818 init_arp_ring(); 819 820 init_msg_ring(); 821 822 enable_kni = ff_global_cfg.kni.enable; 823 if (enable_kni) { 824 init_kni(); 825 } 826 827 ret = init_port_start(); 828 if (ret < 0) { 829 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 830 } 831 832 init_clock(); 833 834 return 0; 835 } 836 837 static void 838 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 839 { 840 uint8_t rx_csum = ctx->hw_features.rx_csum; 841 if (rx_csum) { 842 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 843 return; 844 } 845 } 846 847 /* 848 * FIXME: should we save pkt->vlan_tci 849 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 850 */ 851 852 void *data = rte_pktmbuf_mtod(pkt, void*); 853 uint16_t len = rte_pktmbuf_data_len(pkt); 854 855 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 856 if (hdr == NULL) { 857 rte_pktmbuf_free(pkt); 858 return; 859 } 860 861 struct rte_mbuf *pn = pkt->next; 862 void *prev = hdr; 863 while(pn != NULL) { 864 data = rte_pktmbuf_mtod(pkt, void*); 865 len = rte_pktmbuf_data_len(pkt); 866 867 void *mb = ff_mbuf_get(prev, data, len); 868 if (mb == NULL) { 869 ff_mbuf_free(hdr); 870 rte_pktmbuf_free(pkt); 871 return; 872 } 873 pn = pn->next; 874 prev = mb; 875 } 876 877 ff_veth_process_packet(ctx->ifp, hdr); 878 } 879 880 static enum FilterReturn 881 protocol_filter(const void *data, uint16_t len) 882 { 883 if(len < sizeof(struct ether_hdr)) 884 return FILTER_UNKNOWN; 885 886 const struct ether_hdr *hdr; 887 hdr = (const struct ether_hdr *)data; 888 889 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 890 return FILTER_ARP; 891 892 if (!enable_kni) { 893 return FILTER_UNKNOWN; 894 } 895 896 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 897 return FILTER_UNKNOWN; 898 899 return ff_kni_proto_filter(data + sizeof(struct ether_hdr), 900 len - sizeof(struct ether_hdr)); 901 } 902 903 static inline void 904 process_packets(uint8_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 905 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 906 { 907 struct lcore_conf *qconf = &lcore_conf; 908 909 uint16_t i; 910 for (i = 0; i < count; i++) { 911 struct rte_mbuf *rtem = bufs[i]; 912 913 if (unlikely(qconf->pcap[port_id] != NULL)) { 914 ff_dump_packets(qconf->pcap[port_id], rtem); 915 } 916 917 void *data = rte_pktmbuf_mtod(rtem, void*); 918 uint16_t len = rte_pktmbuf_data_len(rtem); 919 920 enum FilterReturn filter = protocol_filter(data, len); 921 if (filter == FILTER_ARP) { 922 struct rte_mempool *mbuf_pool; 923 struct rte_mbuf *mbuf_clone; 924 if (pkts_from_ring == 0) { 925 uint16_t i; 926 int16_t nb_queues = qconf->nb_queue_list[port_id]; 927 assert(nb_queues != -1); 928 929 for(i = 0; i < nb_queues; ++i) { 930 if(i == queue_id) 931 continue; 932 933 unsigned socket_id = 0; 934 if (numa_on) { 935 uint16_t lcore_id = qconf->port_cfgs[port_id].lcore_list[i]; 936 socket_id = rte_lcore_to_socket_id(lcore_id); 937 } 938 mbuf_pool = pktmbuf_pool[socket_id]; 939 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 940 if(mbuf_clone) { 941 int ret = rte_ring_enqueue(arp_ring[i][port_id], mbuf_clone); 942 if (ret < 0) 943 rte_pktmbuf_free(mbuf_clone); 944 } 945 } 946 } 947 948 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 949 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 950 mbuf_clone = rte_pktmbuf_clone(rtem, mbuf_pool); 951 if(mbuf_clone) { 952 ff_kni_enqueue(port_id, mbuf_clone); 953 } 954 } 955 956 ff_veth_input(ctx, rtem); 957 } else if (enable_kni && ((filter == FILTER_KNI && kni_accept) || 958 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 959 ff_kni_enqueue(port_id, rtem); 960 } else { 961 ff_veth_input(ctx, rtem); 962 } 963 } 964 } 965 966 static inline int 967 process_arp_ring(uint8_t port_id, uint16_t queue_id, 968 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 969 { 970 /* read packet from ring buf and to process */ 971 uint16_t nb_rb; 972 nb_rb = rte_ring_dequeue_burst(arp_ring[queue_id][port_id], 973 (void **)pkts_burst, MAX_PKT_BURST); 974 975 if(nb_rb > 0) { 976 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 977 } 978 979 return 0; 980 } 981 982 static inline void 983 handle_sysctl_msg(struct ff_msg *msg, uint16_t proc_id) 984 { 985 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 986 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 987 msg->sysctl.newlen); 988 989 if (ret < 0) { 990 msg->result = errno; 991 } else { 992 msg->result = 0; 993 } 994 995 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 996 } 997 998 static inline void 999 handle_ioctl_msg(struct ff_msg *msg, uint16_t proc_id) 1000 { 1001 int fd, ret; 1002 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 1003 if (fd < 0) { 1004 ret = -1; 1005 goto done; 1006 } 1007 1008 ret = ff_ioctl(fd, msg->ioctl.cmd, msg->ioctl.data); 1009 1010 ff_close(fd); 1011 1012 done: 1013 if (ret < 0) { 1014 msg->result = errno; 1015 } else { 1016 msg->result = 0; 1017 } 1018 1019 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1020 } 1021 1022 static inline void 1023 handle_route_msg(struct ff_msg *msg, uint16_t proc_id) 1024 { 1025 msg->result = ff_rtioctl(msg->route.fib, msg->route.data, 1026 &msg->route.len, msg->route.maxlen); 1027 1028 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1029 } 1030 1031 static struct ff_top_args ff_status; 1032 static inline void 1033 handle_top_msg(struct ff_msg *msg, uint16_t proc_id) 1034 { 1035 msg->top = ff_status; 1036 msg->result = 0; 1037 1038 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1039 } 1040 1041 static inline void 1042 handle_default_msg(struct ff_msg *msg, uint16_t proc_id) 1043 { 1044 msg->result = EINVAL; 1045 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1046 } 1047 1048 static inline void 1049 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1050 { 1051 switch (msg->msg_type) { 1052 case FF_SYSCTL: 1053 handle_sysctl_msg(msg, proc_id); 1054 break; 1055 case FF_IOCTL: 1056 handle_ioctl_msg(msg, proc_id); 1057 break; 1058 case FF_ROUTE: 1059 handle_route_msg(msg, proc_id); 1060 break; 1061 case FF_TOP: 1062 handle_top_msg(msg, proc_id); 1063 break; 1064 default: 1065 handle_default_msg(msg, proc_id); 1066 break; 1067 } 1068 } 1069 1070 static inline int 1071 process_msg_ring(uint16_t proc_id) 1072 { 1073 void *msg; 1074 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1075 1076 if (unlikely(ret == 0)) { 1077 handle_msg((struct ff_msg *)msg, proc_id); 1078 } 1079 1080 return 0; 1081 } 1082 1083 /* Send burst of packets on an output interface */ 1084 static inline int 1085 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1086 { 1087 struct rte_mbuf **m_table; 1088 int ret; 1089 uint16_t queueid; 1090 1091 queueid = qconf->tx_queue_id[port]; 1092 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1093 1094 if (unlikely(qconf->pcap[port] != NULL)) { 1095 uint16_t i; 1096 for (i = 0; i < n; i++) { 1097 ff_dump_packets(qconf->pcap[port], m_table[i]); 1098 } 1099 } 1100 1101 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1102 if (unlikely(ret < n)) { 1103 do { 1104 rte_pktmbuf_free(m_table[ret]); 1105 } while (++ret < n); 1106 } 1107 1108 return 0; 1109 } 1110 1111 /* Enqueue a single packet, and send burst if queue is filled */ 1112 static inline int 1113 send_single_packet(struct rte_mbuf *m, uint8_t port) 1114 { 1115 uint16_t len; 1116 struct lcore_conf *qconf; 1117 1118 qconf = &lcore_conf; 1119 len = qconf->tx_mbufs[port].len; 1120 qconf->tx_mbufs[port].m_table[len] = m; 1121 len++; 1122 1123 /* enough pkts to be sent */ 1124 if (unlikely(len == MAX_PKT_BURST)) { 1125 send_burst(qconf, MAX_PKT_BURST, port); 1126 len = 0; 1127 } 1128 1129 qconf->tx_mbufs[port].len = len; 1130 return 0; 1131 } 1132 1133 int 1134 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1135 int total) 1136 { 1137 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1138 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1139 if (head == NULL) { 1140 ff_mbuf_free(m); 1141 return -1; 1142 } 1143 1144 head->pkt_len = total; 1145 head->nb_segs = 0; 1146 1147 int off = 0; 1148 struct rte_mbuf *cur = head, *prev = NULL; 1149 while(total > 0) { 1150 if (cur == NULL) { 1151 cur = rte_pktmbuf_alloc(mbuf_pool); 1152 if (cur == NULL) { 1153 rte_pktmbuf_free(head); 1154 ff_mbuf_free(m); 1155 return -1; 1156 } 1157 } 1158 1159 void *data = rte_pktmbuf_mtod(cur, void*); 1160 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1161 int ret = ff_mbuf_copydata(m, data, off, len); 1162 if (ret < 0) { 1163 rte_pktmbuf_free(head); 1164 ff_mbuf_free(m); 1165 return -1; 1166 } 1167 1168 if (prev != NULL) { 1169 prev->next = cur; 1170 } 1171 prev = cur; 1172 1173 cur->data_len = len; 1174 off += len; 1175 total -= len; 1176 head->nb_segs++; 1177 cur = NULL; 1178 } 1179 1180 struct ff_tx_offload offload = {0}; 1181 ff_mbuf_tx_offload(m, &offload); 1182 1183 if (offload.ip_csum) { 1184 head->ol_flags |= PKT_TX_IP_CKSUM; 1185 head->l2_len = sizeof(struct ether_hdr); 1186 head->l3_len = sizeof(struct ipv4_hdr); 1187 } 1188 1189 if (ctx->hw_features.tx_csum_l4) { 1190 if (offload.tcp_csum) { 1191 head->ol_flags |= PKT_TX_TCP_CKSUM; 1192 head->l2_len = sizeof(struct ether_hdr); 1193 head->l3_len = sizeof(struct ipv4_hdr); 1194 } 1195 1196 if (offload.tso_seg_size) { 1197 head->ol_flags |= PKT_TX_TCP_SEG; 1198 head->l4_len = sizeof(struct tcp_hdr); 1199 head->tso_segsz = offload.tso_seg_size; 1200 } 1201 1202 if (offload.udp_csum) { 1203 head->ol_flags |= PKT_TX_UDP_CKSUM; 1204 head->l2_len = sizeof(struct ether_hdr); 1205 head->l3_len = sizeof(struct ipv4_hdr); 1206 } 1207 } 1208 1209 ff_mbuf_free(m); 1210 1211 return send_single_packet(head, ctx->port_id); 1212 } 1213 1214 static int 1215 main_loop(void *arg) 1216 { 1217 struct loop_routine *lr = (struct loop_routine *)arg; 1218 1219 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1220 unsigned lcore_id; 1221 uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc; 1222 int i, j, nb_rx, idle; 1223 uint8_t port_id, queue_id; 1224 struct lcore_conf *qconf; 1225 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1226 US_PER_S * BURST_TX_DRAIN_US; 1227 struct ff_dpdk_if_context *ctx; 1228 1229 prev_tsc = 0; 1230 usch_tsc = 0; 1231 1232 lcore_id = rte_lcore_id(); 1233 qconf = &lcore_conf; 1234 1235 if (qconf->nb_rx_queue == 0) { 1236 printf("lcore %u has nothing to do\n", lcore_id); 1237 return 0; 1238 } 1239 1240 while (1) { 1241 cur_tsc = rte_rdtsc(); 1242 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1243 rte_timer_manage(); 1244 } 1245 1246 idle = 1; 1247 sys_tsc = 0; 1248 usr_tsc = 0; 1249 1250 /* 1251 * TX burst queue drain 1252 */ 1253 diff_tsc = cur_tsc - prev_tsc; 1254 if (unlikely(diff_tsc > drain_tsc)) { 1255 /* 1256 * This could be optimized (use queueid instead of 1257 * portid), but it is not called so often 1258 */ 1259 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { 1260 if (qconf->tx_mbufs[port_id].len == 0) 1261 continue; 1262 1263 idle = 0; 1264 send_burst(qconf, 1265 qconf->tx_mbufs[port_id].len, 1266 port_id); 1267 qconf->tx_mbufs[port_id].len = 0; 1268 } 1269 1270 prev_tsc = cur_tsc; 1271 } 1272 1273 /* 1274 * Read packet from RX queues 1275 */ 1276 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1277 port_id = qconf->rx_queue_list[i].port_id; 1278 queue_id = qconf->rx_queue_list[i].queue_id; 1279 ctx = veth_ctx[port_id]; 1280 1281 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1282 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1283 } 1284 1285 process_arp_ring(port_id, queue_id, pkts_burst, ctx); 1286 1287 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1288 MAX_PKT_BURST); 1289 if (nb_rx == 0) 1290 continue; 1291 1292 idle = 0; 1293 1294 /* Prefetch first packets */ 1295 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1296 rte_prefetch0(rte_pktmbuf_mtod( 1297 pkts_burst[j], void *)); 1298 } 1299 1300 /* Prefetch and handle already prefetched packets */ 1301 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1302 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1303 j + PREFETCH_OFFSET], void *)); 1304 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1305 } 1306 1307 /* Handle remaining prefetched packets */ 1308 for (; j < nb_rx; j++) { 1309 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1310 } 1311 } 1312 1313 process_msg_ring(qconf->proc_id); 1314 1315 div_tsc = rte_rdtsc(); 1316 1317 if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc > drain_tsc))) { 1318 usch_tsc = cur_tsc; 1319 lr->loop(lr->arg); 1320 } 1321 1322 end_tsc = rte_rdtsc(); 1323 1324 if (usch_tsc == cur_tsc) { 1325 usr_tsc = end_tsc - div_tsc; 1326 } 1327 1328 if (!idle) { 1329 sys_tsc = div_tsc - cur_tsc; 1330 ff_status.sys_tsc += sys_tsc; 1331 } 1332 1333 ff_status.usr_tsc += usr_tsc; 1334 ff_status.work_tsc += end_tsc - cur_tsc; 1335 ff_status.idle_tsc += end_tsc - cur_tsc - usr_tsc - sys_tsc; 1336 1337 ff_status.loops++; 1338 } 1339 } 1340 1341 int 1342 ff_dpdk_if_up(void) { 1343 int nb_ports = ff_global_cfg.dpdk.nb_ports; 1344 int i; 1345 for (i = 0; i < nb_ports; i++) { 1346 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 1347 // if port's lcore list does't contain current core, just skip this port 1348 if (lcore_conf.tx_queue_id[port_id] < 0) { 1349 continue; 1350 } 1351 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 1352 veth_ctx[port_id] = ff_veth_attach(pconf); 1353 if (veth_ctx[port_id] == NULL) { 1354 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1355 } 1356 } 1357 1358 return 0; 1359 } 1360 1361 void 1362 ff_dpdk_run(loop_func_t loop, void *arg) { 1363 struct loop_routine *lr = rte_malloc(NULL, 1364 sizeof(struct loop_routine), 0); 1365 lr->loop = loop; 1366 lr->arg = arg; 1367 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1368 rte_eal_mp_wait_lcore(); 1369 rte_free(lr); 1370 } 1371 1372 void 1373 ff_dpdk_pktmbuf_free(void *m) 1374 { 1375 rte_pktmbuf_free((struct rte_mbuf *)m); 1376 } 1377 1378 static uint32_t 1379 toeplitz_hash(unsigned keylen, const uint8_t *key, 1380 unsigned datalen, const uint8_t *data) 1381 { 1382 uint32_t hash = 0, v; 1383 u_int i, b; 1384 1385 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1386 1387 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1388 for (i = 0; i < datalen; i++) { 1389 for (b = 0; b < 8; b++) { 1390 if (data[i] & (1<<(7-b))) 1391 hash ^= v; 1392 v <<= 1; 1393 if ((i + 4) < keylen && 1394 (key[i+4] & (1<<(7-b)))) 1395 v |= 1; 1396 } 1397 } 1398 return (hash); 1399 } 1400 1401 int 1402 ff_rss_check(void *softc, uint32_t saddr, uint32_t daddr, 1403 uint16_t sport, uint16_t dport) 1404 { 1405 struct lcore_conf *qconf = &lcore_conf; 1406 struct ff_dpdk_if_context *ctx = ff_veth_softc_to_hostc(softc); 1407 uint16_t nb_queues = qconf->nb_queue_list[ctx->port_id]; 1408 1409 if (nb_queues == 1) { 1410 return 1; 1411 } 1412 1413 uint16_t reta_size = rss_reta_size[ctx->port_id]; 1414 uint16_t queueid = qconf->tx_queue_id[ctx->port_id]; 1415 1416 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1417 sizeof(dport)]; 1418 1419 unsigned datalen = 0; 1420 1421 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1422 datalen += sizeof(saddr); 1423 1424 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1425 datalen += sizeof(daddr); 1426 1427 bcopy(&sport, &data[datalen], sizeof(sport)); 1428 datalen += sizeof(sport); 1429 1430 bcopy(&dport, &data[datalen], sizeof(dport)); 1431 datalen += sizeof(dport); 1432 1433 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), 1434 default_rsskey_40bytes, datalen, data); 1435 1436 return ((hash & (reta_size - 1)) % nb_queues) == queueid; 1437 } 1438