1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 #include <assert.h> 27 #include <unistd.h> 28 #include <sys/mman.h> 29 #include <errno.h> 30 31 #include <rte_common.h> 32 #include <rte_byteorder.h> 33 #include <rte_log.h> 34 #include <rte_memory.h> 35 #include <rte_memcpy.h> 36 #include <rte_memzone.h> 37 #include <rte_config.h> 38 #include <rte_eal.h> 39 #include <rte_pci.h> 40 #include <rte_mbuf.h> 41 #include <rte_memory.h> 42 #include <rte_lcore.h> 43 #include <rte_launch.h> 44 #include <rte_ethdev.h> 45 #include <rte_debug.h> 46 #include <rte_common.h> 47 #include <rte_ether.h> 48 #include <rte_malloc.h> 49 #include <rte_cycles.h> 50 #include <rte_timer.h> 51 #include <rte_thash.h> 52 #include <rte_ip.h> 53 #include <rte_tcp.h> 54 #include <rte_udp.h> 55 56 #include "ff_dpdk_if.h" 57 #include "ff_dpdk_pcap.h" 58 #include "ff_dpdk_kni.h" 59 #include "ff_config.h" 60 #include "ff_veth.h" 61 #include "ff_host_interface.h" 62 #include "ff_msg.h" 63 #include "ff_api.h" 64 #include "ff_memory.h" 65 66 #ifdef FF_KNI 67 #define KNI_MBUF_MAX 2048 68 #define KNI_QUEUE_SIZE 2048 69 70 static int enable_kni; 71 static int kni_accept; 72 #endif 73 74 static int numa_on; 75 76 static unsigned idle_sleep; 77 78 static struct rte_timer freebsd_clock; 79 80 // Mellanox Linux's driver key 81 static uint8_t default_rsskey_40bytes[40] = { 82 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 83 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 84 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 85 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 86 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 87 }; 88 89 static struct rte_eth_conf default_port_conf = { 90 .rxmode = { 91 .mq_mode = ETH_MQ_RX_RSS, 92 .max_rx_pkt_len = ETHER_MAX_LEN, 93 .split_hdr_size = 0, /**< hdr buf size */ 94 .header_split = 0, /**< Header Split disabled */ 95 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 96 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 97 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 98 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 99 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 100 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 101 .enable_lro = 0, /**< LRO disabled */ 102 }, 103 .rx_adv_conf = { 104 .rss_conf = { 105 .rss_key = default_rsskey_40bytes, 106 .rss_key_len = 40, 107 .rss_hf = ETH_RSS_PROTO_MASK, 108 }, 109 }, 110 .txmode = { 111 .mq_mode = ETH_MQ_TX_NONE, 112 }, 113 }; 114 115 struct lcore_conf lcore_conf; 116 117 struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 118 119 static struct rte_ring **dispatch_ring[RTE_MAX_ETHPORTS]; 120 static dispatch_func_t packet_dispatcher; 121 122 static uint16_t rss_reta_size[RTE_MAX_ETHPORTS]; 123 124 struct ff_msg_ring { 125 char ring_name[2][RTE_RING_NAMESIZE]; 126 /* ring[0] for lcore recv msg, other send */ 127 /* ring[1] for lcore send msg, other read */ 128 struct rte_ring *ring[2]; 129 } __rte_cache_aligned; 130 131 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 132 static struct rte_mempool *message_pool; 133 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 134 135 static struct ff_top_args ff_top_status; 136 static struct ff_traffic_args ff_traffic; 137 extern void ff_hardclock(void); 138 139 static void 140 ff_hardclock_job(__rte_unused struct rte_timer *timer, 141 __rte_unused void *arg) { 142 ff_hardclock(); 143 ff_update_current_ts(); 144 } 145 146 struct ff_dpdk_if_context * 147 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 148 { 149 struct ff_dpdk_if_context *ctx; 150 151 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 152 if (ctx == NULL) 153 return NULL; 154 155 ctx->sc = sc; 156 ctx->ifp = ifp; 157 ctx->port_id = cfg->port_id; 158 ctx->hw_features = cfg->hw_features; 159 160 return ctx; 161 } 162 163 void 164 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 165 { 166 free(ctx); 167 } 168 169 static void 170 check_all_ports_link_status(void) 171 { 172 #define CHECK_INTERVAL 100 /* 100ms */ 173 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 174 175 uint16_t portid; 176 uint8_t count, all_ports_up, print_flag = 0; 177 struct rte_eth_link link; 178 179 printf("\nChecking link status"); 180 fflush(stdout); 181 182 int i, nb_ports; 183 nb_ports = ff_global_cfg.dpdk.nb_ports; 184 for (count = 0; count <= MAX_CHECK_TIME; count++) { 185 all_ports_up = 1; 186 for (i = 0; i < nb_ports; i++) { 187 uint16_t portid = ff_global_cfg.dpdk.portid_list[i]; 188 memset(&link, 0, sizeof(link)); 189 rte_eth_link_get_nowait(portid, &link); 190 191 /* print link status if flag set */ 192 if (print_flag == 1) { 193 if (link.link_status) { 194 printf("Port %d Link Up - speed %u " 195 "Mbps - %s\n", (int)portid, 196 (unsigned)link.link_speed, 197 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 198 ("full-duplex") : ("half-duplex\n")); 199 } else { 200 printf("Port %d Link Down\n", (int)portid); 201 } 202 continue; 203 } 204 /* clear all_ports_up flag if any link down */ 205 if (link.link_status == 0) { 206 all_ports_up = 0; 207 break; 208 } 209 } 210 211 /* after finally printing all link status, get out */ 212 if (print_flag == 1) 213 break; 214 215 if (all_ports_up == 0) { 216 printf("."); 217 fflush(stdout); 218 rte_delay_ms(CHECK_INTERVAL); 219 } 220 221 /* set the print_flag if all ports up or timeout */ 222 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 223 print_flag = 1; 224 printf("done\n"); 225 } 226 } 227 } 228 229 static int 230 init_lcore_conf(void) 231 { 232 uint8_t nb_dev_ports = rte_eth_dev_count(); 233 if (nb_dev_ports == 0) { 234 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 235 } 236 237 if (ff_global_cfg.dpdk.max_portid >= nb_dev_ports) { 238 rte_exit(EXIT_FAILURE, "this machine doesn't have port %d.\n", 239 ff_global_cfg.dpdk.max_portid); 240 } 241 242 lcore_conf.port_cfgs = ff_global_cfg.dpdk.port_cfgs; 243 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 244 245 uint16_t proc_id; 246 for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { 247 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[proc_id]; 248 if (!lcore_config[lcore_id].detected) { 249 rte_exit(EXIT_FAILURE, "lcore %u unavailable\n", lcore_id); 250 } 251 } 252 253 uint16_t socket_id = 0; 254 if (numa_on) { 255 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 256 } 257 258 lcore_conf.socket_id = socket_id; 259 260 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[lcore_conf.proc_id]; 261 int j; 262 for (j = 0; j < ff_global_cfg.dpdk.nb_ports; ++j) { 263 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 264 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 265 266 int queueid = -1; 267 int i; 268 for (i = 0; i < pconf->nb_lcores; i++) { 269 if (pconf->lcore_list[i] == lcore_id) { 270 queueid = i; 271 } 272 } 273 if (queueid < 0) { 274 continue; 275 } 276 printf("lcore: %u, port: %u, queue: %u\n", lcore_id, port_id, queueid); 277 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 278 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 279 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = queueid; 280 lcore_conf.nb_rx_queue++; 281 282 lcore_conf.tx_queue_id[port_id] = queueid; 283 lcore_conf.tx_port_id[lcore_conf.nb_tx_port] = port_id; 284 lcore_conf.nb_tx_port++; 285 286 lcore_conf.pcap[port_id] = pconf->pcap; 287 lcore_conf.nb_queue_list[port_id] = pconf->nb_lcores; 288 } 289 290 if (lcore_conf.nb_rx_queue == 0) { 291 rte_exit(EXIT_FAILURE, "lcore %u has nothing to do\n", lcore_id); 292 } 293 294 return 0; 295 } 296 297 static int 298 init_mem_pool(void) 299 { 300 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 301 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 302 uint32_t nb_tx_queue = nb_lcores; 303 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 304 305 unsigned nb_mbuf = RTE_MAX ( 306 (nb_rx_queue*RX_QUEUE_SIZE + 307 nb_ports*nb_lcores*MAX_PKT_BURST + 308 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 309 nb_lcores*MEMPOOL_CACHE_SIZE + 310 #ifdef FF_KNI 311 nb_ports*KNI_MBUF_MAX + 312 nb_ports*KNI_QUEUE_SIZE + 313 #endif 314 nb_lcores*nb_ports*DISPATCH_RING_SIZE), 315 (unsigned)8192); 316 317 unsigned socketid = 0; 318 uint16_t i, lcore_id; 319 char s[64]; 320 321 for (i = 0; i < ff_global_cfg.dpdk.nb_procs; i++) { 322 lcore_id = ff_global_cfg.dpdk.proc_lcore[i]; 323 if (numa_on) { 324 socketid = rte_lcore_to_socket_id(lcore_id); 325 } 326 327 if (socketid >= NB_SOCKETS) { 328 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 329 socketid, i, NB_SOCKETS); 330 } 331 332 if (pktmbuf_pool[socketid] != NULL) { 333 continue; 334 } 335 336 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 337 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 338 pktmbuf_pool[socketid] = 339 rte_pktmbuf_pool_create(s, nb_mbuf, 340 MEMPOOL_CACHE_SIZE, 0, 341 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 342 } else { 343 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 344 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 345 } 346 347 if (pktmbuf_pool[socketid] == NULL) { 348 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 349 } else { 350 printf("create mbuf pool on socket %d\n", socketid); 351 } 352 353 #ifdef FF_USE_PAGE_ARRAY 354 nb_mbuf = RTE_MAX ( 355 nb_ports*nb_lcores*MAX_PKT_BURST + 356 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 357 nb_lcores*MEMPOOL_CACHE_SIZE, 358 (unsigned)4096); 359 ff_init_ref_pool(nb_mbuf, socketid); 360 #endif 361 } 362 363 return 0; 364 } 365 366 static struct rte_ring * 367 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 368 { 369 struct rte_ring *ring; 370 371 if (name == NULL) { 372 rte_exit(EXIT_FAILURE, "create ring failed, no name!\n"); 373 } 374 375 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 376 ring = rte_ring_create(name, count, socket_id, flags); 377 } else { 378 ring = rte_ring_lookup(name); 379 } 380 381 if (ring == NULL) { 382 rte_exit(EXIT_FAILURE, "create ring:%s failed!\n", name); 383 } 384 385 return ring; 386 } 387 388 static int 389 init_dispatch_ring(void) 390 { 391 int j; 392 char name_buf[RTE_RING_NAMESIZE]; 393 int queueid; 394 395 unsigned socketid = lcore_conf.socket_id; 396 397 /* Create ring according to ports actually being used. */ 398 int nb_ports = ff_global_cfg.dpdk.nb_ports; 399 for (j = 0; j < nb_ports; j++) { 400 uint16_t portid = ff_global_cfg.dpdk.portid_list[j]; 401 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[portid]; 402 int nb_queues = pconf->nb_lcores; 403 if (dispatch_ring[portid] == NULL) { 404 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_p%d", portid); 405 406 dispatch_ring[portid] = rte_zmalloc(name_buf, 407 sizeof(struct rte_ring *) * nb_queues, 408 RTE_CACHE_LINE_SIZE); 409 if (dispatch_ring[portid] == NULL) { 410 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 411 "failed\n", name_buf); 412 } 413 } 414 415 for(queueid = 0; queueid < nb_queues; ++queueid) { 416 snprintf(name_buf, RTE_RING_NAMESIZE, "dispatch_ring_p%d_q%d", 417 portid, queueid); 418 dispatch_ring[portid][queueid] = create_ring(name_buf, 419 DISPATCH_RING_SIZE, socketid, RING_F_SC_DEQ); 420 421 if (dispatch_ring[portid][queueid] == NULL) 422 rte_panic("create ring:%s failed!\n", name_buf); 423 424 printf("create ring:%s success, %u ring entries are now free!\n", 425 name_buf, rte_ring_free_count(dispatch_ring[portid][queueid])); 426 } 427 } 428 429 return 0; 430 } 431 432 static void 433 ff_msg_init(struct rte_mempool *mp, 434 __attribute__((unused)) void *opaque_arg, 435 void *obj, __attribute__((unused)) unsigned i) 436 { 437 struct ff_msg *msg = (struct ff_msg *)obj; 438 msg->msg_type = FF_UNKNOWN; 439 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 440 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 441 } 442 443 static int 444 init_msg_ring(void) 445 { 446 uint16_t i; 447 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 448 unsigned socketid = lcore_conf.socket_id; 449 450 /* Create message buffer pool */ 451 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 452 message_pool = rte_mempool_create(FF_MSG_POOL, 453 MSG_RING_SIZE * 2 * nb_procs, 454 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 455 NULL, NULL, ff_msg_init, NULL, 456 socketid, 0); 457 } else { 458 message_pool = rte_mempool_lookup(FF_MSG_POOL); 459 } 460 461 if (message_pool == NULL) { 462 rte_panic("Create msg mempool failed\n"); 463 } 464 465 for(i = 0; i < nb_procs; ++i) { 466 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 467 "%s%u", FF_MSG_RING_IN, i); 468 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 469 "%s%u", FF_MSG_RING_OUT, i); 470 471 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 472 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 473 if (msg_ring[i].ring[0] == NULL) 474 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 475 476 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 477 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 478 if (msg_ring[i].ring[1] == NULL) 479 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 480 } 481 482 return 0; 483 } 484 485 #ifdef FF_KNI 486 static int 487 init_kni(void) 488 { 489 int nb_ports = rte_eth_dev_count(); 490 kni_accept = 0; 491 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 492 kni_accept = 1; 493 494 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 495 ff_global_cfg.kni.udp_port); 496 497 unsigned socket_id = lcore_conf.socket_id; 498 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 499 500 nb_ports = ff_global_cfg.dpdk.nb_ports; 501 int i, ret; 502 for (i = 0; i < nb_ports; i++) { 503 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 504 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 505 } 506 507 return 0; 508 } 509 #endif 510 511 static void 512 set_rss_table(uint16_t port_id, uint16_t reta_size, uint16_t nb_queues) 513 { 514 if (reta_size == 0) { 515 return; 516 } 517 518 int reta_conf_size = RTE_MAX(1, reta_size / RTE_RETA_GROUP_SIZE); 519 struct rte_eth_rss_reta_entry64 reta_conf[reta_conf_size]; 520 521 /* config HW indirection table */ 522 unsigned i, j, hash=0; 523 for (i = 0; i < reta_conf_size; i++) { 524 reta_conf[i].mask = ~0ULL; 525 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 526 reta_conf[i].reta[j] = hash++ % nb_queues; 527 } 528 } 529 530 if (rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size)) { 531 rte_exit(EXIT_FAILURE, "port[%d], failed to update rss table\n", 532 port_id); 533 } 534 } 535 536 static int 537 init_port_start(void) 538 { 539 int nb_ports = ff_global_cfg.dpdk.nb_ports; 540 unsigned socketid = 0; 541 struct rte_mempool *mbuf_pool; 542 uint16_t i; 543 544 for (i = 0; i < nb_ports; i++) { 545 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 546 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 547 uint16_t nb_queues = pconf->nb_lcores; 548 549 struct rte_eth_dev_info dev_info; 550 rte_eth_dev_info_get(port_id, &dev_info); 551 552 if (nb_queues > dev_info.max_rx_queues) { 553 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 554 nb_queues, 555 dev_info.max_rx_queues); 556 } 557 558 if (nb_queues > dev_info.max_tx_queues) { 559 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 560 nb_queues, 561 dev_info.max_tx_queues); 562 } 563 564 struct ether_addr addr; 565 rte_eth_macaddr_get(port_id, &addr); 566 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 567 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 568 (unsigned)port_id, 569 addr.addr_bytes[0], addr.addr_bytes[1], 570 addr.addr_bytes[2], addr.addr_bytes[3], 571 addr.addr_bytes[4], addr.addr_bytes[5]); 572 573 rte_memcpy(pconf->mac, 574 addr.addr_bytes, ETHER_ADDR_LEN); 575 576 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 577 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 578 ETH_TXQ_FLAGS_NOREFCOUNT; 579 580 /* Disable features that are not supported by port's HW */ 581 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 582 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 583 } 584 585 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 586 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 587 } 588 589 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 590 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 591 } 592 593 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 594 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 595 } 596 597 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 598 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 599 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 600 } 601 602 struct rte_eth_conf port_conf = {0}; 603 604 /* Set RSS mode */ 605 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 606 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 607 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 608 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 609 610 /* Set Rx VLAN stripping */ 611 if (ff_global_cfg.dpdk.vlan_strip) { 612 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 613 port_conf.rxmode.hw_vlan_strip = 1; 614 } 615 } 616 617 /* Enable HW CRC stripping */ 618 port_conf.rxmode.hw_strip_crc = 1; 619 620 /* FIXME: Enable TCP LRO ?*/ 621 #if 0 622 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 623 printf("LRO is supported\n"); 624 port_conf.rxmode.enable_lro = 1; 625 pconf->hw_features.rx_lro = 1; 626 } 627 #endif 628 629 /* Set Rx checksum checking */ 630 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 631 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 632 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 633 printf("RX checksum offload supported\n"); 634 port_conf.rxmode.hw_ip_checksum = 1; 635 pconf->hw_features.rx_csum = 1; 636 } 637 638 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 639 printf("TX ip checksum offload supported\n"); 640 pconf->hw_features.tx_csum_ip = 1; 641 } 642 643 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 644 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 645 printf("TX TCP&UDP checksum offload supported\n"); 646 pconf->hw_features.tx_csum_l4 = 1; 647 } 648 649 if (ff_global_cfg.dpdk.tso) { 650 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 651 printf("TSO is supported\n"); 652 pconf->hw_features.tx_tso = 1; 653 } 654 } else { 655 printf("TSO is disabled\n"); 656 } 657 658 if (dev_info.reta_size) { 659 /* reta size must be power of 2 */ 660 assert((dev_info.reta_size & (dev_info.reta_size - 1)) == 0); 661 662 rss_reta_size[port_id] = dev_info.reta_size; 663 printf("port[%d]: rss table size: %d\n", port_id, 664 dev_info.reta_size); 665 } 666 667 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 668 continue; 669 } 670 671 int ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &port_conf); 672 if (ret != 0) { 673 return ret; 674 } 675 uint16_t q; 676 for (q = 0; q < nb_queues; q++) { 677 if (numa_on) { 678 uint16_t lcore_id = lcore_conf.port_cfgs[port_id].lcore_list[q]; 679 socketid = rte_lcore_to_socket_id(lcore_id); 680 } 681 mbuf_pool = pktmbuf_pool[socketid]; 682 683 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 684 socketid, &dev_info.default_txconf); 685 if (ret < 0) { 686 return ret; 687 } 688 689 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 690 socketid, &dev_info.default_rxconf, mbuf_pool); 691 if (ret < 0) { 692 return ret; 693 } 694 } 695 696 ret = rte_eth_dev_start(port_id); 697 if (ret < 0) { 698 return ret; 699 } 700 701 if (nb_queues > 1) { 702 /* set HW rss hash function to Toeplitz. */ 703 if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) { 704 struct rte_eth_hash_filter_info info = {0}; 705 info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG; 706 info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; 707 708 if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH, 709 RTE_ETH_FILTER_SET, &info) < 0) { 710 rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n", 711 port_id); 712 } 713 } 714 715 set_rss_table(port_id, dev_info.reta_size, nb_queues); 716 } 717 718 /* Enable RX in promiscuous mode for the Ethernet device. */ 719 if (ff_global_cfg.dpdk.promiscuous) { 720 rte_eth_promiscuous_enable(port_id); 721 ret = rte_eth_promiscuous_get(port_id); 722 if (ret == 1) { 723 printf("set port %u to promiscuous mode ok\n", port_id); 724 } else { 725 printf("set port %u to promiscuous mode error\n", port_id); 726 } 727 } 728 729 /* Enable pcap dump */ 730 if (pconf->pcap) { 731 ff_enable_pcap(pconf->pcap); 732 } 733 } 734 735 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 736 check_all_ports_link_status(); 737 } 738 739 return 0; 740 } 741 742 static int 743 init_clock(void) 744 { 745 rte_timer_subsystem_init(); 746 uint64_t hz = rte_get_timer_hz(); 747 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 748 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 749 750 rte_timer_init(&freebsd_clock); 751 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 752 rte_lcore_id(), &ff_hardclock_job, NULL); 753 754 ff_update_current_ts(); 755 756 return 0; 757 } 758 759 int 760 ff_dpdk_init(int argc, char **argv) 761 { 762 if (ff_global_cfg.dpdk.nb_procs < 1 || 763 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 764 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 765 ff_global_cfg.dpdk.proc_id < 0) { 766 printf("param num_procs[%d] or proc_id[%d] error!\n", 767 ff_global_cfg.dpdk.nb_procs, 768 ff_global_cfg.dpdk.proc_id); 769 exit(1); 770 } 771 772 int ret = rte_eal_init(argc, argv); 773 if (ret < 0) { 774 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 775 } 776 777 numa_on = ff_global_cfg.dpdk.numa_on; 778 779 idle_sleep = ff_global_cfg.dpdk.idle_sleep; 780 781 init_lcore_conf(); 782 783 init_mem_pool(); 784 785 init_dispatch_ring(); 786 787 init_msg_ring(); 788 789 #ifdef FF_KNI 790 enable_kni = ff_global_cfg.kni.enable; 791 if (enable_kni) { 792 init_kni(); 793 } 794 #endif 795 796 #ifdef FF_USE_PAGE_ARRAY 797 ff_mmap_init(); 798 #endif 799 800 801 ret = init_port_start(); 802 if (ret < 0) { 803 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 804 } 805 806 init_clock(); 807 808 return 0; 809 } 810 811 static void 812 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 813 { 814 uint8_t rx_csum = ctx->hw_features.rx_csum; 815 if (rx_csum) { 816 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 817 rte_pktmbuf_free(pkt); 818 return; 819 } 820 } 821 822 /* 823 * FIXME: should we save pkt->vlan_tci 824 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 825 */ 826 827 void *data = rte_pktmbuf_mtod(pkt, void*); 828 uint16_t len = rte_pktmbuf_data_len(pkt); 829 830 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 831 if (hdr == NULL) { 832 rte_pktmbuf_free(pkt); 833 return; 834 } 835 836 struct rte_mbuf *pn = pkt->next; 837 void *prev = hdr; 838 while(pn != NULL) { 839 data = rte_pktmbuf_mtod(pn, void*); 840 len = rte_pktmbuf_data_len(pn); 841 842 void *mb = ff_mbuf_get(prev, data, len); 843 if (mb == NULL) { 844 ff_mbuf_free(hdr); 845 rte_pktmbuf_free(pkt); 846 return; 847 } 848 pn = pn->next; 849 prev = mb; 850 } 851 852 ff_veth_process_packet(ctx->ifp, hdr); 853 } 854 855 static enum FilterReturn 856 protocol_filter(const void *data, uint16_t len) 857 { 858 if(len < ETHER_HDR_LEN) 859 return FILTER_UNKNOWN; 860 861 const struct ether_hdr *hdr; 862 hdr = (const struct ether_hdr *)data; 863 864 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 865 return FILTER_ARP; 866 867 #ifndef FF_KNI 868 return FILTER_UNKNOWN; 869 #else 870 if (!enable_kni) { 871 return FILTER_UNKNOWN; 872 } 873 874 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 875 return FILTER_UNKNOWN; 876 877 return ff_kni_proto_filter(data + ETHER_HDR_LEN, 878 len - ETHER_HDR_LEN); 879 #endif 880 } 881 882 static inline void 883 pktmbuf_deep_attach(struct rte_mbuf *mi, const struct rte_mbuf *m) 884 { 885 struct rte_mbuf *md; 886 void *src, *dst; 887 888 dst = rte_pktmbuf_mtod(mi, void *); 889 src = rte_pktmbuf_mtod(m, void *); 890 891 mi->data_len = m->data_len; 892 rte_memcpy(dst, src, m->data_len); 893 894 mi->port = m->port; 895 mi->vlan_tci = m->vlan_tci; 896 mi->vlan_tci_outer = m->vlan_tci_outer; 897 mi->tx_offload = m->tx_offload; 898 mi->hash = m->hash; 899 mi->ol_flags = m->ol_flags; 900 mi->packet_type = m->packet_type; 901 } 902 903 /* copied from rte_pktmbuf_clone */ 904 static inline struct rte_mbuf * 905 pktmbuf_deep_clone(const struct rte_mbuf *md, 906 struct rte_mempool *mp) 907 { 908 struct rte_mbuf *mc, *mi, **prev; 909 uint32_t pktlen; 910 uint8_t nseg; 911 912 if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL)) 913 return NULL; 914 915 mi = mc; 916 prev = &mi->next; 917 pktlen = md->pkt_len; 918 nseg = 0; 919 920 do { 921 nseg++; 922 pktmbuf_deep_attach(mi, md); 923 *prev = mi; 924 prev = &mi->next; 925 } while ((md = md->next) != NULL && 926 (mi = rte_pktmbuf_alloc(mp)) != NULL); 927 928 *prev = NULL; 929 mc->nb_segs = nseg; 930 mc->pkt_len = pktlen; 931 932 /* Allocation of new indirect segment failed */ 933 if (unlikely (mi == NULL)) { 934 rte_pktmbuf_free(mc); 935 return NULL; 936 } 937 938 __rte_mbuf_sanity_check(mc, 1); 939 return mc; 940 } 941 942 static inline void 943 process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 944 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 945 { 946 struct lcore_conf *qconf = &lcore_conf; 947 uint16_t nb_queues = qconf->nb_queue_list[port_id]; 948 949 uint16_t i; 950 for (i = 0; i < count; i++) { 951 struct rte_mbuf *rtem = bufs[i]; 952 953 if (unlikely(qconf->pcap[port_id] != NULL)) { 954 if (!pkts_from_ring) { 955 ff_dump_packets(qconf->pcap[port_id], rtem); 956 } 957 } 958 959 void *data = rte_pktmbuf_mtod(rtem, void*); 960 uint16_t len = rte_pktmbuf_data_len(rtem); 961 962 if (!pkts_from_ring) { 963 ff_traffic.rx_packets++; 964 ff_traffic.rx_bytes += len; 965 } 966 967 if (!pkts_from_ring && packet_dispatcher) { 968 int ret = (*packet_dispatcher)(data, len, queue_id, nb_queues); 969 if (ret < 0 || ret >= nb_queues) { 970 rte_pktmbuf_free(rtem); 971 continue; 972 } 973 974 if (ret != queue_id) { 975 ret = rte_ring_enqueue(dispatch_ring[port_id][ret], rtem); 976 if (ret < 0) 977 rte_pktmbuf_free(rtem); 978 979 continue; 980 } 981 } 982 983 enum FilterReturn filter = protocol_filter(data, len); 984 if (filter == FILTER_ARP) { 985 struct rte_mempool *mbuf_pool; 986 struct rte_mbuf *mbuf_clone; 987 if (!pkts_from_ring) { 988 uint16_t j; 989 for(j = 0; j < nb_queues; ++j) { 990 if(j == queue_id) 991 continue; 992 993 unsigned socket_id = 0; 994 if (numa_on) { 995 uint16_t lcore_id = qconf->port_cfgs[port_id].lcore_list[j]; 996 socket_id = rte_lcore_to_socket_id(lcore_id); 997 } 998 mbuf_pool = pktmbuf_pool[socket_id]; 999 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1000 if(mbuf_clone) { 1001 int ret = rte_ring_enqueue(dispatch_ring[port_id][j], 1002 mbuf_clone); 1003 if (ret < 0) 1004 rte_pktmbuf_free(mbuf_clone); 1005 } 1006 } 1007 } 1008 1009 #ifdef FF_KNI 1010 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1011 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 1012 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1013 if(mbuf_clone) { 1014 ff_kni_enqueue(port_id, mbuf_clone); 1015 } 1016 } 1017 #endif 1018 ff_veth_input(ctx, rtem); 1019 #ifdef FF_KNI 1020 } else if (enable_kni && 1021 ((filter == FILTER_KNI && kni_accept) || 1022 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 1023 ff_kni_enqueue(port_id, rtem); 1024 #endif 1025 } else { 1026 ff_veth_input(ctx, rtem); 1027 } 1028 } 1029 } 1030 1031 static inline int 1032 process_dispatch_ring(uint16_t port_id, uint16_t queue_id, 1033 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 1034 { 1035 /* read packet from ring buf and to process */ 1036 uint16_t nb_rb; 1037 nb_rb = rte_ring_dequeue_burst(dispatch_ring[port_id][queue_id], 1038 (void **)pkts_burst, MAX_PKT_BURST, NULL); 1039 1040 if(nb_rb > 0) { 1041 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 1042 } 1043 1044 return 0; 1045 } 1046 1047 static inline void 1048 handle_sysctl_msg(struct ff_msg *msg) 1049 { 1050 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 1051 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 1052 msg->sysctl.newlen); 1053 1054 if (ret < 0) { 1055 msg->result = errno; 1056 } else { 1057 msg->result = 0; 1058 } 1059 } 1060 1061 static inline void 1062 handle_ioctl_msg(struct ff_msg *msg) 1063 { 1064 int fd, ret; 1065 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 1066 if (fd < 0) { 1067 ret = -1; 1068 goto done; 1069 } 1070 1071 ret = ff_ioctl_freebsd(fd, msg->ioctl.cmd, msg->ioctl.data); 1072 1073 ff_close(fd); 1074 1075 done: 1076 if (ret < 0) { 1077 msg->result = errno; 1078 } else { 1079 msg->result = 0; 1080 } 1081 } 1082 1083 static inline void 1084 handle_route_msg(struct ff_msg *msg) 1085 { 1086 int ret = ff_rtioctl(msg->route.fib, msg->route.data, 1087 &msg->route.len, msg->route.maxlen); 1088 if (ret < 0) { 1089 msg->result = errno; 1090 } else { 1091 msg->result = 0; 1092 } 1093 } 1094 1095 static inline void 1096 handle_top_msg(struct ff_msg *msg) 1097 { 1098 msg->top = ff_top_status; 1099 msg->result = 0; 1100 } 1101 1102 #ifdef FF_NETGRAPH 1103 static inline void 1104 handle_ngctl_msg(struct ff_msg *msg) 1105 { 1106 int ret = ff_ngctl(msg->ngctl.cmd, msg->ngctl.data); 1107 if (ret < 0) { 1108 msg->result = errno; 1109 } else { 1110 msg->result = 0; 1111 msg->ngctl.ret = ret; 1112 } 1113 } 1114 #endif 1115 1116 #ifdef FF_IPFW 1117 static inline void 1118 handle_ipfw_msg(struct ff_msg *msg) 1119 { 1120 int fd, ret; 1121 fd = ff_socket(AF_INET, SOCK_RAW, IPPROTO_RAW); 1122 if (fd < 0) { 1123 ret = -1; 1124 goto done; 1125 } 1126 1127 switch (msg->ipfw.cmd) { 1128 case FF_IPFW_GET: 1129 ret = ff_getsockopt_freebsd(fd, msg->ipfw.level, 1130 msg->ipfw.optname, msg->ipfw.optval, 1131 msg->ipfw.optlen); 1132 break; 1133 case FF_IPFW_SET: 1134 ret = ff_setsockopt_freebsd(fd, msg->ipfw.level, 1135 msg->ipfw.optname, msg->ipfw.optval, 1136 *(msg->ipfw.optlen)); 1137 break; 1138 default: 1139 ret = -1; 1140 errno = ENOTSUP; 1141 break; 1142 } 1143 1144 ff_close(fd); 1145 1146 done: 1147 if (ret < 0) { 1148 msg->result = errno; 1149 } else { 1150 msg->result = 0; 1151 } 1152 } 1153 #endif 1154 1155 static inline void 1156 handle_traffic_msg(struct ff_msg *msg) 1157 { 1158 msg->traffic = ff_traffic; 1159 msg->result = 0; 1160 } 1161 1162 static inline void 1163 handle_default_msg(struct ff_msg *msg) 1164 { 1165 msg->result = ENOTSUP; 1166 } 1167 1168 static inline void 1169 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1170 { 1171 switch (msg->msg_type) { 1172 case FF_SYSCTL: 1173 handle_sysctl_msg(msg); 1174 break; 1175 case FF_IOCTL: 1176 handle_ioctl_msg(msg); 1177 break; 1178 case FF_ROUTE: 1179 handle_route_msg(msg); 1180 break; 1181 case FF_TOP: 1182 handle_top_msg(msg); 1183 break; 1184 #ifdef FF_NETGRAPH 1185 case FF_NGCTL: 1186 handle_ngctl_msg(msg); 1187 break; 1188 #endif 1189 #ifdef FF_IPFW 1190 case FF_IPFW_CTL: 1191 handle_ipfw_msg(msg); 1192 break; 1193 #endif 1194 case FF_TRAFFIC: 1195 handle_traffic_msg(msg); 1196 break; 1197 default: 1198 handle_default_msg(msg); 1199 break; 1200 } 1201 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1202 } 1203 1204 static inline int 1205 process_msg_ring(uint16_t proc_id) 1206 { 1207 void *msg; 1208 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1209 1210 if (unlikely(ret == 0)) { 1211 handle_msg((struct ff_msg *)msg, proc_id); 1212 } 1213 1214 return 0; 1215 } 1216 1217 /* Send burst of packets on an output interface */ 1218 static inline int 1219 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1220 { 1221 struct rte_mbuf **m_table; 1222 int ret; 1223 uint16_t queueid; 1224 1225 queueid = qconf->tx_queue_id[port]; 1226 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1227 1228 if (unlikely(qconf->pcap[port] != NULL)) { 1229 uint16_t i; 1230 for (i = 0; i < n; i++) { 1231 ff_dump_packets(qconf->pcap[port], m_table[i]); 1232 } 1233 } 1234 1235 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1236 ff_traffic.tx_packets += ret; 1237 uint16_t i; 1238 for (i = 0; i < ret; i++) { 1239 ff_traffic.tx_bytes += rte_pktmbuf_pkt_len(m_table[i]); 1240 #ifdef FF_USE_PAGE_ARRAY 1241 if (qconf->tx_mbufs[port].bsd_m_table[i]) 1242 ff_enq_tx_bsdmbuf(port, qconf->tx_mbufs[port].bsd_m_table[i], m_table[i]->nb_segs); 1243 #endif 1244 } 1245 if (unlikely(ret < n)) { 1246 do { 1247 rte_pktmbuf_free(m_table[ret]); 1248 #ifdef FF_USE_PAGE_ARRAY 1249 if ( qconf->tx_mbufs[port].bsd_m_table[ret] ) 1250 ff_mbuf_free(qconf->tx_mbufs[port].bsd_m_table[ret]); 1251 #endif 1252 } while (++ret < n); 1253 } 1254 return 0; 1255 } 1256 1257 /* Enqueue a single packet, and send burst if queue is filled */ 1258 static inline int 1259 send_single_packet(struct rte_mbuf *m, uint8_t port) 1260 { 1261 uint16_t len; 1262 struct lcore_conf *qconf; 1263 1264 qconf = &lcore_conf; 1265 len = qconf->tx_mbufs[port].len; 1266 qconf->tx_mbufs[port].m_table[len] = m; 1267 len++; 1268 1269 /* enough pkts to be sent */ 1270 if (unlikely(len == MAX_PKT_BURST)) { 1271 send_burst(qconf, MAX_PKT_BURST, port); 1272 len = 0; 1273 } 1274 1275 qconf->tx_mbufs[port].len = len; 1276 return 0; 1277 } 1278 1279 int 1280 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1281 int total) 1282 { 1283 #ifdef FF_USE_PAGE_ARRAY 1284 struct lcore_conf *qconf = &lcore_conf; 1285 int len = 0; 1286 1287 len = ff_if_send_onepkt(ctx, m,total); 1288 if (unlikely(len == MAX_PKT_BURST)) { 1289 send_burst(qconf, MAX_PKT_BURST, ctx->port_id); 1290 len = 0; 1291 } 1292 qconf->tx_mbufs[ctx->port_id].len = len; 1293 return 0; 1294 #endif 1295 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1296 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1297 if (head == NULL) { 1298 ff_mbuf_free(m); 1299 return -1; 1300 } 1301 1302 head->pkt_len = total; 1303 head->nb_segs = 0; 1304 1305 int off = 0; 1306 struct rte_mbuf *cur = head, *prev = NULL; 1307 while(total > 0) { 1308 if (cur == NULL) { 1309 cur = rte_pktmbuf_alloc(mbuf_pool); 1310 if (cur == NULL) { 1311 rte_pktmbuf_free(head); 1312 ff_mbuf_free(m); 1313 return -1; 1314 } 1315 } 1316 1317 if (prev != NULL) { 1318 prev->next = cur; 1319 } 1320 head->nb_segs++; 1321 1322 prev = cur; 1323 void *data = rte_pktmbuf_mtod(cur, void*); 1324 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1325 int ret = ff_mbuf_copydata(m, data, off, len); 1326 if (ret < 0) { 1327 rte_pktmbuf_free(head); 1328 ff_mbuf_free(m); 1329 return -1; 1330 } 1331 1332 1333 cur->data_len = len; 1334 off += len; 1335 total -= len; 1336 cur = NULL; 1337 } 1338 1339 struct ff_tx_offload offload = {0}; 1340 ff_mbuf_tx_offload(m, &offload); 1341 1342 void *data = rte_pktmbuf_mtod(head, void*); 1343 1344 if (offload.ip_csum) { 1345 /* ipv6 not supported yet */ 1346 struct ipv4_hdr *iph; 1347 int iph_len; 1348 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1349 iph_len = (iph->version_ihl & 0x0f) << 2; 1350 1351 head->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 1352 head->l2_len = ETHER_HDR_LEN; 1353 head->l3_len = iph_len; 1354 } 1355 1356 if (ctx->hw_features.tx_csum_l4) { 1357 struct ipv4_hdr *iph; 1358 int iph_len; 1359 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1360 iph_len = (iph->version_ihl & 0x0f) << 2; 1361 1362 if (offload.tcp_csum) { 1363 head->ol_flags |= PKT_TX_TCP_CKSUM; 1364 head->l2_len = ETHER_HDR_LEN; 1365 head->l3_len = iph_len; 1366 } 1367 1368 /* 1369 * TCP segmentation offload. 1370 * 1371 * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag 1372 * implies PKT_TX_TCP_CKSUM) 1373 * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 1374 * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and 1375 * write the IP checksum to 0 in the packet 1376 * - fill the mbuf offload information: l2_len, 1377 * l3_len, l4_len, tso_segsz 1378 * - calculate the pseudo header checksum without taking ip_len 1379 * in account, and set it in the TCP header. Refer to 1380 * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be 1381 * used as helpers. 1382 */ 1383 if (offload.tso_seg_size) { 1384 struct tcp_hdr *tcph; 1385 int tcph_len; 1386 tcph = (struct tcp_hdr *)((char *)iph + iph_len); 1387 tcph_len = (tcph->data_off & 0xf0) >> 2; 1388 tcph->cksum = rte_ipv4_phdr_cksum(iph, PKT_TX_TCP_SEG); 1389 1390 head->ol_flags |= PKT_TX_TCP_SEG; 1391 head->l4_len = tcph_len; 1392 head->tso_segsz = offload.tso_seg_size; 1393 } 1394 1395 if (offload.udp_csum) { 1396 head->ol_flags |= PKT_TX_UDP_CKSUM; 1397 head->l2_len = ETHER_HDR_LEN; 1398 head->l3_len = iph_len; 1399 } 1400 } 1401 1402 ff_mbuf_free(m); 1403 1404 return send_single_packet(head, ctx->port_id); 1405 } 1406 1407 static int 1408 main_loop(void *arg) 1409 { 1410 struct loop_routine *lr = (struct loop_routine *)arg; 1411 1412 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1413 uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc, idle_sleep_tsc; 1414 int i, j, nb_rx, idle; 1415 uint16_t port_id, queue_id; 1416 struct lcore_conf *qconf; 1417 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1418 US_PER_S * BURST_TX_DRAIN_US; 1419 struct ff_dpdk_if_context *ctx; 1420 1421 prev_tsc = 0; 1422 usch_tsc = 0; 1423 1424 qconf = &lcore_conf; 1425 1426 while (1) { 1427 cur_tsc = rte_rdtsc(); 1428 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1429 rte_timer_manage(); 1430 } 1431 1432 idle = 1; 1433 sys_tsc = 0; 1434 usr_tsc = 0; 1435 1436 /* 1437 * TX burst queue drain 1438 */ 1439 diff_tsc = cur_tsc - prev_tsc; 1440 if (unlikely(diff_tsc > drain_tsc)) { 1441 for (i = 0; i < qconf->nb_tx_port; i++) { 1442 port_id = qconf->tx_port_id[i]; 1443 if (qconf->tx_mbufs[port_id].len == 0) 1444 continue; 1445 1446 idle = 0; 1447 1448 send_burst(qconf, 1449 qconf->tx_mbufs[port_id].len, 1450 port_id); 1451 qconf->tx_mbufs[port_id].len = 0; 1452 } 1453 1454 prev_tsc = cur_tsc; 1455 } 1456 1457 /* 1458 * Read packet from RX queues 1459 */ 1460 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1461 port_id = qconf->rx_queue_list[i].port_id; 1462 queue_id = qconf->rx_queue_list[i].queue_id; 1463 ctx = veth_ctx[port_id]; 1464 1465 #ifdef FF_KNI 1466 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1467 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1468 } 1469 #endif 1470 1471 process_dispatch_ring(port_id, queue_id, pkts_burst, ctx); 1472 1473 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1474 MAX_PKT_BURST); 1475 if (nb_rx == 0) 1476 continue; 1477 1478 idle = 0; 1479 1480 /* Prefetch first packets */ 1481 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1482 rte_prefetch0(rte_pktmbuf_mtod( 1483 pkts_burst[j], void *)); 1484 } 1485 1486 /* Prefetch and handle already prefetched packets */ 1487 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1488 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1489 j + PREFETCH_OFFSET], void *)); 1490 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1491 } 1492 1493 /* Handle remaining prefetched packets */ 1494 for (; j < nb_rx; j++) { 1495 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1496 } 1497 } 1498 1499 process_msg_ring(qconf->proc_id); 1500 1501 div_tsc = rte_rdtsc(); 1502 1503 if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc > drain_tsc))) { 1504 usch_tsc = cur_tsc; 1505 lr->loop(lr->arg); 1506 } 1507 1508 idle_sleep_tsc = rte_rdtsc(); 1509 if (likely(idle && idle_sleep)) { 1510 usleep(idle_sleep); 1511 end_tsc = rte_rdtsc(); 1512 } else { 1513 end_tsc = idle_sleep_tsc; 1514 } 1515 1516 end_tsc = rte_rdtsc(); 1517 1518 if (usch_tsc == cur_tsc) { 1519 usr_tsc = idle_sleep_tsc - div_tsc; 1520 } 1521 1522 if (!idle) { 1523 sys_tsc = div_tsc - cur_tsc; 1524 ff_top_status.sys_tsc += sys_tsc; 1525 } 1526 1527 ff_top_status.usr_tsc += usr_tsc; 1528 ff_top_status.work_tsc += end_tsc - cur_tsc; 1529 ff_top_status.idle_tsc += end_tsc - cur_tsc - usr_tsc - sys_tsc; 1530 1531 ff_top_status.loops++; 1532 } 1533 1534 return 0; 1535 } 1536 1537 int 1538 ff_dpdk_if_up(void) { 1539 int i; 1540 struct lcore_conf *qconf = &lcore_conf; 1541 for (i = 0; i < qconf->nb_tx_port; i++) { 1542 uint16_t port_id = qconf->tx_port_id[i]; 1543 1544 struct ff_port_cfg *pconf = &qconf->port_cfgs[port_id]; 1545 veth_ctx[port_id] = ff_veth_attach(pconf); 1546 if (veth_ctx[port_id] == NULL) { 1547 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1548 } 1549 } 1550 1551 return 0; 1552 } 1553 1554 void 1555 ff_dpdk_run(loop_func_t loop, void *arg) { 1556 struct loop_routine *lr = rte_malloc(NULL, 1557 sizeof(struct loop_routine), 0); 1558 lr->loop = loop; 1559 lr->arg = arg; 1560 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1561 rte_eal_mp_wait_lcore(); 1562 rte_free(lr); 1563 } 1564 1565 void 1566 ff_dpdk_pktmbuf_free(void *m) 1567 { 1568 rte_pktmbuf_free((struct rte_mbuf *)m); 1569 } 1570 1571 static uint32_t 1572 toeplitz_hash(unsigned keylen, const uint8_t *key, 1573 unsigned datalen, const uint8_t *data) 1574 { 1575 uint32_t hash = 0, v; 1576 u_int i, b; 1577 1578 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1579 1580 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1581 for (i = 0; i < datalen; i++) { 1582 for (b = 0; b < 8; b++) { 1583 if (data[i] & (1<<(7-b))) 1584 hash ^= v; 1585 v <<= 1; 1586 if ((i + 4) < keylen && 1587 (key[i+4] & (1<<(7-b)))) 1588 v |= 1; 1589 } 1590 } 1591 return (hash); 1592 } 1593 1594 int 1595 ff_rss_check(void *softc, uint32_t saddr, uint32_t daddr, 1596 uint16_t sport, uint16_t dport) 1597 { 1598 struct lcore_conf *qconf = &lcore_conf; 1599 struct ff_dpdk_if_context *ctx = ff_veth_softc_to_hostc(softc); 1600 uint16_t nb_queues = qconf->nb_queue_list[ctx->port_id]; 1601 1602 if (nb_queues <= 1) { 1603 return 1; 1604 } 1605 1606 uint16_t reta_size = rss_reta_size[ctx->port_id]; 1607 uint16_t queueid = qconf->tx_queue_id[ctx->port_id]; 1608 1609 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1610 sizeof(dport)]; 1611 1612 unsigned datalen = 0; 1613 1614 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1615 datalen += sizeof(saddr); 1616 1617 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1618 datalen += sizeof(daddr); 1619 1620 bcopy(&sport, &data[datalen], sizeof(sport)); 1621 datalen += sizeof(sport); 1622 1623 bcopy(&dport, &data[datalen], sizeof(dport)); 1624 datalen += sizeof(dport); 1625 1626 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), 1627 default_rsskey_40bytes, datalen, data); 1628 1629 return ((hash & (reta_size - 1)) % nb_queues) == queueid; 1630 } 1631 1632 void 1633 ff_regist_packet_dispatcher(dispatch_func_t func) 1634 { 1635 packet_dispatcher = func; 1636 } 1637 1638 uint64_t 1639 ff_get_tsc_ns() 1640 { 1641 uint64_t cur_tsc = rte_rdtsc(); 1642 uint64_t hz = rte_get_tsc_hz(); 1643 return ((double)cur_tsc/(double)hz) * NS_PER_S; 1644 } 1645 1646 1647