1 /* 2 * Copyright (C) 2017 THL A29 Limited, a Tencent company. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, this 9 * list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * 25 */ 26 #include <assert.h> 27 #include <unistd.h> 28 #include <sys/mman.h> 29 #include <errno.h> 30 31 #include <rte_common.h> 32 #include <rte_byteorder.h> 33 #include <rte_log.h> 34 #include <rte_memory.h> 35 #include <rte_memcpy.h> 36 #include <rte_memzone.h> 37 #include <rte_config.h> 38 #include <rte_eal.h> 39 #include <rte_pci.h> 40 #include <rte_mbuf.h> 41 #include <rte_memory.h> 42 #include <rte_lcore.h> 43 #include <rte_launch.h> 44 #include <rte_ethdev.h> 45 #include <rte_debug.h> 46 #include <rte_common.h> 47 #include <rte_ether.h> 48 #include <rte_malloc.h> 49 #include <rte_cycles.h> 50 #include <rte_timer.h> 51 #include <rte_thash.h> 52 #include <rte_ip.h> 53 #include <rte_tcp.h> 54 #include <rte_udp.h> 55 56 #include "ff_dpdk_if.h" 57 #include "ff_dpdk_pcap.h" 58 #include "ff_dpdk_kni.h" 59 #include "ff_config.h" 60 #include "ff_veth.h" 61 #include "ff_host_interface.h" 62 #include "ff_msg.h" 63 #include "ff_api.h" 64 65 #define MEMPOOL_CACHE_SIZE 256 66 67 #define DISPATCH_RING_SIZE 2048 68 69 #define MSG_RING_SIZE 32 70 71 #define PAGE_SIZE 4096 72 #define PAGE_SHIFT 12 73 #define PAGE_MASK (PAGE_SIZE - 1) 74 #define trunc_page(x) ((x) & ~PAGE_MASK) 75 #define round_page(x) (((x) + PAGE_MASK) & ~PAGE_MASK) 76 77 /* 78 * Configurable number of RX/TX ring descriptors 79 */ 80 #define RX_QUEUE_SIZE 512 81 #define TX_QUEUE_SIZE 512 82 83 #define MAX_PKT_BURST 32 84 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ 85 86 /* 87 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send. 88 */ 89 #define MAX_TX_BURST (MAX_PKT_BURST / 2) 90 91 #define NB_SOCKETS 8 92 93 /* Configure how many packets ahead to prefetch, when reading packets */ 94 #define PREFETCH_OFFSET 3 95 96 #define MAX_RX_QUEUE_PER_LCORE 16 97 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS 98 #define MAX_RX_QUEUE_PER_PORT 128 99 100 #ifdef FF_KNI 101 #define KNI_MBUF_MAX 2048 102 #define KNI_QUEUE_SIZE 2048 103 104 static int enable_kni; 105 static int kni_accept; 106 #endif 107 108 static int numa_on; 109 110 static unsigned idle_sleep; 111 112 static struct rte_timer freebsd_clock; 113 114 // Mellanox Linux's driver key 115 static uint8_t default_rsskey_40bytes[40] = { 116 0xd1, 0x81, 0xc6, 0x2c, 0xf7, 0xf4, 0xdb, 0x5b, 117 0x19, 0x83, 0xa2, 0xfc, 0x94, 0x3e, 0x1a, 0xdb, 118 0xd9, 0x38, 0x9e, 0x6b, 0xd1, 0x03, 0x9c, 0x2c, 119 0xa7, 0x44, 0x99, 0xad, 0x59, 0x3d, 0x56, 0xd9, 120 0xf3, 0x25, 0x3c, 0x06, 0x2a, 0xdc, 0x1f, 0xfc 121 }; 122 123 static struct rte_eth_conf default_port_conf = { 124 .rxmode = { 125 .mq_mode = ETH_MQ_RX_RSS, 126 .max_rx_pkt_len = ETHER_MAX_LEN, 127 .split_hdr_size = 0, /**< hdr buf size */ 128 .header_split = 0, /**< Header Split disabled */ 129 .hw_ip_checksum = 0, /**< IP checksum offload disabled */ 130 .hw_vlan_filter = 0, /**< VLAN filtering disabled */ 131 .hw_vlan_strip = 0, /**< VLAN strip disabled. */ 132 .hw_vlan_extend = 0, /**< Extended VLAN disabled. */ 133 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ 134 .hw_strip_crc = 0, /**< CRC stripped by hardware */ 135 .enable_lro = 0, /**< LRO disabled */ 136 }, 137 .rx_adv_conf = { 138 .rss_conf = { 139 .rss_key = default_rsskey_40bytes, 140 .rss_key_len = 40, 141 .rss_hf = ETH_RSS_PROTO_MASK, 142 }, 143 }, 144 .txmode = { 145 .mq_mode = ETH_MQ_TX_NONE, 146 }, 147 }; 148 149 struct mbuf_table { 150 uint16_t len; 151 struct rte_mbuf *m_table[MAX_PKT_BURST]; 152 #ifdef _USE_PAGE_ARRAY_ 153 void* bsd_m_table[MAX_PKT_BURST]; // save bsd mbuf address which will be freed. 154 #endif 155 156 }; 157 158 struct lcore_rx_queue { 159 uint16_t port_id; 160 uint16_t queue_id; 161 } __rte_cache_aligned; 162 163 struct lcore_conf { 164 uint16_t proc_id; 165 uint16_t socket_id; 166 uint16_t nb_queue_list[RTE_MAX_ETHPORTS]; 167 struct ff_port_cfg *port_cfgs; 168 169 uint16_t nb_rx_queue; 170 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE]; 171 uint16_t nb_tx_port; 172 uint16_t tx_port_id[RTE_MAX_ETHPORTS]; 173 uint16_t tx_queue_id[RTE_MAX_ETHPORTS]; 174 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS]; 175 char *pcap[RTE_MAX_ETHPORTS]; 176 } __rte_cache_aligned; 177 178 static struct lcore_conf lcore_conf; 179 180 static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; 181 182 static struct rte_ring **dispatch_ring[RTE_MAX_ETHPORTS]; 183 static dispatch_func_t packet_dispatcher; 184 185 static uint16_t rss_reta_size[RTE_MAX_ETHPORTS]; 186 187 struct ff_msg_ring { 188 char ring_name[2][RTE_RING_NAMESIZE]; 189 /* ring[0] for lcore recv msg, other send */ 190 /* ring[1] for lcore send msg, other read */ 191 struct rte_ring *ring[2]; 192 } __rte_cache_aligned; 193 194 static struct ff_msg_ring msg_ring[RTE_MAX_LCORE]; 195 static struct rte_mempool *message_pool; 196 197 struct ff_dpdk_if_context { 198 void *sc; 199 void *ifp; 200 uint16_t port_id; 201 struct ff_hw_features hw_features; 202 } __rte_cache_aligned; 203 204 static struct ff_dpdk_if_context *veth_ctx[RTE_MAX_ETHPORTS]; 205 206 static struct ff_top_args ff_top_status; 207 static struct ff_traffic_args ff_traffic; 208 209 #ifdef _USE_PAGE_ARRAY_ 210 211 // ff_ref_pool allocate rte_mbuf without data space, which data point to bsd mbuf's data address. 212 static struct rte_mempool *ff_ref_pool[NB_SOCKETS]; 213 214 // mbuf_txring save mbuf which had bursted into NIC, m_tables has same length with NIC dev's sw_ring. 215 // Then when txring.m_table[x] is reused, the packet in txring.m_table[x] had been transmited by NIC. 216 // that means the mbuf can be freed safely. 217 struct mbuf_txring{ 218 void* m_table[TX_QUEUE_SIZE]; 219 uint16_t head; // next available element. 220 }; 221 #define Head_INC(h) {\ 222 if ( ++h >= TX_QUEUE_SIZE ) \ 223 h = 0;\ 224 }; 225 226 #define Head_DEC(h) do{\ 227 if ( --h < 0 ) \ 228 h = TX_QUEUE_SIZE-1;\ 229 }while(0); 230 231 // bsd mbuf was moved into nic_tx_ring from tmp_tables, after rte_eth_tx_burst() succeed. 232 static struct mbuf_txring nic_tx_ring[RTE_MAX_ETHPORTS]; 233 static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num); 234 static inline void ff_txring_init(struct mbuf_txring* r, uint32_t len); 235 static int ff_dpdk_if_send_ex(struct ff_dpdk_if_context *ctx, void *m, int total); 236 static int ff_mmap_init(); 237 238 typedef struct _list_manager_s 239 { 240 uint64_t *ele; 241 int size; 242 //int FreeNum; 243 int top; 244 }StackList_t; 245 246 static StackList_t ff_mpage_ctl = {0}; 247 static uint64_t ff_page_start = NULL, ff_page_end = NULL; 248 static phys_addr_t* ff_mpage_phy = NULL; 249 static inline void* StkList_pop(StackList_t *p); 250 static inline int StkList_push(StackList_t * p, uint64_t val); 251 #endif 252 253 extern void ff_hardclock(void); 254 255 static void 256 ff_hardclock_job(__rte_unused struct rte_timer *timer, 257 __rte_unused void *arg) { 258 ff_hardclock(); 259 ff_update_current_ts(); 260 } 261 262 struct ff_dpdk_if_context * 263 ff_dpdk_register_if(void *sc, void *ifp, struct ff_port_cfg *cfg) 264 { 265 struct ff_dpdk_if_context *ctx; 266 267 ctx = calloc(1, sizeof(struct ff_dpdk_if_context)); 268 if (ctx == NULL) 269 return NULL; 270 271 ctx->sc = sc; 272 ctx->ifp = ifp; 273 ctx->port_id = cfg->port_id; 274 ctx->hw_features = cfg->hw_features; 275 276 return ctx; 277 } 278 279 void 280 ff_dpdk_deregister_if(struct ff_dpdk_if_context *ctx) 281 { 282 free(ctx); 283 } 284 285 static void 286 check_all_ports_link_status(void) 287 { 288 #define CHECK_INTERVAL 100 /* 100ms */ 289 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 290 291 uint16_t portid; 292 uint8_t count, all_ports_up, print_flag = 0; 293 struct rte_eth_link link; 294 295 printf("\nChecking link status"); 296 fflush(stdout); 297 298 int i, nb_ports; 299 nb_ports = ff_global_cfg.dpdk.nb_ports; 300 for (count = 0; count <= MAX_CHECK_TIME; count++) { 301 all_ports_up = 1; 302 for (i = 0; i < nb_ports; i++) { 303 uint16_t portid = ff_global_cfg.dpdk.portid_list[i]; 304 memset(&link, 0, sizeof(link)); 305 rte_eth_link_get_nowait(portid, &link); 306 307 /* print link status if flag set */ 308 if (print_flag == 1) { 309 if (link.link_status) { 310 printf("Port %d Link Up - speed %u " 311 "Mbps - %s\n", (int)portid, 312 (unsigned)link.link_speed, 313 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 314 ("full-duplex") : ("half-duplex\n")); 315 } else { 316 printf("Port %d Link Down\n", (int)portid); 317 } 318 continue; 319 } 320 /* clear all_ports_up flag if any link down */ 321 if (link.link_status == 0) { 322 all_ports_up = 0; 323 break; 324 } 325 } 326 327 /* after finally printing all link status, get out */ 328 if (print_flag == 1) 329 break; 330 331 if (all_ports_up == 0) { 332 printf("."); 333 fflush(stdout); 334 rte_delay_ms(CHECK_INTERVAL); 335 } 336 337 /* set the print_flag if all ports up or timeout */ 338 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 339 print_flag = 1; 340 printf("done\n"); 341 } 342 } 343 } 344 345 static int 346 init_lcore_conf(void) 347 { 348 uint8_t nb_dev_ports = rte_eth_dev_count(); 349 if (nb_dev_ports == 0) { 350 rte_exit(EXIT_FAILURE, "No probed ethernet devices\n"); 351 } 352 353 if (ff_global_cfg.dpdk.max_portid >= nb_dev_ports) { 354 rte_exit(EXIT_FAILURE, "this machine doesn't have port %d.\n", 355 ff_global_cfg.dpdk.max_portid); 356 } 357 358 lcore_conf.port_cfgs = ff_global_cfg.dpdk.port_cfgs; 359 lcore_conf.proc_id = ff_global_cfg.dpdk.proc_id; 360 361 uint16_t proc_id; 362 for (proc_id = 0; proc_id < ff_global_cfg.dpdk.nb_procs; proc_id++) { 363 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[proc_id]; 364 if (!lcore_config[lcore_id].detected) { 365 rte_exit(EXIT_FAILURE, "lcore %u unavailable\n", lcore_id); 366 } 367 } 368 369 uint16_t socket_id = 0; 370 if (numa_on) { 371 socket_id = rte_lcore_to_socket_id(rte_lcore_id()); 372 } 373 374 lcore_conf.socket_id = socket_id; 375 376 uint16_t lcore_id = ff_global_cfg.dpdk.proc_lcore[lcore_conf.proc_id]; 377 int j; 378 for (j = 0; j < ff_global_cfg.dpdk.nb_ports; ++j) { 379 uint16_t port_id = ff_global_cfg.dpdk.portid_list[j]; 380 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 381 382 int queueid = -1; 383 int i; 384 for (i = 0; i < pconf->nb_lcores; i++) { 385 if (pconf->lcore_list[i] == lcore_id) { 386 queueid = i; 387 } 388 } 389 if (queueid < 0) { 390 continue; 391 } 392 printf("lcore: %u, port: %u, queue: %u\n", lcore_id, port_id, queueid); 393 uint16_t nb_rx_queue = lcore_conf.nb_rx_queue; 394 lcore_conf.rx_queue_list[nb_rx_queue].port_id = port_id; 395 lcore_conf.rx_queue_list[nb_rx_queue].queue_id = queueid; 396 lcore_conf.nb_rx_queue++; 397 398 lcore_conf.tx_queue_id[port_id] = queueid; 399 lcore_conf.tx_port_id[lcore_conf.nb_tx_port] = port_id; 400 lcore_conf.nb_tx_port++; 401 402 lcore_conf.pcap[port_id] = pconf->pcap; 403 lcore_conf.nb_queue_list[port_id] = pconf->nb_lcores; 404 } 405 406 if (lcore_conf.nb_rx_queue == 0) { 407 rte_exit(EXIT_FAILURE, "lcore %u has nothing to do\n", lcore_id); 408 } 409 410 return 0; 411 } 412 413 static int 414 init_mem_pool(void) 415 { 416 uint8_t nb_ports = ff_global_cfg.dpdk.nb_ports; 417 uint32_t nb_lcores = ff_global_cfg.dpdk.nb_procs; 418 uint32_t nb_tx_queue = nb_lcores; 419 uint32_t nb_rx_queue = lcore_conf.nb_rx_queue * nb_lcores; 420 421 unsigned nb_mbuf = RTE_MAX ( 422 (nb_rx_queue*RX_QUEUE_SIZE + 423 nb_ports*nb_lcores*MAX_PKT_BURST + 424 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 425 nb_lcores*MEMPOOL_CACHE_SIZE + 426 #ifdef FF_KNI 427 nb_ports*KNI_MBUF_MAX + 428 nb_ports*KNI_QUEUE_SIZE + 429 #endif 430 nb_lcores*nb_ports*DISPATCH_RING_SIZE), 431 (unsigned)8192); 432 433 unsigned socketid = 0; 434 uint16_t i, lcore_id; 435 char s[64]; 436 437 for (i = 0; i < ff_global_cfg.dpdk.nb_procs; i++) { 438 lcore_id = ff_global_cfg.dpdk.proc_lcore[i]; 439 if (numa_on) { 440 socketid = rte_lcore_to_socket_id(lcore_id); 441 } 442 443 if (socketid >= NB_SOCKETS) { 444 rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n", 445 socketid, i, NB_SOCKETS); 446 } 447 448 if (pktmbuf_pool[socketid] != NULL) { 449 continue; 450 } 451 452 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 453 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 454 pktmbuf_pool[socketid] = 455 rte_pktmbuf_pool_create(s, nb_mbuf, 456 MEMPOOL_CACHE_SIZE, 0, 457 RTE_MBUF_DEFAULT_BUF_SIZE, socketid); 458 } else { 459 snprintf(s, sizeof(s), "mbuf_pool_%d", socketid); 460 pktmbuf_pool[socketid] = rte_mempool_lookup(s); 461 } 462 463 if (pktmbuf_pool[socketid] == NULL) { 464 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool on socket %d\n", socketid); 465 } else { 466 printf("create mbuf pool on socket %d\n", socketid); 467 } 468 469 #ifdef _USE_PAGE_ARRAY_ 470 if (ff_ref_pool[socketid] != NULL) { 471 continue; 472 } 473 nb_mbuf = RTE_MAX ( 474 nb_ports*nb_lcores*MAX_PKT_BURST + 475 nb_ports*nb_tx_queue*TX_QUEUE_SIZE + 476 nb_lcores*MEMPOOL_CACHE_SIZE, 477 (unsigned)4096); 478 snprintf(s, sizeof(s), "ff_ref_pool_%d", socketid); 479 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 480 ff_ref_pool[socketid] = rte_pktmbuf_pool_create(s, nb_mbuf, MEMPOOL_CACHE_SIZE, 0, 0, socketid); 481 } else { 482 ff_ref_pool[socketid] = rte_mempool_lookup(s); 483 } 484 #endif 485 } 486 487 return 0; 488 } 489 490 static struct rte_ring * 491 create_ring(const char *name, unsigned count, int socket_id, unsigned flags) 492 { 493 struct rte_ring *ring; 494 495 if (name == NULL) { 496 rte_exit(EXIT_FAILURE, "create ring failed, no name!\n"); 497 } 498 499 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 500 ring = rte_ring_create(name, count, socket_id, flags); 501 } else { 502 ring = rte_ring_lookup(name); 503 } 504 505 if (ring == NULL) { 506 rte_exit(EXIT_FAILURE, "create ring:%s failed!\n", name); 507 } 508 509 return ring; 510 } 511 512 static int 513 init_dispatch_ring(void) 514 { 515 int j; 516 char name_buf[RTE_RING_NAMESIZE]; 517 int queueid; 518 519 unsigned socketid = lcore_conf.socket_id; 520 521 /* Create ring according to ports actually being used. */ 522 int nb_ports = ff_global_cfg.dpdk.nb_ports; 523 for (j = 0; j < nb_ports; j++) { 524 uint16_t portid = ff_global_cfg.dpdk.portid_list[j]; 525 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[portid]; 526 int nb_queues = pconf->nb_lcores; 527 if (dispatch_ring[portid] == NULL) { 528 snprintf(name_buf, RTE_RING_NAMESIZE, "ring_ptr_p%d", portid); 529 530 dispatch_ring[portid] = rte_zmalloc(name_buf, 531 sizeof(struct rte_ring *) * nb_queues, 532 RTE_CACHE_LINE_SIZE); 533 if (dispatch_ring[portid] == NULL) { 534 rte_exit(EXIT_FAILURE, "rte_zmalloc(%s (struct rte_ring*)) " 535 "failed\n", name_buf); 536 } 537 } 538 539 for(queueid = 0; queueid < nb_queues; ++queueid) { 540 snprintf(name_buf, RTE_RING_NAMESIZE, "dispatch_ring_p%d_q%d", 541 portid, queueid); 542 dispatch_ring[portid][queueid] = create_ring(name_buf, 543 DISPATCH_RING_SIZE, socketid, RING_F_SC_DEQ); 544 545 if (dispatch_ring[portid][queueid] == NULL) 546 rte_panic("create ring:%s failed!\n", name_buf); 547 548 printf("create ring:%s success, %u ring entries are now free!\n", 549 name_buf, rte_ring_free_count(dispatch_ring[portid][queueid])); 550 } 551 } 552 553 return 0; 554 } 555 556 static void 557 ff_msg_init(struct rte_mempool *mp, 558 __attribute__((unused)) void *opaque_arg, 559 void *obj, __attribute__((unused)) unsigned i) 560 { 561 struct ff_msg *msg = (struct ff_msg *)obj; 562 msg->msg_type = FF_UNKNOWN; 563 msg->buf_addr = (char *)msg + sizeof(struct ff_msg); 564 msg->buf_len = mp->elt_size - sizeof(struct ff_msg); 565 } 566 567 static int 568 init_msg_ring(void) 569 { 570 uint16_t i; 571 uint16_t nb_procs = ff_global_cfg.dpdk.nb_procs; 572 unsigned socketid = lcore_conf.socket_id; 573 574 /* Create message buffer pool */ 575 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 576 message_pool = rte_mempool_create(FF_MSG_POOL, 577 MSG_RING_SIZE * 2 * nb_procs, 578 MAX_MSG_BUF_SIZE, MSG_RING_SIZE / 2, 0, 579 NULL, NULL, ff_msg_init, NULL, 580 socketid, 0); 581 } else { 582 message_pool = rte_mempool_lookup(FF_MSG_POOL); 583 } 584 585 if (message_pool == NULL) { 586 rte_panic("Create msg mempool failed\n"); 587 } 588 589 for(i = 0; i < nb_procs; ++i) { 590 snprintf(msg_ring[i].ring_name[0], RTE_RING_NAMESIZE, 591 "%s%u", FF_MSG_RING_IN, i); 592 snprintf(msg_ring[i].ring_name[1], RTE_RING_NAMESIZE, 593 "%s%u", FF_MSG_RING_OUT, i); 594 595 msg_ring[i].ring[0] = create_ring(msg_ring[i].ring_name[0], 596 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 597 if (msg_ring[i].ring[0] == NULL) 598 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 599 600 msg_ring[i].ring[1] = create_ring(msg_ring[i].ring_name[1], 601 MSG_RING_SIZE, socketid, RING_F_SP_ENQ | RING_F_SC_DEQ); 602 if (msg_ring[i].ring[1] == NULL) 603 rte_panic("create ring::%s failed!\n", msg_ring[i].ring_name[0]); 604 } 605 606 return 0; 607 } 608 609 #ifdef FF_KNI 610 static int 611 init_kni(void) 612 { 613 int nb_ports = rte_eth_dev_count(); 614 kni_accept = 0; 615 if(strcasecmp(ff_global_cfg.kni.method, "accept") == 0) 616 kni_accept = 1; 617 618 ff_kni_init(nb_ports, ff_global_cfg.kni.tcp_port, 619 ff_global_cfg.kni.udp_port); 620 621 unsigned socket_id = lcore_conf.socket_id; 622 struct rte_mempool *mbuf_pool = pktmbuf_pool[socket_id]; 623 624 nb_ports = ff_global_cfg.dpdk.nb_ports; 625 int i, ret; 626 for (i = 0; i < nb_ports; i++) { 627 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 628 ff_kni_alloc(port_id, socket_id, mbuf_pool, KNI_QUEUE_SIZE); 629 } 630 631 return 0; 632 } 633 #endif 634 635 static void 636 set_rss_table(uint16_t port_id, uint16_t reta_size, uint16_t nb_queues) 637 { 638 if (reta_size == 0) { 639 return; 640 } 641 642 int reta_conf_size = RTE_MAX(1, reta_size / RTE_RETA_GROUP_SIZE); 643 struct rte_eth_rss_reta_entry64 reta_conf[reta_conf_size]; 644 645 /* config HW indirection table */ 646 unsigned i, j, hash=0; 647 for (i = 0; i < reta_conf_size; i++) { 648 reta_conf[i].mask = ~0ULL; 649 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) { 650 reta_conf[i].reta[j] = hash++ % nb_queues; 651 } 652 } 653 654 if (rte_eth_dev_rss_reta_update(port_id, reta_conf, reta_size)) { 655 rte_exit(EXIT_FAILURE, "port[%d], failed to update rss table\n", 656 port_id); 657 } 658 } 659 660 static int 661 init_port_start(void) 662 { 663 int nb_ports = ff_global_cfg.dpdk.nb_ports; 664 unsigned socketid = 0; 665 struct rte_mempool *mbuf_pool; 666 uint16_t i; 667 668 for (i = 0; i < nb_ports; i++) { 669 uint16_t port_id = ff_global_cfg.dpdk.portid_list[i]; 670 struct ff_port_cfg *pconf = &ff_global_cfg.dpdk.port_cfgs[port_id]; 671 uint16_t nb_queues = pconf->nb_lcores; 672 673 struct rte_eth_dev_info dev_info; 674 rte_eth_dev_info_get(port_id, &dev_info); 675 676 if (nb_queues > dev_info.max_rx_queues) { 677 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_rx_queues[%d]\n", 678 nb_queues, 679 dev_info.max_rx_queues); 680 } 681 682 if (nb_queues > dev_info.max_tx_queues) { 683 rte_exit(EXIT_FAILURE, "num_procs[%d] bigger than max_tx_queues[%d]\n", 684 nb_queues, 685 dev_info.max_tx_queues); 686 } 687 688 struct ether_addr addr; 689 rte_eth_macaddr_get(port_id, &addr); 690 printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8 691 " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n", 692 (unsigned)port_id, 693 addr.addr_bytes[0], addr.addr_bytes[1], 694 addr.addr_bytes[2], addr.addr_bytes[3], 695 addr.addr_bytes[4], addr.addr_bytes[5]); 696 697 rte_memcpy(pconf->mac, 698 addr.addr_bytes, ETHER_ADDR_LEN); 699 700 /* Clear txq_flags - we do not need multi-mempool and refcnt */ 701 dev_info.default_txconf.txq_flags = ETH_TXQ_FLAGS_NOMULTMEMP | 702 ETH_TXQ_FLAGS_NOREFCOUNT; 703 704 /* Disable features that are not supported by port's HW */ 705 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { 706 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMUDP; 707 } 708 709 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 710 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMTCP; 711 } 712 713 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM)) { 714 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOXSUMSCTP; 715 } 716 717 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT)) { 718 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL; 719 } 720 721 if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) && 722 !(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TSO)) { 723 dev_info.default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS; 724 } 725 726 struct rte_eth_conf port_conf = {0}; 727 728 /* Set RSS mode */ 729 port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 730 port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_PROTO_MASK; 731 port_conf.rx_adv_conf.rss_conf.rss_key = default_rsskey_40bytes; 732 port_conf.rx_adv_conf.rss_conf.rss_key_len = 40; 733 734 /* Set Rx VLAN stripping */ 735 if (ff_global_cfg.dpdk.vlan_strip) { 736 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) { 737 port_conf.rxmode.hw_vlan_strip = 1; 738 } 739 } 740 741 /* Enable HW CRC stripping */ 742 port_conf.rxmode.hw_strip_crc = 1; 743 744 /* FIXME: Enable TCP LRO ?*/ 745 #if 0 746 if (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_LRO) { 747 printf("LRO is supported\n"); 748 port_conf.rxmode.enable_lro = 1; 749 pconf->hw_features.rx_lro = 1; 750 } 751 #endif 752 753 /* Set Rx checksum checking */ 754 if ((dev_info.rx_offload_capa & DEV_RX_OFFLOAD_IPV4_CKSUM) && 755 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_UDP_CKSUM) && 756 (dev_info.rx_offload_capa & DEV_RX_OFFLOAD_TCP_CKSUM)) { 757 printf("RX checksum offload supported\n"); 758 port_conf.rxmode.hw_ip_checksum = 1; 759 pconf->hw_features.rx_csum = 1; 760 } 761 762 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)) { 763 printf("TX ip checksum offload supported\n"); 764 pconf->hw_features.tx_csum_ip = 1; 765 } 766 767 if ((dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) && 768 (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM)) { 769 printf("TX TCP&UDP checksum offload supported\n"); 770 pconf->hw_features.tx_csum_l4 = 1; 771 } 772 773 if (ff_global_cfg.dpdk.tso) { 774 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) { 775 printf("TSO is supported\n"); 776 pconf->hw_features.tx_tso = 1; 777 } 778 } else { 779 printf("TSO is disabled\n"); 780 } 781 782 if (dev_info.reta_size) { 783 /* reta size must be power of 2 */ 784 assert((dev_info.reta_size & (dev_info.reta_size - 1)) == 0); 785 786 rss_reta_size[port_id] = dev_info.reta_size; 787 printf("port[%d]: rss table size: %d\n", port_id, 788 dev_info.reta_size); 789 } 790 791 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 792 continue; 793 } 794 795 int ret = rte_eth_dev_configure(port_id, nb_queues, nb_queues, &port_conf); 796 if (ret != 0) { 797 return ret; 798 } 799 uint16_t q; 800 for (q = 0; q < nb_queues; q++) { 801 if (numa_on) { 802 uint16_t lcore_id = lcore_conf.port_cfgs[port_id].lcore_list[q]; 803 socketid = rte_lcore_to_socket_id(lcore_id); 804 } 805 mbuf_pool = pktmbuf_pool[socketid]; 806 807 ret = rte_eth_tx_queue_setup(port_id, q, TX_QUEUE_SIZE, 808 socketid, &dev_info.default_txconf); 809 if (ret < 0) { 810 return ret; 811 } 812 813 ret = rte_eth_rx_queue_setup(port_id, q, RX_QUEUE_SIZE, 814 socketid, &dev_info.default_rxconf, mbuf_pool); 815 if (ret < 0) { 816 return ret; 817 } 818 } 819 820 ret = rte_eth_dev_start(port_id); 821 if (ret < 0) { 822 return ret; 823 } 824 825 if (nb_queues > 1) { 826 /* set HW rss hash function to Toeplitz. */ 827 if (!rte_eth_dev_filter_supported(port_id, RTE_ETH_FILTER_HASH)) { 828 struct rte_eth_hash_filter_info info = {0}; 829 info.info_type = RTE_ETH_HASH_FILTER_GLOBAL_CONFIG; 830 info.info.global_conf.hash_func = RTE_ETH_HASH_FUNCTION_TOEPLITZ; 831 832 if (rte_eth_dev_filter_ctrl(port_id, RTE_ETH_FILTER_HASH, 833 RTE_ETH_FILTER_SET, &info) < 0) { 834 rte_exit(EXIT_FAILURE, "port[%d] set hash func failed\n", 835 port_id); 836 } 837 } 838 839 set_rss_table(port_id, dev_info.reta_size, nb_queues); 840 } 841 842 /* Enable RX in promiscuous mode for the Ethernet device. */ 843 if (ff_global_cfg.dpdk.promiscuous) { 844 rte_eth_promiscuous_enable(port_id); 845 ret = rte_eth_promiscuous_get(port_id); 846 if (ret == 1) { 847 printf("set port %u to promiscuous mode ok\n", port_id); 848 } else { 849 printf("set port %u to promiscuous mode error\n", port_id); 850 } 851 } 852 853 /* Enable pcap dump */ 854 if (pconf->pcap) { 855 ff_enable_pcap(pconf->pcap); 856 } 857 } 858 859 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 860 check_all_ports_link_status(); 861 } 862 863 return 0; 864 } 865 866 static int 867 init_clock(void) 868 { 869 rte_timer_subsystem_init(); 870 uint64_t hz = rte_get_timer_hz(); 871 uint64_t intrs = MS_PER_S/ff_global_cfg.freebsd.hz; 872 uint64_t tsc = (hz + MS_PER_S - 1) / MS_PER_S*intrs; 873 874 rte_timer_init(&freebsd_clock); 875 rte_timer_reset(&freebsd_clock, tsc, PERIODICAL, 876 rte_lcore_id(), &ff_hardclock_job, NULL); 877 878 ff_update_current_ts(); 879 880 return 0; 881 } 882 883 int 884 ff_dpdk_init(int argc, char **argv) 885 { 886 if (ff_global_cfg.dpdk.nb_procs < 1 || 887 ff_global_cfg.dpdk.nb_procs > RTE_MAX_LCORE || 888 ff_global_cfg.dpdk.proc_id >= ff_global_cfg.dpdk.nb_procs || 889 ff_global_cfg.dpdk.proc_id < 0) { 890 printf("param num_procs[%d] or proc_id[%d] error!\n", 891 ff_global_cfg.dpdk.nb_procs, 892 ff_global_cfg.dpdk.proc_id); 893 exit(1); 894 } 895 896 int ret = rte_eal_init(argc, argv); 897 if (ret < 0) { 898 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 899 } 900 901 numa_on = ff_global_cfg.dpdk.numa_on; 902 903 idle_sleep = ff_global_cfg.dpdk.idle_sleep; 904 905 init_lcore_conf(); 906 907 init_mem_pool(); 908 909 init_dispatch_ring(); 910 911 init_msg_ring(); 912 913 #ifdef FF_KNI 914 enable_kni = ff_global_cfg.kni.enable; 915 if (enable_kni) { 916 init_kni(); 917 } 918 #endif 919 920 #ifdef _USE_PAGE_ARRAY_ 921 ff_mmap_init(); 922 #endif 923 924 925 ret = init_port_start(); 926 if (ret < 0) { 927 rte_exit(EXIT_FAILURE, "init_port_start failed\n"); 928 } 929 930 init_clock(); 931 932 return 0; 933 } 934 935 static void 936 ff_veth_input(const struct ff_dpdk_if_context *ctx, struct rte_mbuf *pkt) 937 { 938 uint8_t rx_csum = ctx->hw_features.rx_csum; 939 if (rx_csum) { 940 if (pkt->ol_flags & (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)) { 941 rte_pktmbuf_free(pkt); 942 return; 943 } 944 } 945 946 /* 947 * FIXME: should we save pkt->vlan_tci 948 * if (pkt->ol_flags & PKT_RX_VLAN_PKT) 949 */ 950 951 void *data = rte_pktmbuf_mtod(pkt, void*); 952 uint16_t len = rte_pktmbuf_data_len(pkt); 953 954 void *hdr = ff_mbuf_gethdr(pkt, pkt->pkt_len, data, len, rx_csum); 955 if (hdr == NULL) { 956 rte_pktmbuf_free(pkt); 957 return; 958 } 959 960 struct rte_mbuf *pn = pkt->next; 961 void *prev = hdr; 962 while(pn != NULL) { 963 data = rte_pktmbuf_mtod(pn, void*); 964 len = rte_pktmbuf_data_len(pn); 965 966 void *mb = ff_mbuf_get(prev, data, len); 967 if (mb == NULL) { 968 ff_mbuf_free(hdr); 969 rte_pktmbuf_free(pkt); 970 return; 971 } 972 pn = pn->next; 973 prev = mb; 974 } 975 976 ff_veth_process_packet(ctx->ifp, hdr); 977 } 978 979 static enum FilterReturn 980 protocol_filter(const void *data, uint16_t len) 981 { 982 if(len < ETHER_HDR_LEN) 983 return FILTER_UNKNOWN; 984 985 const struct ether_hdr *hdr; 986 hdr = (const struct ether_hdr *)data; 987 988 if(ntohs(hdr->ether_type) == ETHER_TYPE_ARP) 989 return FILTER_ARP; 990 991 #ifndef FF_KNI 992 return FILTER_UNKNOWN; 993 #else 994 if (!enable_kni) { 995 return FILTER_UNKNOWN; 996 } 997 998 if(ntohs(hdr->ether_type) != ETHER_TYPE_IPv4) 999 return FILTER_UNKNOWN; 1000 1001 return ff_kni_proto_filter(data + ETHER_HDR_LEN, 1002 len - ETHER_HDR_LEN); 1003 #endif 1004 } 1005 1006 static inline void 1007 pktmbuf_deep_attach(struct rte_mbuf *mi, const struct rte_mbuf *m) 1008 { 1009 struct rte_mbuf *md; 1010 void *src, *dst; 1011 1012 dst = rte_pktmbuf_mtod(mi, void *); 1013 src = rte_pktmbuf_mtod(m, void *); 1014 1015 mi->data_len = m->data_len; 1016 rte_memcpy(dst, src, m->data_len); 1017 1018 mi->port = m->port; 1019 mi->vlan_tci = m->vlan_tci; 1020 mi->vlan_tci_outer = m->vlan_tci_outer; 1021 mi->tx_offload = m->tx_offload; 1022 mi->hash = m->hash; 1023 mi->ol_flags = m->ol_flags; 1024 mi->packet_type = m->packet_type; 1025 } 1026 1027 /* copied from rte_pktmbuf_clone */ 1028 static inline struct rte_mbuf * 1029 pktmbuf_deep_clone(const struct rte_mbuf *md, 1030 struct rte_mempool *mp) 1031 { 1032 struct rte_mbuf *mc, *mi, **prev; 1033 uint32_t pktlen; 1034 uint8_t nseg; 1035 1036 if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL)) 1037 return NULL; 1038 1039 mi = mc; 1040 prev = &mi->next; 1041 pktlen = md->pkt_len; 1042 nseg = 0; 1043 1044 do { 1045 nseg++; 1046 pktmbuf_deep_attach(mi, md); 1047 *prev = mi; 1048 prev = &mi->next; 1049 } while ((md = md->next) != NULL && 1050 (mi = rte_pktmbuf_alloc(mp)) != NULL); 1051 1052 *prev = NULL; 1053 mc->nb_segs = nseg; 1054 mc->pkt_len = pktlen; 1055 1056 /* Allocation of new indirect segment failed */ 1057 if (unlikely (mi == NULL)) { 1058 rte_pktmbuf_free(mc); 1059 return NULL; 1060 } 1061 1062 __rte_mbuf_sanity_check(mc, 1); 1063 return mc; 1064 } 1065 1066 static inline void 1067 process_packets(uint16_t port_id, uint16_t queue_id, struct rte_mbuf **bufs, 1068 uint16_t count, const struct ff_dpdk_if_context *ctx, int pkts_from_ring) 1069 { 1070 struct lcore_conf *qconf = &lcore_conf; 1071 uint16_t nb_queues = qconf->nb_queue_list[port_id]; 1072 1073 uint16_t i; 1074 for (i = 0; i < count; i++) { 1075 struct rte_mbuf *rtem = bufs[i]; 1076 1077 if (unlikely(qconf->pcap[port_id] != NULL)) { 1078 if (!pkts_from_ring) { 1079 ff_dump_packets(qconf->pcap[port_id], rtem); 1080 } 1081 } 1082 1083 void *data = rte_pktmbuf_mtod(rtem, void*); 1084 uint16_t len = rte_pktmbuf_data_len(rtem); 1085 1086 if (!pkts_from_ring) { 1087 ff_traffic.rx_packets++; 1088 ff_traffic.rx_bytes += len; 1089 } 1090 1091 if (!pkts_from_ring && packet_dispatcher) { 1092 int ret = (*packet_dispatcher)(data, len, queue_id, nb_queues); 1093 if (ret < 0 || ret >= nb_queues) { 1094 rte_pktmbuf_free(rtem); 1095 continue; 1096 } 1097 1098 if (ret != queue_id) { 1099 ret = rte_ring_enqueue(dispatch_ring[port_id][ret], rtem); 1100 if (ret < 0) 1101 rte_pktmbuf_free(rtem); 1102 1103 continue; 1104 } 1105 } 1106 1107 enum FilterReturn filter = protocol_filter(data, len); 1108 if (filter == FILTER_ARP) { 1109 struct rte_mempool *mbuf_pool; 1110 struct rte_mbuf *mbuf_clone; 1111 if (!pkts_from_ring) { 1112 uint16_t j; 1113 for(j = 0; j < nb_queues; ++j) { 1114 if(j == queue_id) 1115 continue; 1116 1117 unsigned socket_id = 0; 1118 if (numa_on) { 1119 uint16_t lcore_id = qconf->port_cfgs[port_id].lcore_list[j]; 1120 socket_id = rte_lcore_to_socket_id(lcore_id); 1121 } 1122 mbuf_pool = pktmbuf_pool[socket_id]; 1123 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1124 if(mbuf_clone) { 1125 int ret = rte_ring_enqueue(dispatch_ring[port_id][j], 1126 mbuf_clone); 1127 if (ret < 0) 1128 rte_pktmbuf_free(mbuf_clone); 1129 } 1130 } 1131 } 1132 1133 #ifdef FF_KNI 1134 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1135 mbuf_pool = pktmbuf_pool[qconf->socket_id]; 1136 mbuf_clone = pktmbuf_deep_clone(rtem, mbuf_pool); 1137 if(mbuf_clone) { 1138 ff_kni_enqueue(port_id, mbuf_clone); 1139 } 1140 } 1141 #endif 1142 ff_veth_input(ctx, rtem); 1143 #ifdef FF_KNI 1144 } else if (enable_kni && 1145 ((filter == FILTER_KNI && kni_accept) || 1146 (filter == FILTER_UNKNOWN && !kni_accept)) ) { 1147 ff_kni_enqueue(port_id, rtem); 1148 #endif 1149 } else { 1150 ff_veth_input(ctx, rtem); 1151 } 1152 } 1153 } 1154 1155 static inline int 1156 process_dispatch_ring(uint16_t port_id, uint16_t queue_id, 1157 struct rte_mbuf **pkts_burst, const struct ff_dpdk_if_context *ctx) 1158 { 1159 /* read packet from ring buf and to process */ 1160 uint16_t nb_rb; 1161 nb_rb = rte_ring_dequeue_burst(dispatch_ring[port_id][queue_id], 1162 (void **)pkts_burst, MAX_PKT_BURST, NULL); 1163 1164 if(nb_rb > 0) { 1165 process_packets(port_id, queue_id, pkts_burst, nb_rb, ctx, 1); 1166 } 1167 1168 return 0; 1169 } 1170 1171 static inline void 1172 handle_sysctl_msg(struct ff_msg *msg) 1173 { 1174 int ret = ff_sysctl(msg->sysctl.name, msg->sysctl.namelen, 1175 msg->sysctl.old, msg->sysctl.oldlenp, msg->sysctl.new, 1176 msg->sysctl.newlen); 1177 1178 if (ret < 0) { 1179 msg->result = errno; 1180 } else { 1181 msg->result = 0; 1182 } 1183 } 1184 1185 static inline void 1186 handle_ioctl_msg(struct ff_msg *msg) 1187 { 1188 int fd, ret; 1189 fd = ff_socket(AF_INET, SOCK_DGRAM, 0); 1190 if (fd < 0) { 1191 ret = -1; 1192 goto done; 1193 } 1194 1195 ret = ff_ioctl_freebsd(fd, msg->ioctl.cmd, msg->ioctl.data); 1196 1197 ff_close(fd); 1198 1199 done: 1200 if (ret < 0) { 1201 msg->result = errno; 1202 } else { 1203 msg->result = 0; 1204 } 1205 } 1206 1207 static inline void 1208 handle_route_msg(struct ff_msg *msg) 1209 { 1210 int ret = ff_rtioctl(msg->route.fib, msg->route.data, 1211 &msg->route.len, msg->route.maxlen); 1212 if (ret < 0) { 1213 msg->result = errno; 1214 } else { 1215 msg->result = 0; 1216 } 1217 } 1218 1219 static inline void 1220 handle_top_msg(struct ff_msg *msg) 1221 { 1222 msg->top = ff_top_status; 1223 msg->result = 0; 1224 } 1225 1226 #ifdef FF_NETGRAPH 1227 static inline void 1228 handle_ngctl_msg(struct ff_msg *msg) 1229 { 1230 int ret = ff_ngctl(msg->ngctl.cmd, msg->ngctl.data); 1231 if (ret < 0) { 1232 msg->result = errno; 1233 } else { 1234 msg->result = 0; 1235 msg->ngctl.ret = ret; 1236 } 1237 } 1238 #endif 1239 1240 #ifdef FF_IPFW 1241 static inline void 1242 handle_ipfw_msg(struct ff_msg *msg) 1243 { 1244 int fd, ret; 1245 fd = ff_socket(AF_INET, SOCK_RAW, IPPROTO_RAW); 1246 if (fd < 0) { 1247 ret = -1; 1248 goto done; 1249 } 1250 1251 switch (msg->ipfw.cmd) { 1252 case FF_IPFW_GET: 1253 ret = ff_getsockopt_freebsd(fd, msg->ipfw.level, 1254 msg->ipfw.optname, msg->ipfw.optval, 1255 msg->ipfw.optlen); 1256 break; 1257 case FF_IPFW_SET: 1258 ret = ff_setsockopt_freebsd(fd, msg->ipfw.level, 1259 msg->ipfw.optname, msg->ipfw.optval, 1260 *(msg->ipfw.optlen)); 1261 break; 1262 default: 1263 ret = -1; 1264 errno = ENOTSUP; 1265 break; 1266 } 1267 1268 ff_close(fd); 1269 1270 done: 1271 if (ret < 0) { 1272 msg->result = errno; 1273 } else { 1274 msg->result = 0; 1275 } 1276 } 1277 #endif 1278 1279 static inline void 1280 handle_traffic_msg(struct ff_msg *msg) 1281 { 1282 msg->traffic = ff_traffic; 1283 msg->result = 0; 1284 } 1285 1286 static inline void 1287 handle_default_msg(struct ff_msg *msg) 1288 { 1289 msg->result = ENOTSUP; 1290 } 1291 1292 static inline void 1293 handle_msg(struct ff_msg *msg, uint16_t proc_id) 1294 { 1295 switch (msg->msg_type) { 1296 case FF_SYSCTL: 1297 handle_sysctl_msg(msg); 1298 break; 1299 case FF_IOCTL: 1300 handle_ioctl_msg(msg); 1301 break; 1302 case FF_ROUTE: 1303 handle_route_msg(msg); 1304 break; 1305 case FF_TOP: 1306 handle_top_msg(msg); 1307 break; 1308 #ifdef FF_NETGRAPH 1309 case FF_NGCTL: 1310 handle_ngctl_msg(msg); 1311 break; 1312 #endif 1313 #ifdef FF_IPFW 1314 case FF_IPFW_CTL: 1315 handle_ipfw_msg(msg); 1316 break; 1317 #endif 1318 case FF_TRAFFIC: 1319 handle_traffic_msg(msg); 1320 break; 1321 default: 1322 handle_default_msg(msg); 1323 break; 1324 } 1325 rte_ring_enqueue(msg_ring[proc_id].ring[1], msg); 1326 } 1327 1328 static inline int 1329 process_msg_ring(uint16_t proc_id) 1330 { 1331 void *msg; 1332 int ret = rte_ring_dequeue(msg_ring[proc_id].ring[0], &msg); 1333 1334 if (unlikely(ret == 0)) { 1335 handle_msg((struct ff_msg *)msg, proc_id); 1336 } 1337 1338 return 0; 1339 } 1340 1341 /* Send burst of packets on an output interface */ 1342 static inline int 1343 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port) 1344 { 1345 struct rte_mbuf **m_table; 1346 int ret; 1347 uint16_t queueid; 1348 1349 queueid = qconf->tx_queue_id[port]; 1350 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table; 1351 1352 if (unlikely(qconf->pcap[port] != NULL)) { 1353 uint16_t i; 1354 for (i = 0; i < n; i++) { 1355 ff_dump_packets(qconf->pcap[port], m_table[i]); 1356 } 1357 } 1358 1359 ret = rte_eth_tx_burst(port, queueid, m_table, n); 1360 ff_traffic.tx_packets += ret; 1361 uint16_t i; 1362 for (i = 0; i < ret; i++) { 1363 ff_traffic.tx_bytes += rte_pktmbuf_pkt_len(m_table[i]); 1364 #ifdef _USE_PAGE_ARRAY_ 1365 if (qconf->tx_mbufs[port].bsd_m_table[i]) 1366 ff_txring_enqueue(&nic_tx_ring[port], qconf->tx_mbufs[port].bsd_m_table[i], m_table[i]->nb_segs); 1367 #endif 1368 } 1369 if (unlikely(ret < n)) { 1370 do { 1371 rte_pktmbuf_free(m_table[ret]); 1372 #ifdef _USE_PAGE_ARRAY_ 1373 if ( qconf->tx_mbufs[port].bsd_m_table[ret] ) 1374 ff_mbuf_free(qconf->tx_mbufs[port].bsd_m_table[ret]); 1375 #endif 1376 } while (++ret < n); 1377 } 1378 return 0; 1379 } 1380 1381 /* Enqueue a single packet, and send burst if queue is filled */ 1382 static inline int 1383 send_single_packet(struct rte_mbuf *m, uint8_t port) 1384 { 1385 uint16_t len; 1386 struct lcore_conf *qconf; 1387 1388 qconf = &lcore_conf; 1389 len = qconf->tx_mbufs[port].len; 1390 qconf->tx_mbufs[port].m_table[len] = m; 1391 len++; 1392 1393 /* enough pkts to be sent */ 1394 if (unlikely(len == MAX_PKT_BURST)) { 1395 send_burst(qconf, MAX_PKT_BURST, port); 1396 len = 0; 1397 } 1398 1399 qconf->tx_mbufs[port].len = len; 1400 return 0; 1401 } 1402 1403 int 1404 ff_dpdk_if_send(struct ff_dpdk_if_context *ctx, void *m, 1405 int total) 1406 { 1407 #ifdef _USE_PAGE_ARRAY_ 1408 return ff_dpdk_if_send_ex(ctx, m,total); 1409 #endif 1410 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 1411 struct rte_mbuf *head = rte_pktmbuf_alloc(mbuf_pool); 1412 if (head == NULL) { 1413 ff_mbuf_free(m); 1414 return -1; 1415 } 1416 1417 head->pkt_len = total; 1418 head->nb_segs = 0; 1419 1420 int off = 0; 1421 struct rte_mbuf *cur = head, *prev = NULL; 1422 while(total > 0) { 1423 if (cur == NULL) { 1424 cur = rte_pktmbuf_alloc(mbuf_pool); 1425 if (cur == NULL) { 1426 rte_pktmbuf_free(head); 1427 ff_mbuf_free(m); 1428 return -1; 1429 } 1430 } 1431 1432 if (prev != NULL) { 1433 prev->next = cur; 1434 } 1435 head->nb_segs++; 1436 1437 prev = cur; 1438 void *data = rte_pktmbuf_mtod(cur, void*); 1439 int len = total > RTE_MBUF_DEFAULT_DATAROOM ? RTE_MBUF_DEFAULT_DATAROOM : total; 1440 int ret = ff_mbuf_copydata(m, data, off, len); 1441 if (ret < 0) { 1442 rte_pktmbuf_free(head); 1443 ff_mbuf_free(m); 1444 return -1; 1445 } 1446 1447 1448 cur->data_len = len; 1449 off += len; 1450 total -= len; 1451 cur = NULL; 1452 } 1453 1454 struct ff_tx_offload offload = {0}; 1455 ff_mbuf_tx_offload(m, &offload); 1456 1457 void *data = rte_pktmbuf_mtod(head, void*); 1458 1459 if (offload.ip_csum) { 1460 /* ipv6 not supported yet */ 1461 struct ipv4_hdr *iph; 1462 int iph_len; 1463 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1464 iph_len = (iph->version_ihl & 0x0f) << 2; 1465 1466 head->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 1467 head->l2_len = ETHER_HDR_LEN; 1468 head->l3_len = iph_len; 1469 } 1470 1471 if (ctx->hw_features.tx_csum_l4) { 1472 struct ipv4_hdr *iph; 1473 int iph_len; 1474 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1475 iph_len = (iph->version_ihl & 0x0f) << 2; 1476 1477 if (offload.tcp_csum) { 1478 head->ol_flags |= PKT_TX_TCP_CKSUM; 1479 head->l2_len = ETHER_HDR_LEN; 1480 head->l3_len = iph_len; 1481 } 1482 1483 /* 1484 * TCP segmentation offload. 1485 * 1486 * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag 1487 * implies PKT_TX_TCP_CKSUM) 1488 * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 1489 * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and 1490 * write the IP checksum to 0 in the packet 1491 * - fill the mbuf offload information: l2_len, 1492 * l3_len, l4_len, tso_segsz 1493 * - calculate the pseudo header checksum without taking ip_len 1494 * in account, and set it in the TCP header. Refer to 1495 * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be 1496 * used as helpers. 1497 */ 1498 if (offload.tso_seg_size) { 1499 struct tcp_hdr *tcph; 1500 int tcph_len; 1501 tcph = (struct tcp_hdr *)((char *)iph + iph_len); 1502 tcph_len = (tcph->data_off & 0xf0) >> 2; 1503 tcph->cksum = rte_ipv4_phdr_cksum(iph, PKT_TX_TCP_SEG); 1504 1505 head->ol_flags |= PKT_TX_TCP_SEG; 1506 head->l4_len = tcph_len; 1507 head->tso_segsz = offload.tso_seg_size; 1508 } 1509 1510 if (offload.udp_csum) { 1511 head->ol_flags |= PKT_TX_UDP_CKSUM; 1512 head->l2_len = ETHER_HDR_LEN; 1513 head->l3_len = iph_len; 1514 } 1515 } 1516 1517 ff_mbuf_free(m); 1518 1519 return send_single_packet(head, ctx->port_id); 1520 } 1521 1522 static int 1523 main_loop(void *arg) 1524 { 1525 struct loop_routine *lr = (struct loop_routine *)arg; 1526 1527 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1528 uint64_t prev_tsc, diff_tsc, cur_tsc, usch_tsc, div_tsc, usr_tsc, sys_tsc, end_tsc, idle_sleep_tsc; 1529 int i, j, nb_rx, idle; 1530 uint16_t port_id, queue_id; 1531 struct lcore_conf *qconf; 1532 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / 1533 US_PER_S * BURST_TX_DRAIN_US; 1534 struct ff_dpdk_if_context *ctx; 1535 1536 prev_tsc = 0; 1537 usch_tsc = 0; 1538 1539 qconf = &lcore_conf; 1540 1541 while (1) { 1542 cur_tsc = rte_rdtsc(); 1543 if (unlikely(freebsd_clock.expire < cur_tsc)) { 1544 rte_timer_manage(); 1545 } 1546 1547 idle = 1; 1548 sys_tsc = 0; 1549 usr_tsc = 0; 1550 1551 /* 1552 * TX burst queue drain 1553 */ 1554 diff_tsc = cur_tsc - prev_tsc; 1555 if (unlikely(diff_tsc > drain_tsc)) { 1556 for (i = 0; i < qconf->nb_tx_port; i++) { 1557 port_id = qconf->tx_port_id[i]; 1558 if (qconf->tx_mbufs[port_id].len == 0) 1559 continue; 1560 1561 idle = 0; 1562 1563 send_burst(qconf, 1564 qconf->tx_mbufs[port_id].len, 1565 port_id); 1566 qconf->tx_mbufs[port_id].len = 0; 1567 } 1568 1569 prev_tsc = cur_tsc; 1570 } 1571 1572 /* 1573 * Read packet from RX queues 1574 */ 1575 for (i = 0; i < qconf->nb_rx_queue; ++i) { 1576 port_id = qconf->rx_queue_list[i].port_id; 1577 queue_id = qconf->rx_queue_list[i].queue_id; 1578 ctx = veth_ctx[port_id]; 1579 1580 #ifdef FF_KNI 1581 if (enable_kni && rte_eal_process_type() == RTE_PROC_PRIMARY) { 1582 ff_kni_process(port_id, queue_id, pkts_burst, MAX_PKT_BURST); 1583 } 1584 #endif 1585 1586 process_dispatch_ring(port_id, queue_id, pkts_burst, ctx); 1587 1588 nb_rx = rte_eth_rx_burst(port_id, queue_id, pkts_burst, 1589 MAX_PKT_BURST); 1590 if (nb_rx == 0) 1591 continue; 1592 1593 idle = 0; 1594 1595 /* Prefetch first packets */ 1596 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { 1597 rte_prefetch0(rte_pktmbuf_mtod( 1598 pkts_burst[j], void *)); 1599 } 1600 1601 /* Prefetch and handle already prefetched packets */ 1602 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { 1603 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ 1604 j + PREFETCH_OFFSET], void *)); 1605 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1606 } 1607 1608 /* Handle remaining prefetched packets */ 1609 for (; j < nb_rx; j++) { 1610 process_packets(port_id, queue_id, &pkts_burst[j], 1, ctx, 0); 1611 } 1612 } 1613 1614 process_msg_ring(qconf->proc_id); 1615 1616 div_tsc = rte_rdtsc(); 1617 1618 if (likely(lr->loop != NULL && (!idle || cur_tsc - usch_tsc > drain_tsc))) { 1619 usch_tsc = cur_tsc; 1620 lr->loop(lr->arg); 1621 } 1622 1623 idle_sleep_tsc = rte_rdtsc(); 1624 if (likely(idle && idle_sleep)) { 1625 usleep(idle_sleep); 1626 end_tsc = rte_rdtsc(); 1627 } else { 1628 end_tsc = idle_sleep_tsc; 1629 } 1630 1631 end_tsc = rte_rdtsc(); 1632 1633 if (usch_tsc == cur_tsc) { 1634 usr_tsc = idle_sleep_tsc - div_tsc; 1635 } 1636 1637 if (!idle) { 1638 sys_tsc = div_tsc - cur_tsc; 1639 ff_top_status.sys_tsc += sys_tsc; 1640 } 1641 1642 ff_top_status.usr_tsc += usr_tsc; 1643 ff_top_status.work_tsc += end_tsc - cur_tsc; 1644 ff_top_status.idle_tsc += end_tsc - cur_tsc - usr_tsc - sys_tsc; 1645 1646 ff_top_status.loops++; 1647 } 1648 1649 return 0; 1650 } 1651 1652 int 1653 ff_dpdk_if_up(void) { 1654 int i; 1655 struct lcore_conf *qconf = &lcore_conf; 1656 for (i = 0; i < qconf->nb_tx_port; i++) { 1657 uint16_t port_id = qconf->tx_port_id[i]; 1658 1659 struct ff_port_cfg *pconf = &qconf->port_cfgs[port_id]; 1660 veth_ctx[port_id] = ff_veth_attach(pconf); 1661 if (veth_ctx[port_id] == NULL) { 1662 rte_exit(EXIT_FAILURE, "ff_veth_attach failed"); 1663 } 1664 } 1665 1666 return 0; 1667 } 1668 1669 void 1670 ff_dpdk_run(loop_func_t loop, void *arg) { 1671 struct loop_routine *lr = rte_malloc(NULL, 1672 sizeof(struct loop_routine), 0); 1673 lr->loop = loop; 1674 lr->arg = arg; 1675 rte_eal_mp_remote_launch(main_loop, lr, CALL_MASTER); 1676 rte_eal_mp_wait_lcore(); 1677 rte_free(lr); 1678 } 1679 1680 void 1681 ff_dpdk_pktmbuf_free(void *m) 1682 { 1683 rte_pktmbuf_free((struct rte_mbuf *)m); 1684 } 1685 1686 static uint32_t 1687 toeplitz_hash(unsigned keylen, const uint8_t *key, 1688 unsigned datalen, const uint8_t *data) 1689 { 1690 uint32_t hash = 0, v; 1691 u_int i, b; 1692 1693 /* XXXRW: Perhaps an assertion about key length vs. data length? */ 1694 1695 v = (key[0]<<24) + (key[1]<<16) + (key[2] <<8) + key[3]; 1696 for (i = 0; i < datalen; i++) { 1697 for (b = 0; b < 8; b++) { 1698 if (data[i] & (1<<(7-b))) 1699 hash ^= v; 1700 v <<= 1; 1701 if ((i + 4) < keylen && 1702 (key[i+4] & (1<<(7-b)))) 1703 v |= 1; 1704 } 1705 } 1706 return (hash); 1707 } 1708 1709 int 1710 ff_rss_check(void *softc, uint32_t saddr, uint32_t daddr, 1711 uint16_t sport, uint16_t dport) 1712 { 1713 struct lcore_conf *qconf = &lcore_conf; 1714 struct ff_dpdk_if_context *ctx = ff_veth_softc_to_hostc(softc); 1715 uint16_t nb_queues = qconf->nb_queue_list[ctx->port_id]; 1716 1717 if (nb_queues <= 1) { 1718 return 1; 1719 } 1720 1721 uint16_t reta_size = rss_reta_size[ctx->port_id]; 1722 uint16_t queueid = qconf->tx_queue_id[ctx->port_id]; 1723 1724 uint8_t data[sizeof(saddr) + sizeof(daddr) + sizeof(sport) + 1725 sizeof(dport)]; 1726 1727 unsigned datalen = 0; 1728 1729 bcopy(&saddr, &data[datalen], sizeof(saddr)); 1730 datalen += sizeof(saddr); 1731 1732 bcopy(&daddr, &data[datalen], sizeof(daddr)); 1733 datalen += sizeof(daddr); 1734 1735 bcopy(&sport, &data[datalen], sizeof(sport)); 1736 datalen += sizeof(sport); 1737 1738 bcopy(&dport, &data[datalen], sizeof(dport)); 1739 datalen += sizeof(dport); 1740 1741 uint32_t hash = toeplitz_hash(sizeof(default_rsskey_40bytes), 1742 default_rsskey_40bytes, datalen, data); 1743 1744 return ((hash & (reta_size - 1)) % nb_queues) == queueid; 1745 } 1746 1747 void 1748 ff_regist_packet_dispatcher(dispatch_func_t func) 1749 { 1750 packet_dispatcher = func; 1751 } 1752 1753 uint64_t 1754 ff_get_tsc_ns() 1755 { 1756 uint64_t cur_tsc = rte_rdtsc(); 1757 uint64_t hz = rte_get_tsc_hz(); 1758 return ((double)cur_tsc/(double)hz) * NS_PER_S; 1759 } 1760 1761 #ifdef _USE_PAGE_ARRAY_ 1762 static int StkList_init(StackList_t*p, int size) 1763 { 1764 int i = 0; 1765 1766 if (p==NULL || size<=0) 1767 { 1768 return -1; 1769 } 1770 p->size = size; 1771 p->top = 0; 1772 if ( posix_memalign((void**)&p->ele, sizeof(uint64_t), sizeof(uint64_t)*size) != 0) 1773 return -2; 1774 1775 return 0; 1776 } 1777 1778 static inline void* StkList_pop(StackList_t *p) 1779 { 1780 int head = 0; 1781 1782 if(p==NULL) 1783 return NULL; 1784 1785 if (p->top > 0 ) 1786 { 1787 return (void*)p->ele[--p->top]; 1788 } 1789 else 1790 return NULL; 1791 } 1792 1793 //id: the id of element to be freed. 1794 //return code: -1: faile; >=0:OK. 1795 static inline int StkList_push(StackList_t *p, const uint64_t val) 1796 { 1797 int tail = 0; 1798 1799 if(p==NULL) 1800 return -1; 1801 if (p->top < p->size) 1802 { 1803 p->ele[p->top++] = val; 1804 return 0; 1805 } 1806 else 1807 return -1; 1808 } 1809 1810 static int StkList_Size(StackList_t * p) 1811 { 1812 return p->size; 1813 } 1814 1815 // set (void*) to rte_mbuf's priv_data. 1816 static inline int ff_mbuf_set_uint64(struct rte_mbuf* p, uint64_t data) 1817 { 1818 if (rte_pktmbuf_priv_size(p->pool) >= sizeof(uint64_t)) 1819 *((uint64_t*)(p+1)) = data; 1820 return 0; 1821 } 1822 1823 /************************* 1824 * if mbuf has num segment in all, Dev's sw_ring will use num descriptions. ff_txring also use num segments as below: 1825 * <--- num-1 ---->|ptr| head | 1826 * ---------------------------------------------- 1827 * | 0 | 0 | ..............| 0 | p | XXX | 1828 *----------------------------------------------- 1829 *************************/ 1830 static inline int ff_txring_enqueue(struct mbuf_txring* q, void *p, int seg_num) 1831 { 1832 int i = 0; 1833 for( i=0; i<seg_num-1; i++) 1834 { 1835 if ( q->m_table[q->head] ) 1836 { 1837 ff_mbuf_free(q->m_table[q->head]); 1838 q->m_table[q->head] = NULL; 1839 } 1840 Head_INC(q->head); 1841 } 1842 if ( q->m_table[q->head] ) 1843 ff_mbuf_free(q->m_table[q->head]); 1844 q->m_table[q->head] = p; 1845 Head_INC(q->head); 1846 1847 return 0; 1848 } 1849 1850 // pop out from head-1 . 1851 static inline int ff_txring_pop(struct mbuf_txring* q, int num) 1852 { 1853 int i = 0; 1854 1855 for (i=0; i<num; i++) 1856 { 1857 Head_DEC(q->head); 1858 if ( (i==0 && q->m_table[q->head]==NULL) || (i>0 && q->m_table[q->head]!=NULL) ) 1859 { 1860 rte_panic("ff_txring_pop fatal error!"); 1861 } 1862 if ( q->m_table[q->head] != NULL ) 1863 { 1864 ff_mbuf_free(q->m_table[q->head]); 1865 q->m_table[q->head] = NULL; 1866 } 1867 } 1868 } 1869 1870 static inline void ff_txring_init(struct mbuf_txring* q, uint32_t num) 1871 { 1872 memset(q, 0, sizeof(struct mbuf_txring)*num); 1873 } 1874 1875 static int ff_mmap_init() 1876 { 1877 int err = 0; 1878 int i = 0; 1879 uint64_t virt_addr = NULL; 1880 phys_addr_t phys_addr = 0; 1881 uint64_t bsd_memsz = (ff_global_cfg.freebsd.mem_size << 20); 1882 unsigned int bsd_pagesz = 0; 1883 1884 ff_page_start = (uint64_t)mmap( NULL, bsd_memsz, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_POPULATE, -1, 0); 1885 if (ff_page_start == (uint64_t)-1) 1886 { 1887 rte_panic("ff_mmap_init get ff_page_start failed, err=%d.\n", errno); 1888 return -1; 1889 } 1890 1891 if ( mlock((void*)ff_page_start, bsd_memsz)<0 ) 1892 { 1893 rte_panic("mlock failed, err=%d.\n", errno); 1894 return -1; 1895 } 1896 ff_page_end = ff_page_start + bsd_memsz; 1897 1898 rte_log(RTE_LOG_INFO, RTE_LOGTYPE_USER1, "ff_mmap_init mmap %d pages, %d MB.\n", bsd_pagesz, ff_global_cfg.freebsd.mem_size); 1899 printf("ff_mmap_init mem[0x%lx:0x%lx]\n", ff_page_start, ff_page_end); 1900 1901 bsd_pagesz = (bsd_memsz>>12); 1902 if(posix_memalign((void**)&ff_mpage_phy, sizeof(phys_addr_t), bsd_pagesz*sizeof(phys_addr_t))!=0) 1903 { 1904 rte_panic("posix_memalign get ff_mpage_phy failed, err=%d.\n", errno); 1905 return -1; 1906 } 1907 1908 StkList_init(&ff_mpage_ctl, bsd_pagesz); 1909 1910 for (i=0; i<bsd_pagesz; i++ ) 1911 { 1912 virt_addr = ff_page_start + PAGE_SIZE*i; 1913 memset((void*)virt_addr, 0, PAGE_SIZE); 1914 1915 StkList_push( &ff_mpage_ctl, virt_addr); 1916 ff_mpage_phy[i] = rte_mem_virt2phy((const void*)virt_addr); 1917 if ( ff_mpage_phy[i] == RTE_BAD_IOVA ) 1918 { 1919 rte_panic("rte_mem_virt2phy return invalid address."); 1920 return -1; 1921 } 1922 } 1923 1924 ff_txring_init(&nic_tx_ring[0], RTE_MAX_ETHPORTS); 1925 1926 return 0; 1927 } 1928 1929 // 1: vma in fstack page table; 0: vma not in fstack pages, in DPDK pool. 1930 static inline int ff_chk_vma(const uint64_t virtaddr) 1931 { 1932 return !!( virtaddr > ff_page_start && virtaddr < ff_page_end ); 1933 } 1934 1935 /* 1936 * Get physical address of any mapped virtual address in the current process. 1937 */ 1938 static inline uint64_t ff_mem_virt2phy(const void* virtaddr) 1939 { 1940 uint64_t addr = 0; 1941 uint32_t pages = 0; 1942 1943 pages = (((uint64_t)virtaddr - (uint64_t)ff_page_start)>>PAGE_SHIFT); 1944 if (pages >= StkList_Size(&ff_mpage_ctl)) 1945 { 1946 rte_panic("ff_mbuf_virt2phy get invalid pages %d.", pages); 1947 return -1; 1948 } 1949 1950 addr = ff_mpage_phy[pages] + ((const uint64_t)virtaddr & PAGE_MASK); 1951 return addr; 1952 } 1953 1954 void* ff_mem_get_page() 1955 { 1956 return (void*)StkList_pop(&ff_mpage_ctl); 1957 } 1958 1959 int ff_mem_free_addr(void* p) 1960 { 1961 StkList_push(&ff_mpage_ctl, (const uint64_t)p); 1962 return 0; 1963 } 1964 1965 static inline void ff_offload_set(struct ff_dpdk_if_context *ctx, void* m, struct rte_mbuf *head) 1966 { 1967 void* data = NULL; 1968 struct ff_tx_offload offload = {0}; 1969 1970 ff_mbuf_tx_offload(m, &offload); 1971 data = rte_pktmbuf_mtod(head, void*); 1972 1973 if (offload.ip_csum) { 1974 /* ipv6 not supported yet */ 1975 struct ipv4_hdr *iph; 1976 int iph_len; 1977 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1978 iph_len = (iph->version_ihl & 0x0f) << 2; 1979 1980 head->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4; 1981 head->l2_len = ETHER_HDR_LEN; 1982 head->l3_len = iph_len; 1983 } 1984 1985 if (ctx->hw_features.tx_csum_l4) { 1986 struct ipv4_hdr *iph; 1987 int iph_len; 1988 iph = (struct ipv4_hdr *)(data + ETHER_HDR_LEN); 1989 iph_len = (iph->version_ihl & 0x0f) << 2; 1990 1991 if (offload.tcp_csum) { 1992 head->ol_flags |= PKT_TX_TCP_CKSUM; 1993 head->l2_len = ETHER_HDR_LEN; 1994 head->l3_len = iph_len; 1995 } 1996 1997 /* 1998 * TCP segmentation offload. 1999 * 2000 * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag 2001 * implies PKT_TX_TCP_CKSUM) 2002 * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 2003 * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and 2004 * write the IP checksum to 0 in the packet 2005 * - fill the mbuf offload information: l2_len, 2006 * l3_len, l4_len, tso_segsz 2007 * - calculate the pseudo header checksum without taking ip_len 2008 * in account, and set it in the TCP header. Refer to 2009 * rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum() that can be 2010 * used as helpers. 2011 */ 2012 if (offload.tso_seg_size) { 2013 struct tcp_hdr *tcph; 2014 int tcph_len; 2015 tcph = (struct tcp_hdr *)((char *)iph + iph_len); 2016 tcph_len = (tcph->data_off & 0xf0) >> 2; 2017 tcph->cksum = rte_ipv4_phdr_cksum(iph, PKT_TX_TCP_SEG); 2018 2019 head->ol_flags |= PKT_TX_TCP_SEG; 2020 head->l4_len = tcph_len; 2021 head->tso_segsz = offload.tso_seg_size; 2022 } 2023 2024 if (offload.udp_csum) { 2025 head->ol_flags |= PKT_TX_UDP_CKSUM; 2026 head->l2_len = ETHER_HDR_LEN; 2027 head->l3_len = iph_len; 2028 } 2029 } 2030 } 2031 2032 // create rte_buf refer to data which is transmit from bsd stack by EXT_CLUSTER. 2033 static inline struct rte_mbuf* ff_extcl_to_rte(void* m) 2034 { 2035 struct rte_mempool *mbuf_pool = pktmbuf_pool[lcore_conf.socket_id]; 2036 struct rte_mbuf* src_mbuf = NULL; 2037 struct rte_mbuf *p_head = NULL; 2038 2039 src_mbuf = (struct rte_mbuf*)ff_rte_frm_extcl(m); 2040 if ( NULL==src_mbuf ) 2041 { 2042 return NULL; 2043 } 2044 p_head = rte_pktmbuf_clone(src_mbuf, mbuf_pool); 2045 if (p_head == NULL) 2046 { 2047 return NULL; 2048 } 2049 2050 return p_head; 2051 } 2052 2053 // create rte_mbuf refer to data in bsd mbuf. 2054 static inline struct rte_mbuf* ff_bsd_to_rte(void* m, int total) 2055 { 2056 struct rte_mempool *mbuf_pool = ff_ref_pool[lcore_conf.socket_id]; 2057 struct rte_mbuf *p_head = NULL; 2058 struct rte_mbuf *cur = NULL, *prev = NULL, *tmp=NULL; 2059 void* data = NULL; 2060 void* p_bsdbuf = NULL; 2061 unsigned len = 0; 2062 2063 p_head = rte_pktmbuf_alloc(mbuf_pool); 2064 if (p_head == NULL) 2065 { 2066 return NULL; 2067 } 2068 p_head->pkt_len = total; 2069 p_head->nb_segs = 0; 2070 cur = p_head; 2071 p_bsdbuf = m; 2072 while( p_bsdbuf ){ 2073 if (cur == NULL) { 2074 cur = rte_pktmbuf_alloc(mbuf_pool); 2075 if (cur == NULL) { 2076 rte_pktmbuf_free(p_head); 2077 return NULL; 2078 } 2079 } 2080 ff_next_mbuf(&p_bsdbuf, &data, &len); // p_bsdbuf move to next mbuf. 2081 cur->buf_addr = data; 2082 cur->buf_physaddr = ff_mem_virt2phy((const void*)(cur->buf_addr)); 2083 cur->data_off = 0; 2084 cur->data_len = len; 2085 2086 p_head->nb_segs++; 2087 if (prev != NULL) { 2088 prev->next = cur; 2089 } 2090 prev = cur; 2091 cur = NULL; 2092 } 2093 2094 return p_head; 2095 } 2096 2097 int ff_dpdk_if_send_ex(struct ff_dpdk_if_context *ctx, void *m, int total) 2098 { 2099 struct rte_mbuf *head = NULL; 2100 void *src_buf = NULL; 2101 void* p_data = NULL; 2102 struct lcore_conf *qconf = NULL; 2103 unsigned len = 0; 2104 2105 if ( !m ) 2106 { 2107 rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_dpdk_if_send_ex input invalid NULL address."); 2108 return -1; 2109 } 2110 p_data = ff_mbuf_mtod(m); 2111 if ( ff_chk_vma((uint64_t)p_data)) 2112 { 2113 head = ff_bsd_to_rte(m, total); 2114 } 2115 else if( ff_extcl_to_rte(m) ==NULL ) 2116 { 2117 rte_panic("data address 0x%lx is out of page bound or malloc by DPDK recver.", (uint64_t)p_data); 2118 return -1; 2119 } 2120 if (head == NULL) 2121 { 2122 rte_log(RTE_LOG_CRIT, RTE_LOGTYPE_USER1, "ff_dpdk_if_send_ex call ff_bsd_to_rte failed."); 2123 ff_mbuf_free(m); 2124 return -1; 2125 } 2126 2127 ff_offload_set(ctx, m, head); 2128 qconf = &lcore_conf; 2129 len = qconf->tx_mbufs[ctx->port_id].len; 2130 qconf->tx_mbufs[ctx->port_id].m_table[len] = head; 2131 qconf->tx_mbufs[ctx->port_id].bsd_m_table[len] = m; 2132 len++; 2133 2134 /* enough pkts to be sent */ 2135 if (unlikely(len == MAX_PKT_BURST)) { 2136 send_burst(qconf, MAX_PKT_BURST, ctx->port_id); 2137 len = 0; 2138 } 2139 qconf->tx_mbufs[ctx->port_id].len = len; 2140 2141 return 0; 2142 } 2143 2144 #endif 2145 2146 2147 2148