1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_malloc_heap.h> 31 #include <rte_memory.h> 32 #include <rte_memcpy.h> 33 #include <rte_launch.h> 34 #include <rte_eal.h> 35 #include <rte_alarm.h> 36 #include <rte_per_lcore.h> 37 #include <rte_lcore.h> 38 #include <rte_atomic.h> 39 #include <rte_branch_prediction.h> 40 #include <rte_mempool.h> 41 #include <rte_malloc.h> 42 #include <rte_mbuf.h> 43 #include <rte_mbuf_pool_ops.h> 44 #include <rte_interrupts.h> 45 #include <rte_pci.h> 46 #include <rte_ether.h> 47 #include <rte_ethdev.h> 48 #include <rte_dev.h> 49 #include <rte_string_fns.h> 50 #ifdef RTE_LIBRTE_IXGBE_PMD 51 #include <rte_pmd_ixgbe.h> 52 #endif 53 #ifdef RTE_LIBRTE_PDUMP 54 #include <rte_pdump.h> 55 #endif 56 #include <rte_flow.h> 57 #include <rte_metrics.h> 58 #ifdef RTE_LIBRTE_BITRATE 59 #include <rte_bitrate.h> 60 #endif 61 #ifdef RTE_LIBRTE_LATENCY_STATS 62 #include <rte_latencystats.h> 63 #endif 64 65 #include "testpmd.h" 66 67 #ifndef MAP_HUGETLB 68 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 69 #define HUGE_FLAG (0x40000) 70 #else 71 #define HUGE_FLAG MAP_HUGETLB 72 #endif 73 74 #ifndef MAP_HUGE_SHIFT 75 /* older kernels (or FreeBSD) will not have this define */ 76 #define HUGE_SHIFT (26) 77 #else 78 #define HUGE_SHIFT MAP_HUGE_SHIFT 79 #endif 80 81 #define EXTMEM_HEAP_NAME "extmem" 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 193 struct fwd_config cur_fwd_config; 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 195 uint32_t retry_enabled; 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 198 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 200 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 201 * specified on command-line. */ 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 203 204 /* 205 * In container, it cannot terminate the process which running with 'stats-period' 206 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 207 */ 208 uint8_t f_quit; 209 210 /* 211 * Configuration of packet segments used by the "txonly" processing engine. 212 */ 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 215 TXONLY_DEF_PACKET_LEN, 216 }; 217 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 218 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 220 /**< Split policy for packets to TX. */ 221 222 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 223 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 224 225 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 226 uint8_t dcb_config = 0; 227 228 /* Whether the dcb is in testing status */ 229 uint8_t dcb_test = 0; 230 231 /* 232 * Configurable number of RX/TX queues. 233 */ 234 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 235 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 236 237 /* 238 * Configurable number of RX/TX ring descriptors. 239 * Defaults are supplied by drivers via ethdev. 240 */ 241 #define RTE_TEST_RX_DESC_DEFAULT 0 242 #define RTE_TEST_TX_DESC_DEFAULT 0 243 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 244 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 245 246 #define RTE_PMD_PARAM_UNSET -1 247 /* 248 * Configurable values of RX and TX ring threshold registers. 249 */ 250 251 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 252 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 253 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 254 255 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 /* 260 * Configurable value of RX free threshold. 261 */ 262 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 263 264 /* 265 * Configurable value of RX drop enable. 266 */ 267 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 268 269 /* 270 * Configurable value of TX free threshold. 271 */ 272 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 273 274 /* 275 * Configurable value of TX RS bit threshold. 276 */ 277 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 278 279 /* 280 * Configurable value of buffered packets before sending. 281 */ 282 uint16_t noisy_tx_sw_bufsz; 283 284 /* 285 * Configurable value of packet buffer timeout. 286 */ 287 uint16_t noisy_tx_sw_buf_flush_time; 288 289 /* 290 * Configurable value for size of VNF internal memory area 291 * used for simulating noisy neighbour behaviour 292 */ 293 uint64_t noisy_lkup_mem_sz; 294 295 /* 296 * Configurable value of number of random writes done in 297 * VNF simulation memory area. 298 */ 299 uint64_t noisy_lkup_num_writes; 300 301 /* 302 * Configurable value of number of random reads done in 303 * VNF simulation memory area. 304 */ 305 uint64_t noisy_lkup_num_reads; 306 307 /* 308 * Configurable value of number of random reads/writes done in 309 * VNF simulation memory area. 310 */ 311 uint64_t noisy_lkup_num_reads_writes; 312 313 /* 314 * Receive Side Scaling (RSS) configuration. 315 */ 316 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 317 318 /* 319 * Port topology configuration 320 */ 321 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 322 323 /* 324 * Avoids to flush all the RX streams before starts forwarding. 325 */ 326 uint8_t no_flush_rx = 0; /* flush by default */ 327 328 /* 329 * Flow API isolated mode. 330 */ 331 uint8_t flow_isolate_all; 332 333 /* 334 * Avoids to check link status when starting/stopping a port. 335 */ 336 uint8_t no_link_check = 0; /* check by default */ 337 338 /* 339 * Enable link status change notification 340 */ 341 uint8_t lsc_interrupt = 1; /* enabled by default */ 342 343 /* 344 * Enable device removal notification. 345 */ 346 uint8_t rmv_interrupt = 1; /* enabled by default */ 347 348 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 349 350 /* After attach, port setup is called on event or by iterator */ 351 bool setup_on_probe_event = true; 352 353 /* Pretty printing of ethdev events */ 354 static const char * const eth_event_desc[] = { 355 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 356 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 357 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 358 [RTE_ETH_EVENT_INTR_RESET] = "reset", 359 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 360 [RTE_ETH_EVENT_IPSEC] = "IPsec", 361 [RTE_ETH_EVENT_MACSEC] = "MACsec", 362 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 363 [RTE_ETH_EVENT_NEW] = "device probed", 364 [RTE_ETH_EVENT_DESTROY] = "device released", 365 [RTE_ETH_EVENT_MAX] = NULL, 366 }; 367 368 /* 369 * Display or mask ether events 370 * Default to all events except VF_MBOX 371 */ 372 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 373 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 374 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 375 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 376 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 377 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 378 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 379 /* 380 * Decide if all memory are locked for performance. 381 */ 382 int do_mlockall = 0; 383 384 /* 385 * NIC bypass mode configuration options. 386 */ 387 388 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 389 /* The NIC bypass watchdog timeout. */ 390 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 391 #endif 392 393 394 #ifdef RTE_LIBRTE_LATENCY_STATS 395 396 /* 397 * Set when latency stats is enabled in the commandline 398 */ 399 uint8_t latencystats_enabled; 400 401 /* 402 * Lcore ID to serive latency statistics. 403 */ 404 lcoreid_t latencystats_lcore_id = -1; 405 406 #endif 407 408 /* 409 * Ethernet device configuration. 410 */ 411 struct rte_eth_rxmode rx_mode = { 412 .max_rx_pkt_len = ETHER_MAX_LEN, /**< Default maximum frame length. */ 413 }; 414 415 struct rte_eth_txmode tx_mode = { 416 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 417 }; 418 419 struct rte_fdir_conf fdir_conf = { 420 .mode = RTE_FDIR_MODE_NONE, 421 .pballoc = RTE_FDIR_PBALLOC_64K, 422 .status = RTE_FDIR_REPORT_STATUS, 423 .mask = { 424 .vlan_tci_mask = 0xFFEF, 425 .ipv4_mask = { 426 .src_ip = 0xFFFFFFFF, 427 .dst_ip = 0xFFFFFFFF, 428 }, 429 .ipv6_mask = { 430 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 431 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 432 }, 433 .src_port_mask = 0xFFFF, 434 .dst_port_mask = 0xFFFF, 435 .mac_addr_byte_mask = 0xFF, 436 .tunnel_type_mask = 1, 437 .tunnel_id_mask = 0xFFFFFFFF, 438 }, 439 .drop_queue = 127, 440 }; 441 442 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 443 444 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 445 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 446 447 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 448 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 449 450 uint16_t nb_tx_queue_stats_mappings = 0; 451 uint16_t nb_rx_queue_stats_mappings = 0; 452 453 /* 454 * Display zero values by default for xstats 455 */ 456 uint8_t xstats_hide_zero; 457 458 unsigned int num_sockets = 0; 459 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 460 461 #ifdef RTE_LIBRTE_BITRATE 462 /* Bitrate statistics */ 463 struct rte_stats_bitrates *bitrate_data; 464 lcoreid_t bitrate_lcore_id; 465 uint8_t bitrate_enabled; 466 #endif 467 468 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 469 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 470 471 struct vxlan_encap_conf vxlan_encap_conf = { 472 .select_ipv4 = 1, 473 .select_vlan = 0, 474 .vni = "\x00\x00\x00", 475 .udp_src = 0, 476 .udp_dst = RTE_BE16(4789), 477 .ipv4_src = IPv4(127, 0, 0, 1), 478 .ipv4_dst = IPv4(255, 255, 255, 255), 479 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 480 "\x00\x00\x00\x00\x00\x00\x00\x01", 481 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 482 "\x00\x00\x00\x00\x00\x00\x11\x11", 483 .vlan_tci = 0, 484 .eth_src = "\x00\x00\x00\x00\x00\x00", 485 .eth_dst = "\xff\xff\xff\xff\xff\xff", 486 }; 487 488 struct nvgre_encap_conf nvgre_encap_conf = { 489 .select_ipv4 = 1, 490 .select_vlan = 0, 491 .tni = "\x00\x00\x00", 492 .ipv4_src = IPv4(127, 0, 0, 1), 493 .ipv4_dst = IPv4(255, 255, 255, 255), 494 .ipv6_src = "\x00\x00\x00\x00\x00\x00\x00\x00" 495 "\x00\x00\x00\x00\x00\x00\x00\x01", 496 .ipv6_dst = "\x00\x00\x00\x00\x00\x00\x00\x00" 497 "\x00\x00\x00\x00\x00\x00\x11\x11", 498 .vlan_tci = 0, 499 .eth_src = "\x00\x00\x00\x00\x00\x00", 500 .eth_dst = "\xff\xff\xff\xff\xff\xff", 501 }; 502 503 /* Forward function declarations */ 504 static void setup_attached_port(portid_t pi); 505 static void map_port_queue_stats_mapping_registers(portid_t pi, 506 struct rte_port *port); 507 static void check_all_ports_link_status(uint32_t port_mask); 508 static int eth_event_callback(portid_t port_id, 509 enum rte_eth_event_type type, 510 void *param, void *ret_param); 511 static void dev_event_callback(const char *device_name, 512 enum rte_dev_event_type type, 513 void *param); 514 515 /* 516 * Check if all the ports are started. 517 * If yes, return positive value. If not, return zero. 518 */ 519 static int all_ports_started(void); 520 521 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 522 uint16_t gso_max_segment_size = ETHER_MAX_LEN - ETHER_CRC_LEN; 523 524 /* 525 * Helper function to check if socket is already discovered. 526 * If yes, return positive value. If not, return zero. 527 */ 528 int 529 new_socket_id(unsigned int socket_id) 530 { 531 unsigned int i; 532 533 for (i = 0; i < num_sockets; i++) { 534 if (socket_ids[i] == socket_id) 535 return 0; 536 } 537 return 1; 538 } 539 540 /* 541 * Setup default configuration. 542 */ 543 static void 544 set_default_fwd_lcores_config(void) 545 { 546 unsigned int i; 547 unsigned int nb_lc; 548 unsigned int sock_num; 549 550 nb_lc = 0; 551 for (i = 0; i < RTE_MAX_LCORE; i++) { 552 if (!rte_lcore_is_enabled(i)) 553 continue; 554 sock_num = rte_lcore_to_socket_id(i); 555 if (new_socket_id(sock_num)) { 556 if (num_sockets >= RTE_MAX_NUMA_NODES) { 557 rte_exit(EXIT_FAILURE, 558 "Total sockets greater than %u\n", 559 RTE_MAX_NUMA_NODES); 560 } 561 socket_ids[num_sockets++] = sock_num; 562 } 563 if (i == rte_get_master_lcore()) 564 continue; 565 fwd_lcores_cpuids[nb_lc++] = i; 566 } 567 nb_lcores = (lcoreid_t) nb_lc; 568 nb_cfg_lcores = nb_lcores; 569 nb_fwd_lcores = 1; 570 } 571 572 static void 573 set_def_peer_eth_addrs(void) 574 { 575 portid_t i; 576 577 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 578 peer_eth_addrs[i].addr_bytes[0] = ETHER_LOCAL_ADMIN_ADDR; 579 peer_eth_addrs[i].addr_bytes[5] = i; 580 } 581 } 582 583 static void 584 set_default_fwd_ports_config(void) 585 { 586 portid_t pt_id; 587 int i = 0; 588 589 RTE_ETH_FOREACH_DEV(pt_id) { 590 fwd_ports_ids[i++] = pt_id; 591 592 /* Update sockets info according to the attached device */ 593 int socket_id = rte_eth_dev_socket_id(pt_id); 594 if (socket_id >= 0 && new_socket_id(socket_id)) { 595 if (num_sockets >= RTE_MAX_NUMA_NODES) { 596 rte_exit(EXIT_FAILURE, 597 "Total sockets greater than %u\n", 598 RTE_MAX_NUMA_NODES); 599 } 600 socket_ids[num_sockets++] = socket_id; 601 } 602 } 603 604 nb_cfg_ports = nb_ports; 605 nb_fwd_ports = nb_ports; 606 } 607 608 void 609 set_def_fwd_config(void) 610 { 611 set_default_fwd_lcores_config(); 612 set_def_peer_eth_addrs(); 613 set_default_fwd_ports_config(); 614 } 615 616 /* extremely pessimistic estimation of memory required to create a mempool */ 617 static int 618 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 619 { 620 unsigned int n_pages, mbuf_per_pg, leftover; 621 uint64_t total_mem, mbuf_mem, obj_sz; 622 623 /* there is no good way to predict how much space the mempool will 624 * occupy because it will allocate chunks on the fly, and some of those 625 * will come from default DPDK memory while some will come from our 626 * external memory, so just assume 128MB will be enough for everyone. 627 */ 628 uint64_t hdr_mem = 128 << 20; 629 630 /* account for possible non-contiguousness */ 631 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 632 if (obj_sz > pgsz) { 633 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 634 return -1; 635 } 636 637 mbuf_per_pg = pgsz / obj_sz; 638 leftover = (nb_mbufs % mbuf_per_pg) > 0; 639 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 640 641 mbuf_mem = n_pages * pgsz; 642 643 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 644 645 if (total_mem > SIZE_MAX) { 646 TESTPMD_LOG(ERR, "Memory size too big\n"); 647 return -1; 648 } 649 *out = (size_t)total_mem; 650 651 return 0; 652 } 653 654 static inline uint32_t 655 bsf64(uint64_t v) 656 { 657 return (uint32_t)__builtin_ctzll(v); 658 } 659 660 static inline uint32_t 661 log2_u64(uint64_t v) 662 { 663 if (v == 0) 664 return 0; 665 v = rte_align64pow2(v); 666 return bsf64(v); 667 } 668 669 static int 670 pagesz_flags(uint64_t page_sz) 671 { 672 /* as per mmap() manpage, all page sizes are log2 of page size 673 * shifted by MAP_HUGE_SHIFT 674 */ 675 int log2 = log2_u64(page_sz); 676 677 return (log2 << HUGE_SHIFT); 678 } 679 680 static void * 681 alloc_mem(size_t memsz, size_t pgsz, bool huge) 682 { 683 void *addr; 684 int flags; 685 686 /* allocate anonymous hugepages */ 687 flags = MAP_ANONYMOUS | MAP_PRIVATE; 688 if (huge) 689 flags |= HUGE_FLAG | pagesz_flags(pgsz); 690 691 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 692 if (addr == MAP_FAILED) 693 return NULL; 694 695 return addr; 696 } 697 698 struct extmem_param { 699 void *addr; 700 size_t len; 701 size_t pgsz; 702 rte_iova_t *iova_table; 703 unsigned int iova_table_len; 704 }; 705 706 static int 707 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 708 bool huge) 709 { 710 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 711 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 712 unsigned int cur_page, n_pages, pgsz_idx; 713 size_t mem_sz, cur_pgsz; 714 rte_iova_t *iovas = NULL; 715 void *addr; 716 int ret; 717 718 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 719 /* skip anything that is too big */ 720 if (pgsizes[pgsz_idx] > SIZE_MAX) 721 continue; 722 723 cur_pgsz = pgsizes[pgsz_idx]; 724 725 /* if we were told not to allocate hugepages, override */ 726 if (!huge) 727 cur_pgsz = sysconf(_SC_PAGESIZE); 728 729 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 730 if (ret < 0) { 731 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 732 return -1; 733 } 734 735 /* allocate our memory */ 736 addr = alloc_mem(mem_sz, cur_pgsz, huge); 737 738 /* if we couldn't allocate memory with a specified page size, 739 * that doesn't mean we can't do it with other page sizes, so 740 * try another one. 741 */ 742 if (addr == NULL) 743 continue; 744 745 /* store IOVA addresses for every page in this memory area */ 746 n_pages = mem_sz / cur_pgsz; 747 748 iovas = malloc(sizeof(*iovas) * n_pages); 749 750 if (iovas == NULL) { 751 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 752 goto fail; 753 } 754 /* lock memory if it's not huge pages */ 755 if (!huge) 756 mlock(addr, mem_sz); 757 758 /* populate IOVA addresses */ 759 for (cur_page = 0; cur_page < n_pages; cur_page++) { 760 rte_iova_t iova; 761 size_t offset; 762 void *cur; 763 764 offset = cur_pgsz * cur_page; 765 cur = RTE_PTR_ADD(addr, offset); 766 767 /* touch the page before getting its IOVA */ 768 *(volatile char *)cur = 0; 769 770 iova = rte_mem_virt2iova(cur); 771 772 iovas[cur_page] = iova; 773 } 774 775 break; 776 } 777 /* if we couldn't allocate anything */ 778 if (iovas == NULL) 779 return -1; 780 781 param->addr = addr; 782 param->len = mem_sz; 783 param->pgsz = cur_pgsz; 784 param->iova_table = iovas; 785 param->iova_table_len = n_pages; 786 787 return 0; 788 fail: 789 if (iovas) 790 free(iovas); 791 if (addr) 792 munmap(addr, mem_sz); 793 794 return -1; 795 } 796 797 static int 798 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 799 { 800 struct extmem_param param; 801 int socket_id, ret; 802 803 memset(¶m, 0, sizeof(param)); 804 805 /* check if our heap exists */ 806 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 807 if (socket_id < 0) { 808 /* create our heap */ 809 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 810 if (ret < 0) { 811 TESTPMD_LOG(ERR, "Cannot create heap\n"); 812 return -1; 813 } 814 } 815 816 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 817 if (ret < 0) { 818 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 819 return -1; 820 } 821 822 /* we now have a valid memory area, so add it to heap */ 823 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 824 param.addr, param.len, param.iova_table, 825 param.iova_table_len, param.pgsz); 826 827 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 828 829 /* not needed any more */ 830 free(param.iova_table); 831 832 if (ret < 0) { 833 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 834 munmap(param.addr, param.len); 835 return -1; 836 } 837 838 /* success */ 839 840 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 841 param.len >> 20); 842 843 return 0; 844 } 845 846 /* 847 * Configuration initialisation done once at init time. 848 */ 849 static struct rte_mempool * 850 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 851 unsigned int socket_id) 852 { 853 char pool_name[RTE_MEMPOOL_NAMESIZE]; 854 struct rte_mempool *rte_mp = NULL; 855 uint32_t mb_size; 856 857 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 858 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 859 860 TESTPMD_LOG(INFO, 861 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 862 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 863 864 switch (mp_alloc_type) { 865 case MP_ALLOC_NATIVE: 866 { 867 /* wrapper to rte_mempool_create() */ 868 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 869 rte_mbuf_best_mempool_ops()); 870 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 871 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 872 break; 873 } 874 case MP_ALLOC_ANON: 875 { 876 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 877 mb_size, (unsigned int) mb_mempool_cache, 878 sizeof(struct rte_pktmbuf_pool_private), 879 socket_id, 0); 880 if (rte_mp == NULL) 881 goto err; 882 883 if (rte_mempool_populate_anon(rte_mp) == 0) { 884 rte_mempool_free(rte_mp); 885 rte_mp = NULL; 886 goto err; 887 } 888 rte_pktmbuf_pool_init(rte_mp, NULL); 889 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 890 break; 891 } 892 case MP_ALLOC_XMEM: 893 case MP_ALLOC_XMEM_HUGE: 894 { 895 int heap_socket; 896 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 897 898 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 899 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 900 901 heap_socket = 902 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 903 if (heap_socket < 0) 904 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 905 906 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 907 rte_mbuf_best_mempool_ops()); 908 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 909 mb_mempool_cache, 0, mbuf_seg_size, 910 heap_socket); 911 break; 912 } 913 default: 914 { 915 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 916 } 917 } 918 919 err: 920 if (rte_mp == NULL) { 921 rte_exit(EXIT_FAILURE, 922 "Creation of mbuf pool for socket %u failed: %s\n", 923 socket_id, rte_strerror(rte_errno)); 924 } else if (verbose_level > 0) { 925 rte_mempool_dump(stdout, rte_mp); 926 } 927 return rte_mp; 928 } 929 930 /* 931 * Check given socket id is valid or not with NUMA mode, 932 * if valid, return 0, else return -1 933 */ 934 static int 935 check_socket_id(const unsigned int socket_id) 936 { 937 static int warning_once = 0; 938 939 if (new_socket_id(socket_id)) { 940 if (!warning_once && numa_support) 941 printf("Warning: NUMA should be configured manually by" 942 " using --port-numa-config and" 943 " --ring-numa-config parameters along with" 944 " --numa.\n"); 945 warning_once = 1; 946 return -1; 947 } 948 return 0; 949 } 950 951 /* 952 * Get the allowed maximum number of RX queues. 953 * *pid return the port id which has minimal value of 954 * max_rx_queues in all ports. 955 */ 956 queueid_t 957 get_allowed_max_nb_rxq(portid_t *pid) 958 { 959 queueid_t allowed_max_rxq = MAX_QUEUE_ID; 960 portid_t pi; 961 struct rte_eth_dev_info dev_info; 962 963 RTE_ETH_FOREACH_DEV(pi) { 964 rte_eth_dev_info_get(pi, &dev_info); 965 if (dev_info.max_rx_queues < allowed_max_rxq) { 966 allowed_max_rxq = dev_info.max_rx_queues; 967 *pid = pi; 968 } 969 } 970 return allowed_max_rxq; 971 } 972 973 /* 974 * Check input rxq is valid or not. 975 * If input rxq is not greater than any of maximum number 976 * of RX queues of all ports, it is valid. 977 * if valid, return 0, else return -1 978 */ 979 int 980 check_nb_rxq(queueid_t rxq) 981 { 982 queueid_t allowed_max_rxq; 983 portid_t pid = 0; 984 985 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 986 if (rxq > allowed_max_rxq) { 987 printf("Fail: input rxq (%u) can't be greater " 988 "than max_rx_queues (%u) of port %u\n", 989 rxq, 990 allowed_max_rxq, 991 pid); 992 return -1; 993 } 994 return 0; 995 } 996 997 /* 998 * Get the allowed maximum number of TX queues. 999 * *pid return the port id which has minimal value of 1000 * max_tx_queues in all ports. 1001 */ 1002 queueid_t 1003 get_allowed_max_nb_txq(portid_t *pid) 1004 { 1005 queueid_t allowed_max_txq = MAX_QUEUE_ID; 1006 portid_t pi; 1007 struct rte_eth_dev_info dev_info; 1008 1009 RTE_ETH_FOREACH_DEV(pi) { 1010 rte_eth_dev_info_get(pi, &dev_info); 1011 if (dev_info.max_tx_queues < allowed_max_txq) { 1012 allowed_max_txq = dev_info.max_tx_queues; 1013 *pid = pi; 1014 } 1015 } 1016 return allowed_max_txq; 1017 } 1018 1019 /* 1020 * Check input txq is valid or not. 1021 * If input txq is not greater than any of maximum number 1022 * of TX queues of all ports, it is valid. 1023 * if valid, return 0, else return -1 1024 */ 1025 int 1026 check_nb_txq(queueid_t txq) 1027 { 1028 queueid_t allowed_max_txq; 1029 portid_t pid = 0; 1030 1031 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1032 if (txq > allowed_max_txq) { 1033 printf("Fail: input txq (%u) can't be greater " 1034 "than max_tx_queues (%u) of port %u\n", 1035 txq, 1036 allowed_max_txq, 1037 pid); 1038 return -1; 1039 } 1040 return 0; 1041 } 1042 1043 static void 1044 init_config(void) 1045 { 1046 portid_t pid; 1047 struct rte_port *port; 1048 struct rte_mempool *mbp; 1049 unsigned int nb_mbuf_per_pool; 1050 lcoreid_t lc_id; 1051 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1052 struct rte_gro_param gro_param; 1053 uint32_t gso_types; 1054 int k; 1055 1056 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1057 1058 /* Configuration of logical cores. */ 1059 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1060 sizeof(struct fwd_lcore *) * nb_lcores, 1061 RTE_CACHE_LINE_SIZE); 1062 if (fwd_lcores == NULL) { 1063 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1064 "failed\n", nb_lcores); 1065 } 1066 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1067 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1068 sizeof(struct fwd_lcore), 1069 RTE_CACHE_LINE_SIZE); 1070 if (fwd_lcores[lc_id] == NULL) { 1071 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1072 "failed\n"); 1073 } 1074 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1075 } 1076 1077 RTE_ETH_FOREACH_DEV(pid) { 1078 port = &ports[pid]; 1079 /* Apply default TxRx configuration for all ports */ 1080 port->dev_conf.txmode = tx_mode; 1081 port->dev_conf.rxmode = rx_mode; 1082 rte_eth_dev_info_get(pid, &port->dev_info); 1083 1084 if (!(port->dev_info.tx_offload_capa & 1085 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1086 port->dev_conf.txmode.offloads &= 1087 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1088 if (!(port->dev_info.tx_offload_capa & 1089 DEV_TX_OFFLOAD_MATCH_METADATA)) 1090 port->dev_conf.txmode.offloads &= 1091 ~DEV_TX_OFFLOAD_MATCH_METADATA; 1092 if (numa_support) { 1093 if (port_numa[pid] != NUMA_NO_CONFIG) 1094 port_per_socket[port_numa[pid]]++; 1095 else { 1096 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1097 1098 /* 1099 * if socket_id is invalid, 1100 * set to the first available socket. 1101 */ 1102 if (check_socket_id(socket_id) < 0) 1103 socket_id = socket_ids[0]; 1104 port_per_socket[socket_id]++; 1105 } 1106 } 1107 1108 /* Apply Rx offloads configuration */ 1109 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1110 port->rx_conf[k].offloads = 1111 port->dev_conf.rxmode.offloads; 1112 /* Apply Tx offloads configuration */ 1113 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1114 port->tx_conf[k].offloads = 1115 port->dev_conf.txmode.offloads; 1116 1117 /* set flag to initialize port/queue */ 1118 port->need_reconfig = 1; 1119 port->need_reconfig_queues = 1; 1120 port->tx_metadata = 0; 1121 } 1122 1123 /* 1124 * Create pools of mbuf. 1125 * If NUMA support is disabled, create a single pool of mbuf in 1126 * socket 0 memory by default. 1127 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1128 * 1129 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1130 * nb_txd can be configured at run time. 1131 */ 1132 if (param_total_num_mbufs) 1133 nb_mbuf_per_pool = param_total_num_mbufs; 1134 else { 1135 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1136 (nb_lcores * mb_mempool_cache) + 1137 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1138 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1139 } 1140 1141 if (numa_support) { 1142 uint8_t i; 1143 1144 for (i = 0; i < num_sockets; i++) 1145 mempools[i] = mbuf_pool_create(mbuf_data_size, 1146 nb_mbuf_per_pool, 1147 socket_ids[i]); 1148 } else { 1149 if (socket_num == UMA_NO_CONFIG) 1150 mempools[0] = mbuf_pool_create(mbuf_data_size, 1151 nb_mbuf_per_pool, 0); 1152 else 1153 mempools[socket_num] = mbuf_pool_create 1154 (mbuf_data_size, 1155 nb_mbuf_per_pool, 1156 socket_num); 1157 } 1158 1159 init_port_config(); 1160 1161 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1162 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1163 /* 1164 * Records which Mbuf pool to use by each logical core, if needed. 1165 */ 1166 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1167 mbp = mbuf_pool_find( 1168 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1169 1170 if (mbp == NULL) 1171 mbp = mbuf_pool_find(0); 1172 fwd_lcores[lc_id]->mbp = mbp; 1173 /* initialize GSO context */ 1174 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1175 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1176 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1177 fwd_lcores[lc_id]->gso_ctx.gso_size = ETHER_MAX_LEN - 1178 ETHER_CRC_LEN; 1179 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1180 } 1181 1182 /* Configuration of packet forwarding streams. */ 1183 if (init_fwd_streams() < 0) 1184 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1185 1186 fwd_config_setup(); 1187 1188 /* create a gro context for each lcore */ 1189 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1190 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1191 gro_param.max_item_per_flow = MAX_PKT_BURST; 1192 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1193 gro_param.socket_id = rte_lcore_to_socket_id( 1194 fwd_lcores_cpuids[lc_id]); 1195 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1196 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1197 rte_exit(EXIT_FAILURE, 1198 "rte_gro_ctx_create() failed\n"); 1199 } 1200 } 1201 1202 #if defined RTE_LIBRTE_PMD_SOFTNIC 1203 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1204 RTE_ETH_FOREACH_DEV(pid) { 1205 port = &ports[pid]; 1206 const char *driver = port->dev_info.driver_name; 1207 1208 if (strcmp(driver, "net_softnic") == 0) 1209 port->softport.fwd_lcore_arg = fwd_lcores; 1210 } 1211 } 1212 #endif 1213 1214 } 1215 1216 1217 void 1218 reconfig(portid_t new_port_id, unsigned socket_id) 1219 { 1220 struct rte_port *port; 1221 1222 /* Reconfiguration of Ethernet ports. */ 1223 port = &ports[new_port_id]; 1224 rte_eth_dev_info_get(new_port_id, &port->dev_info); 1225 1226 /* set flag to initialize port/queue */ 1227 port->need_reconfig = 1; 1228 port->need_reconfig_queues = 1; 1229 port->socket_id = socket_id; 1230 1231 init_port_config(); 1232 } 1233 1234 1235 int 1236 init_fwd_streams(void) 1237 { 1238 portid_t pid; 1239 struct rte_port *port; 1240 streamid_t sm_id, nb_fwd_streams_new; 1241 queueid_t q; 1242 1243 /* set socket id according to numa or not */ 1244 RTE_ETH_FOREACH_DEV(pid) { 1245 port = &ports[pid]; 1246 if (nb_rxq > port->dev_info.max_rx_queues) { 1247 printf("Fail: nb_rxq(%d) is greater than " 1248 "max_rx_queues(%d)\n", nb_rxq, 1249 port->dev_info.max_rx_queues); 1250 return -1; 1251 } 1252 if (nb_txq > port->dev_info.max_tx_queues) { 1253 printf("Fail: nb_txq(%d) is greater than " 1254 "max_tx_queues(%d)\n", nb_txq, 1255 port->dev_info.max_tx_queues); 1256 return -1; 1257 } 1258 if (numa_support) { 1259 if (port_numa[pid] != NUMA_NO_CONFIG) 1260 port->socket_id = port_numa[pid]; 1261 else { 1262 port->socket_id = rte_eth_dev_socket_id(pid); 1263 1264 /* 1265 * if socket_id is invalid, 1266 * set to the first available socket. 1267 */ 1268 if (check_socket_id(port->socket_id) < 0) 1269 port->socket_id = socket_ids[0]; 1270 } 1271 } 1272 else { 1273 if (socket_num == UMA_NO_CONFIG) 1274 port->socket_id = 0; 1275 else 1276 port->socket_id = socket_num; 1277 } 1278 } 1279 1280 q = RTE_MAX(nb_rxq, nb_txq); 1281 if (q == 0) { 1282 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1283 return -1; 1284 } 1285 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1286 if (nb_fwd_streams_new == nb_fwd_streams) 1287 return 0; 1288 /* clear the old */ 1289 if (fwd_streams != NULL) { 1290 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1291 if (fwd_streams[sm_id] == NULL) 1292 continue; 1293 rte_free(fwd_streams[sm_id]); 1294 fwd_streams[sm_id] = NULL; 1295 } 1296 rte_free(fwd_streams); 1297 fwd_streams = NULL; 1298 } 1299 1300 /* init new */ 1301 nb_fwd_streams = nb_fwd_streams_new; 1302 if (nb_fwd_streams) { 1303 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1304 sizeof(struct fwd_stream *) * nb_fwd_streams, 1305 RTE_CACHE_LINE_SIZE); 1306 if (fwd_streams == NULL) 1307 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1308 " (struct fwd_stream *)) failed\n", 1309 nb_fwd_streams); 1310 1311 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1312 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1313 " struct fwd_stream", sizeof(struct fwd_stream), 1314 RTE_CACHE_LINE_SIZE); 1315 if (fwd_streams[sm_id] == NULL) 1316 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1317 "(struct fwd_stream) failed\n"); 1318 } 1319 } 1320 1321 return 0; 1322 } 1323 1324 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1325 static void 1326 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1327 { 1328 unsigned int total_burst; 1329 unsigned int nb_burst; 1330 unsigned int burst_stats[3]; 1331 uint16_t pktnb_stats[3]; 1332 uint16_t nb_pkt; 1333 int burst_percent[3]; 1334 1335 /* 1336 * First compute the total number of packet bursts and the 1337 * two highest numbers of bursts of the same number of packets. 1338 */ 1339 total_burst = 0; 1340 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1341 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1342 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1343 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1344 if (nb_burst == 0) 1345 continue; 1346 total_burst += nb_burst; 1347 if (nb_burst > burst_stats[0]) { 1348 burst_stats[1] = burst_stats[0]; 1349 pktnb_stats[1] = pktnb_stats[0]; 1350 burst_stats[0] = nb_burst; 1351 pktnb_stats[0] = nb_pkt; 1352 } else if (nb_burst > burst_stats[1]) { 1353 burst_stats[1] = nb_burst; 1354 pktnb_stats[1] = nb_pkt; 1355 } 1356 } 1357 if (total_burst == 0) 1358 return; 1359 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1360 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1361 burst_percent[0], (int) pktnb_stats[0]); 1362 if (burst_stats[0] == total_burst) { 1363 printf("]\n"); 1364 return; 1365 } 1366 if (burst_stats[0] + burst_stats[1] == total_burst) { 1367 printf(" + %d%% of %d pkts]\n", 1368 100 - burst_percent[0], pktnb_stats[1]); 1369 return; 1370 } 1371 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1372 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1373 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1374 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1375 return; 1376 } 1377 printf(" + %d%% of %d pkts + %d%% of others]\n", 1378 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1379 } 1380 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1381 1382 static void 1383 fwd_port_stats_display(portid_t port_id, struct rte_eth_stats *stats) 1384 { 1385 struct rte_port *port; 1386 uint8_t i; 1387 1388 static const char *fwd_stats_border = "----------------------"; 1389 1390 port = &ports[port_id]; 1391 printf("\n %s Forward statistics for port %-2d %s\n", 1392 fwd_stats_border, port_id, fwd_stats_border); 1393 1394 if ((!port->rx_queue_stats_mapping_enabled) && (!port->tx_queue_stats_mapping_enabled)) { 1395 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1396 "%-"PRIu64"\n", 1397 stats->ipackets, stats->imissed, 1398 stats->ipackets + stats->imissed); 1399 1400 if (cur_fwd_eng == &csum_fwd_engine) 1401 printf(" Bad-ipcsum: %-14"PRIu64" Bad-l4csum: %-14"PRIu64"Bad-outer-l4csum: %-14"PRIu64"\n", 1402 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1403 port->rx_bad_outer_l4_csum); 1404 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1405 printf(" RX-error: %-"PRIu64"\n", stats->ierrors); 1406 printf(" RX-nombufs: %-14"PRIu64"\n", stats->rx_nombuf); 1407 } 1408 1409 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1410 "%-"PRIu64"\n", 1411 stats->opackets, port->tx_dropped, 1412 stats->opackets + port->tx_dropped); 1413 } 1414 else { 1415 printf(" RX-packets: %14"PRIu64" RX-dropped:%14"PRIu64" RX-total:" 1416 "%14"PRIu64"\n", 1417 stats->ipackets, stats->imissed, 1418 stats->ipackets + stats->imissed); 1419 1420 if (cur_fwd_eng == &csum_fwd_engine) 1421 printf(" Bad-ipcsum:%14"PRIu64" Bad-l4csum:%14"PRIu64" Bad-outer-l4csum: %-14"PRIu64"\n", 1422 port->rx_bad_ip_csum, port->rx_bad_l4_csum, 1423 port->rx_bad_outer_l4_csum); 1424 if ((stats->ierrors + stats->rx_nombuf) > 0) { 1425 printf(" RX-error:%"PRIu64"\n", stats->ierrors); 1426 printf(" RX-nombufs: %14"PRIu64"\n", 1427 stats->rx_nombuf); 1428 } 1429 1430 printf(" TX-packets: %14"PRIu64" TX-dropped:%14"PRIu64" TX-total:" 1431 "%14"PRIu64"\n", 1432 stats->opackets, port->tx_dropped, 1433 stats->opackets + port->tx_dropped); 1434 } 1435 1436 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1437 if (port->rx_stream) 1438 pkt_burst_stats_display("RX", 1439 &port->rx_stream->rx_burst_stats); 1440 if (port->tx_stream) 1441 pkt_burst_stats_display("TX", 1442 &port->tx_stream->tx_burst_stats); 1443 #endif 1444 1445 if (port->rx_queue_stats_mapping_enabled) { 1446 printf("\n"); 1447 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1448 printf(" Stats reg %2d RX-packets:%14"PRIu64 1449 " RX-errors:%14"PRIu64 1450 " RX-bytes:%14"PRIu64"\n", 1451 i, stats->q_ipackets[i], stats->q_errors[i], stats->q_ibytes[i]); 1452 } 1453 printf("\n"); 1454 } 1455 if (port->tx_queue_stats_mapping_enabled) { 1456 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { 1457 printf(" Stats reg %2d TX-packets:%14"PRIu64 1458 " TX-bytes:%14"PRIu64"\n", 1459 i, stats->q_opackets[i], stats->q_obytes[i]); 1460 } 1461 } 1462 1463 printf(" %s--------------------------------%s\n", 1464 fwd_stats_border, fwd_stats_border); 1465 } 1466 1467 static void 1468 fwd_stream_stats_display(streamid_t stream_id) 1469 { 1470 struct fwd_stream *fs; 1471 static const char *fwd_top_stats_border = "-------"; 1472 1473 fs = fwd_streams[stream_id]; 1474 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1475 (fs->fwd_dropped == 0)) 1476 return; 1477 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1478 "TX Port=%2d/Queue=%2d %s\n", 1479 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1480 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1481 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1482 " TX-dropped: %-14"PRIu64, 1483 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1484 1485 /* if checksum mode */ 1486 if (cur_fwd_eng == &csum_fwd_engine) { 1487 printf(" RX- bad IP checksum: %-14"PRIu64 1488 " Rx- bad L4 checksum: %-14"PRIu64 1489 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1490 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1491 fs->rx_bad_outer_l4_csum); 1492 } else { 1493 printf("\n"); 1494 } 1495 1496 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1497 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1498 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1499 #endif 1500 } 1501 1502 static void 1503 flush_fwd_rx_queues(void) 1504 { 1505 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1506 portid_t rxp; 1507 portid_t port_id; 1508 queueid_t rxq; 1509 uint16_t nb_rx; 1510 uint16_t i; 1511 uint8_t j; 1512 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1513 uint64_t timer_period; 1514 1515 /* convert to number of cycles */ 1516 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1517 1518 for (j = 0; j < 2; j++) { 1519 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1520 for (rxq = 0; rxq < nb_rxq; rxq++) { 1521 port_id = fwd_ports_ids[rxp]; 1522 /** 1523 * testpmd can stuck in the below do while loop 1524 * if rte_eth_rx_burst() always returns nonzero 1525 * packets. So timer is added to exit this loop 1526 * after 1sec timer expiry. 1527 */ 1528 prev_tsc = rte_rdtsc(); 1529 do { 1530 nb_rx = rte_eth_rx_burst(port_id, rxq, 1531 pkts_burst, MAX_PKT_BURST); 1532 for (i = 0; i < nb_rx; i++) 1533 rte_pktmbuf_free(pkts_burst[i]); 1534 1535 cur_tsc = rte_rdtsc(); 1536 diff_tsc = cur_tsc - prev_tsc; 1537 timer_tsc += diff_tsc; 1538 } while ((nb_rx > 0) && 1539 (timer_tsc < timer_period)); 1540 timer_tsc = 0; 1541 } 1542 } 1543 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1544 } 1545 } 1546 1547 static void 1548 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1549 { 1550 struct fwd_stream **fsm; 1551 streamid_t nb_fs; 1552 streamid_t sm_id; 1553 #ifdef RTE_LIBRTE_BITRATE 1554 uint64_t tics_per_1sec; 1555 uint64_t tics_datum; 1556 uint64_t tics_current; 1557 uint16_t i, cnt_ports; 1558 1559 cnt_ports = nb_ports; 1560 tics_datum = rte_rdtsc(); 1561 tics_per_1sec = rte_get_timer_hz(); 1562 #endif 1563 fsm = &fwd_streams[fc->stream_idx]; 1564 nb_fs = fc->stream_nb; 1565 do { 1566 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1567 (*pkt_fwd)(fsm[sm_id]); 1568 #ifdef RTE_LIBRTE_BITRATE 1569 if (bitrate_enabled != 0 && 1570 bitrate_lcore_id == rte_lcore_id()) { 1571 tics_current = rte_rdtsc(); 1572 if (tics_current - tics_datum >= tics_per_1sec) { 1573 /* Periodic bitrate calculation */ 1574 for (i = 0; i < cnt_ports; i++) 1575 rte_stats_bitrate_calc(bitrate_data, 1576 ports_ids[i]); 1577 tics_datum = tics_current; 1578 } 1579 } 1580 #endif 1581 #ifdef RTE_LIBRTE_LATENCY_STATS 1582 if (latencystats_enabled != 0 && 1583 latencystats_lcore_id == rte_lcore_id()) 1584 rte_latencystats_update(); 1585 #endif 1586 1587 } while (! fc->stopped); 1588 } 1589 1590 static int 1591 start_pkt_forward_on_core(void *fwd_arg) 1592 { 1593 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1594 cur_fwd_config.fwd_eng->packet_fwd); 1595 return 0; 1596 } 1597 1598 /* 1599 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1600 * Used to start communication flows in network loopback test configurations. 1601 */ 1602 static int 1603 run_one_txonly_burst_on_core(void *fwd_arg) 1604 { 1605 struct fwd_lcore *fwd_lc; 1606 struct fwd_lcore tmp_lcore; 1607 1608 fwd_lc = (struct fwd_lcore *) fwd_arg; 1609 tmp_lcore = *fwd_lc; 1610 tmp_lcore.stopped = 1; 1611 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1612 return 0; 1613 } 1614 1615 /* 1616 * Launch packet forwarding: 1617 * - Setup per-port forwarding context. 1618 * - launch logical cores with their forwarding configuration. 1619 */ 1620 static void 1621 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1622 { 1623 port_fwd_begin_t port_fwd_begin; 1624 unsigned int i; 1625 unsigned int lc_id; 1626 int diag; 1627 1628 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1629 if (port_fwd_begin != NULL) { 1630 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1631 (*port_fwd_begin)(fwd_ports_ids[i]); 1632 } 1633 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1634 lc_id = fwd_lcores_cpuids[i]; 1635 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1636 fwd_lcores[i]->stopped = 0; 1637 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1638 fwd_lcores[i], lc_id); 1639 if (diag != 0) 1640 printf("launch lcore %u failed - diag=%d\n", 1641 lc_id, diag); 1642 } 1643 } 1644 } 1645 1646 /* 1647 * Launch packet forwarding configuration. 1648 */ 1649 void 1650 start_packet_forwarding(int with_tx_first) 1651 { 1652 port_fwd_begin_t port_fwd_begin; 1653 port_fwd_end_t port_fwd_end; 1654 struct rte_port *port; 1655 unsigned int i; 1656 portid_t pt_id; 1657 streamid_t sm_id; 1658 1659 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1660 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1661 1662 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1663 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1664 1665 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1666 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1667 (!nb_rxq || !nb_txq)) 1668 rte_exit(EXIT_FAILURE, 1669 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1670 cur_fwd_eng->fwd_mode_name); 1671 1672 if (all_ports_started() == 0) { 1673 printf("Not all ports were started\n"); 1674 return; 1675 } 1676 if (test_done == 0) { 1677 printf("Packet forwarding already started\n"); 1678 return; 1679 } 1680 1681 1682 if(dcb_test) { 1683 for (i = 0; i < nb_fwd_ports; i++) { 1684 pt_id = fwd_ports_ids[i]; 1685 port = &ports[pt_id]; 1686 if (!port->dcb_flag) { 1687 printf("In DCB mode, all forwarding ports must " 1688 "be configured in this mode.\n"); 1689 return; 1690 } 1691 } 1692 if (nb_fwd_lcores == 1) { 1693 printf("In DCB mode,the nb forwarding cores " 1694 "should be larger than 1.\n"); 1695 return; 1696 } 1697 } 1698 test_done = 0; 1699 1700 fwd_config_setup(); 1701 1702 if(!no_flush_rx) 1703 flush_fwd_rx_queues(); 1704 1705 pkt_fwd_config_display(&cur_fwd_config); 1706 rxtx_config_display(); 1707 1708 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1709 pt_id = fwd_ports_ids[i]; 1710 port = &ports[pt_id]; 1711 rte_eth_stats_get(pt_id, &port->stats); 1712 port->tx_dropped = 0; 1713 1714 map_port_queue_stats_mapping_registers(pt_id, port); 1715 } 1716 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1717 fwd_streams[sm_id]->rx_packets = 0; 1718 fwd_streams[sm_id]->tx_packets = 0; 1719 fwd_streams[sm_id]->fwd_dropped = 0; 1720 fwd_streams[sm_id]->rx_bad_ip_csum = 0; 1721 fwd_streams[sm_id]->rx_bad_l4_csum = 0; 1722 fwd_streams[sm_id]->rx_bad_outer_l4_csum = 0; 1723 1724 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1725 memset(&fwd_streams[sm_id]->rx_burst_stats, 0, 1726 sizeof(fwd_streams[sm_id]->rx_burst_stats)); 1727 memset(&fwd_streams[sm_id]->tx_burst_stats, 0, 1728 sizeof(fwd_streams[sm_id]->tx_burst_stats)); 1729 #endif 1730 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1731 fwd_streams[sm_id]->core_cycles = 0; 1732 #endif 1733 } 1734 if (with_tx_first) { 1735 port_fwd_begin = tx_only_engine.port_fwd_begin; 1736 if (port_fwd_begin != NULL) { 1737 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1738 (*port_fwd_begin)(fwd_ports_ids[i]); 1739 } 1740 while (with_tx_first--) { 1741 launch_packet_forwarding( 1742 run_one_txonly_burst_on_core); 1743 rte_eal_mp_wait_lcore(); 1744 } 1745 port_fwd_end = tx_only_engine.port_fwd_end; 1746 if (port_fwd_end != NULL) { 1747 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1748 (*port_fwd_end)(fwd_ports_ids[i]); 1749 } 1750 } 1751 launch_packet_forwarding(start_pkt_forward_on_core); 1752 } 1753 1754 void 1755 stop_packet_forwarding(void) 1756 { 1757 struct rte_eth_stats stats; 1758 struct rte_port *port; 1759 port_fwd_end_t port_fwd_end; 1760 int i; 1761 portid_t pt_id; 1762 streamid_t sm_id; 1763 lcoreid_t lc_id; 1764 uint64_t total_recv; 1765 uint64_t total_xmit; 1766 uint64_t total_rx_dropped; 1767 uint64_t total_tx_dropped; 1768 uint64_t total_rx_nombuf; 1769 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1770 uint64_t fwd_cycles; 1771 #endif 1772 1773 static const char *acc_stats_border = "+++++++++++++++"; 1774 1775 if (test_done) { 1776 printf("Packet forwarding not started\n"); 1777 return; 1778 } 1779 printf("Telling cores to stop..."); 1780 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1781 fwd_lcores[lc_id]->stopped = 1; 1782 printf("\nWaiting for lcores to finish...\n"); 1783 rte_eal_mp_wait_lcore(); 1784 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1785 if (port_fwd_end != NULL) { 1786 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1787 pt_id = fwd_ports_ids[i]; 1788 (*port_fwd_end)(pt_id); 1789 } 1790 } 1791 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1792 fwd_cycles = 0; 1793 #endif 1794 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1795 struct fwd_stream *fs = fwd_streams[sm_id]; 1796 1797 if (cur_fwd_config.nb_fwd_streams > 1798 cur_fwd_config.nb_fwd_ports) { 1799 fwd_stream_stats_display(sm_id); 1800 ports[fs->tx_port].tx_stream = NULL; 1801 ports[fs->rx_port].rx_stream = NULL; 1802 } else { 1803 ports[fs->tx_port].tx_stream = fs; 1804 ports[fs->rx_port].rx_stream = fs; 1805 } 1806 ports[fs->tx_port].tx_dropped += fs->fwd_dropped; 1807 ports[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1808 ports[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1809 ports[fs->rx_port].rx_bad_outer_l4_csum += 1810 fs->rx_bad_outer_l4_csum; 1811 1812 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1813 fwd_cycles = (uint64_t) (fwd_cycles + 1814 fwd_streams[sm_id]->core_cycles); 1815 #endif 1816 } 1817 total_recv = 0; 1818 total_xmit = 0; 1819 total_rx_dropped = 0; 1820 total_tx_dropped = 0; 1821 total_rx_nombuf = 0; 1822 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1823 pt_id = fwd_ports_ids[i]; 1824 1825 port = &ports[pt_id]; 1826 rte_eth_stats_get(pt_id, &stats); 1827 stats.ipackets -= port->stats.ipackets; 1828 port->stats.ipackets = 0; 1829 stats.opackets -= port->stats.opackets; 1830 port->stats.opackets = 0; 1831 stats.ibytes -= port->stats.ibytes; 1832 port->stats.ibytes = 0; 1833 stats.obytes -= port->stats.obytes; 1834 port->stats.obytes = 0; 1835 stats.imissed -= port->stats.imissed; 1836 port->stats.imissed = 0; 1837 stats.oerrors -= port->stats.oerrors; 1838 port->stats.oerrors = 0; 1839 stats.rx_nombuf -= port->stats.rx_nombuf; 1840 port->stats.rx_nombuf = 0; 1841 1842 total_recv += stats.ipackets; 1843 total_xmit += stats.opackets; 1844 total_rx_dropped += stats.imissed; 1845 total_tx_dropped += port->tx_dropped; 1846 total_rx_nombuf += stats.rx_nombuf; 1847 1848 fwd_port_stats_display(pt_id, &stats); 1849 } 1850 1851 printf("\n %s Accumulated forward statistics for all ports" 1852 "%s\n", 1853 acc_stats_border, acc_stats_border); 1854 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1855 "%-"PRIu64"\n" 1856 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1857 "%-"PRIu64"\n", 1858 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1859 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1860 if (total_rx_nombuf > 0) 1861 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1862 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1863 "%s\n", 1864 acc_stats_border, acc_stats_border); 1865 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1866 if (total_recv > 0) 1867 printf("\n CPU cycles/packet=%u (total cycles=" 1868 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1869 (unsigned int)(fwd_cycles / total_recv), 1870 fwd_cycles, total_recv); 1871 #endif 1872 printf("\nDone.\n"); 1873 test_done = 1; 1874 } 1875 1876 void 1877 dev_set_link_up(portid_t pid) 1878 { 1879 if (rte_eth_dev_set_link_up(pid) < 0) 1880 printf("\nSet link up fail.\n"); 1881 } 1882 1883 void 1884 dev_set_link_down(portid_t pid) 1885 { 1886 if (rte_eth_dev_set_link_down(pid) < 0) 1887 printf("\nSet link down fail.\n"); 1888 } 1889 1890 static int 1891 all_ports_started(void) 1892 { 1893 portid_t pi; 1894 struct rte_port *port; 1895 1896 RTE_ETH_FOREACH_DEV(pi) { 1897 port = &ports[pi]; 1898 /* Check if there is a port which is not started */ 1899 if ((port->port_status != RTE_PORT_STARTED) && 1900 (port->slave_flag == 0)) 1901 return 0; 1902 } 1903 1904 /* No port is not started */ 1905 return 1; 1906 } 1907 1908 int 1909 port_is_stopped(portid_t port_id) 1910 { 1911 struct rte_port *port = &ports[port_id]; 1912 1913 if ((port->port_status != RTE_PORT_STOPPED) && 1914 (port->slave_flag == 0)) 1915 return 0; 1916 return 1; 1917 } 1918 1919 int 1920 all_ports_stopped(void) 1921 { 1922 portid_t pi; 1923 1924 RTE_ETH_FOREACH_DEV(pi) { 1925 if (!port_is_stopped(pi)) 1926 return 0; 1927 } 1928 1929 return 1; 1930 } 1931 1932 int 1933 port_is_started(portid_t port_id) 1934 { 1935 if (port_id_is_invalid(port_id, ENABLED_WARN)) 1936 return 0; 1937 1938 if (ports[port_id].port_status != RTE_PORT_STARTED) 1939 return 0; 1940 1941 return 1; 1942 } 1943 1944 int 1945 start_port(portid_t pid) 1946 { 1947 int diag, need_check_link_status = -1; 1948 portid_t pi; 1949 queueid_t qi; 1950 struct rte_port *port; 1951 struct ether_addr mac_addr; 1952 1953 if (port_id_is_invalid(pid, ENABLED_WARN)) 1954 return 0; 1955 1956 if(dcb_config) 1957 dcb_test = 1; 1958 RTE_ETH_FOREACH_DEV(pi) { 1959 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 1960 continue; 1961 1962 need_check_link_status = 0; 1963 port = &ports[pi]; 1964 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 1965 RTE_PORT_HANDLING) == 0) { 1966 printf("Port %d is now not stopped\n", pi); 1967 continue; 1968 } 1969 1970 if (port->need_reconfig > 0) { 1971 port->need_reconfig = 0; 1972 1973 if (flow_isolate_all) { 1974 int ret = port_flow_isolate(pi, 1); 1975 if (ret) { 1976 printf("Failed to apply isolated" 1977 " mode on port %d\n", pi); 1978 return -1; 1979 } 1980 } 1981 configure_rxtx_dump_callbacks(0); 1982 printf("Configuring Port %d (socket %u)\n", pi, 1983 port->socket_id); 1984 /* configure port */ 1985 diag = rte_eth_dev_configure(pi, nb_rxq, nb_txq, 1986 &(port->dev_conf)); 1987 if (diag != 0) { 1988 if (rte_atomic16_cmpset(&(port->port_status), 1989 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 1990 printf("Port %d can not be set back " 1991 "to stopped\n", pi); 1992 printf("Fail to configure port %d\n", pi); 1993 /* try to reconfigure port next time */ 1994 port->need_reconfig = 1; 1995 return -1; 1996 } 1997 } 1998 if (port->need_reconfig_queues > 0) { 1999 port->need_reconfig_queues = 0; 2000 /* setup tx queues */ 2001 for (qi = 0; qi < nb_txq; qi++) { 2002 if ((numa_support) && 2003 (txring_numa[pi] != NUMA_NO_CONFIG)) 2004 diag = rte_eth_tx_queue_setup(pi, qi, 2005 port->nb_tx_desc[qi], 2006 txring_numa[pi], 2007 &(port->tx_conf[qi])); 2008 else 2009 diag = rte_eth_tx_queue_setup(pi, qi, 2010 port->nb_tx_desc[qi], 2011 port->socket_id, 2012 &(port->tx_conf[qi])); 2013 2014 if (diag == 0) 2015 continue; 2016 2017 /* Fail to setup tx queue, return */ 2018 if (rte_atomic16_cmpset(&(port->port_status), 2019 RTE_PORT_HANDLING, 2020 RTE_PORT_STOPPED) == 0) 2021 printf("Port %d can not be set back " 2022 "to stopped\n", pi); 2023 printf("Fail to configure port %d tx queues\n", 2024 pi); 2025 /* try to reconfigure queues next time */ 2026 port->need_reconfig_queues = 1; 2027 return -1; 2028 } 2029 for (qi = 0; qi < nb_rxq; qi++) { 2030 /* setup rx queues */ 2031 if ((numa_support) && 2032 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2033 struct rte_mempool * mp = 2034 mbuf_pool_find(rxring_numa[pi]); 2035 if (mp == NULL) { 2036 printf("Failed to setup RX queue:" 2037 "No mempool allocation" 2038 " on the socket %d\n", 2039 rxring_numa[pi]); 2040 return -1; 2041 } 2042 2043 diag = rte_eth_rx_queue_setup(pi, qi, 2044 port->nb_rx_desc[qi], 2045 rxring_numa[pi], 2046 &(port->rx_conf[qi]), 2047 mp); 2048 } else { 2049 struct rte_mempool *mp = 2050 mbuf_pool_find(port->socket_id); 2051 if (mp == NULL) { 2052 printf("Failed to setup RX queue:" 2053 "No mempool allocation" 2054 " on the socket %d\n", 2055 port->socket_id); 2056 return -1; 2057 } 2058 diag = rte_eth_rx_queue_setup(pi, qi, 2059 port->nb_rx_desc[qi], 2060 port->socket_id, 2061 &(port->rx_conf[qi]), 2062 mp); 2063 } 2064 if (diag == 0) 2065 continue; 2066 2067 /* Fail to setup rx queue, return */ 2068 if (rte_atomic16_cmpset(&(port->port_status), 2069 RTE_PORT_HANDLING, 2070 RTE_PORT_STOPPED) == 0) 2071 printf("Port %d can not be set back " 2072 "to stopped\n", pi); 2073 printf("Fail to configure port %d rx queues\n", 2074 pi); 2075 /* try to reconfigure queues next time */ 2076 port->need_reconfig_queues = 1; 2077 return -1; 2078 } 2079 } 2080 configure_rxtx_dump_callbacks(verbose_level); 2081 /* start port */ 2082 if (rte_eth_dev_start(pi) < 0) { 2083 printf("Fail to start port %d\n", pi); 2084 2085 /* Fail to setup rx queue, return */ 2086 if (rte_atomic16_cmpset(&(port->port_status), 2087 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2088 printf("Port %d can not be set back to " 2089 "stopped\n", pi); 2090 continue; 2091 } 2092 2093 if (rte_atomic16_cmpset(&(port->port_status), 2094 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2095 printf("Port %d can not be set into started\n", pi); 2096 2097 rte_eth_macaddr_get(pi, &mac_addr); 2098 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2099 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2100 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2101 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2102 2103 /* at least one port started, need checking link status */ 2104 need_check_link_status = 1; 2105 } 2106 2107 if (need_check_link_status == 1 && !no_link_check) 2108 check_all_ports_link_status(RTE_PORT_ALL); 2109 else if (need_check_link_status == 0) 2110 printf("Please stop the ports first\n"); 2111 2112 printf("Done\n"); 2113 return 0; 2114 } 2115 2116 void 2117 stop_port(portid_t pid) 2118 { 2119 portid_t pi; 2120 struct rte_port *port; 2121 int need_check_link_status = 0; 2122 2123 if (dcb_test) { 2124 dcb_test = 0; 2125 dcb_config = 0; 2126 } 2127 2128 if (port_id_is_invalid(pid, ENABLED_WARN)) 2129 return; 2130 2131 printf("Stopping ports...\n"); 2132 2133 RTE_ETH_FOREACH_DEV(pi) { 2134 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2135 continue; 2136 2137 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2138 printf("Please remove port %d from forwarding configuration.\n", pi); 2139 continue; 2140 } 2141 2142 if (port_is_bonding_slave(pi)) { 2143 printf("Please remove port %d from bonded device.\n", pi); 2144 continue; 2145 } 2146 2147 port = &ports[pi]; 2148 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2149 RTE_PORT_HANDLING) == 0) 2150 continue; 2151 2152 rte_eth_dev_stop(pi); 2153 2154 if (rte_atomic16_cmpset(&(port->port_status), 2155 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2156 printf("Port %d can not be set into stopped\n", pi); 2157 need_check_link_status = 1; 2158 } 2159 if (need_check_link_status && !no_link_check) 2160 check_all_ports_link_status(RTE_PORT_ALL); 2161 2162 printf("Done\n"); 2163 } 2164 2165 static void 2166 remove_invalid_ports_in(portid_t *array, portid_t *total) 2167 { 2168 portid_t i; 2169 portid_t new_total = 0; 2170 2171 for (i = 0; i < *total; i++) 2172 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2173 array[new_total] = array[i]; 2174 new_total++; 2175 } 2176 *total = new_total; 2177 } 2178 2179 static void 2180 remove_invalid_ports(void) 2181 { 2182 remove_invalid_ports_in(ports_ids, &nb_ports); 2183 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2184 nb_cfg_ports = nb_fwd_ports; 2185 } 2186 2187 void 2188 close_port(portid_t pid) 2189 { 2190 portid_t pi; 2191 struct rte_port *port; 2192 2193 if (port_id_is_invalid(pid, ENABLED_WARN)) 2194 return; 2195 2196 printf("Closing ports...\n"); 2197 2198 RTE_ETH_FOREACH_DEV(pi) { 2199 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2200 continue; 2201 2202 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2203 printf("Please remove port %d from forwarding configuration.\n", pi); 2204 continue; 2205 } 2206 2207 if (port_is_bonding_slave(pi)) { 2208 printf("Please remove port %d from bonded device.\n", pi); 2209 continue; 2210 } 2211 2212 port = &ports[pi]; 2213 if (rte_atomic16_cmpset(&(port->port_status), 2214 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2215 printf("Port %d is already closed\n", pi); 2216 continue; 2217 } 2218 2219 if (rte_atomic16_cmpset(&(port->port_status), 2220 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2221 printf("Port %d is now not stopped\n", pi); 2222 continue; 2223 } 2224 2225 if (port->flow_list) 2226 port_flow_flush(pi); 2227 rte_eth_dev_close(pi); 2228 2229 remove_invalid_ports(); 2230 2231 if (rte_atomic16_cmpset(&(port->port_status), 2232 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2233 printf("Port %d cannot be set to closed\n", pi); 2234 } 2235 2236 printf("Done\n"); 2237 } 2238 2239 void 2240 reset_port(portid_t pid) 2241 { 2242 int diag; 2243 portid_t pi; 2244 struct rte_port *port; 2245 2246 if (port_id_is_invalid(pid, ENABLED_WARN)) 2247 return; 2248 2249 printf("Resetting ports...\n"); 2250 2251 RTE_ETH_FOREACH_DEV(pi) { 2252 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2253 continue; 2254 2255 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2256 printf("Please remove port %d from forwarding " 2257 "configuration.\n", pi); 2258 continue; 2259 } 2260 2261 if (port_is_bonding_slave(pi)) { 2262 printf("Please remove port %d from bonded device.\n", 2263 pi); 2264 continue; 2265 } 2266 2267 diag = rte_eth_dev_reset(pi); 2268 if (diag == 0) { 2269 port = &ports[pi]; 2270 port->need_reconfig = 1; 2271 port->need_reconfig_queues = 1; 2272 } else { 2273 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2274 } 2275 } 2276 2277 printf("Done\n"); 2278 } 2279 2280 void 2281 attach_port(char *identifier) 2282 { 2283 portid_t pi; 2284 struct rte_dev_iterator iterator; 2285 2286 printf("Attaching a new port...\n"); 2287 2288 if (identifier == NULL) { 2289 printf("Invalid parameters are specified\n"); 2290 return; 2291 } 2292 2293 if (rte_dev_probe(identifier) != 0) { 2294 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2295 return; 2296 } 2297 2298 /* first attach mode: event */ 2299 if (setup_on_probe_event) { 2300 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2301 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2302 if (ports[pi].port_status == RTE_PORT_HANDLING && 2303 ports[pi].need_setup != 0) 2304 setup_attached_port(pi); 2305 return; 2306 } 2307 2308 /* second attach mode: iterator */ 2309 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2310 /* setup ports matching the devargs used for probing */ 2311 if (port_is_forwarding(pi)) 2312 continue; /* port was already attached before */ 2313 setup_attached_port(pi); 2314 } 2315 } 2316 2317 static void 2318 setup_attached_port(portid_t pi) 2319 { 2320 unsigned int socket_id; 2321 2322 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2323 /* if socket_id is invalid, set to the first available socket. */ 2324 if (check_socket_id(socket_id) < 0) 2325 socket_id = socket_ids[0]; 2326 reconfig(pi, socket_id); 2327 rte_eth_promiscuous_enable(pi); 2328 2329 ports_ids[nb_ports++] = pi; 2330 fwd_ports_ids[nb_fwd_ports++] = pi; 2331 nb_cfg_ports = nb_fwd_ports; 2332 ports[pi].need_setup = 0; 2333 ports[pi].port_status = RTE_PORT_STOPPED; 2334 2335 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2336 printf("Done\n"); 2337 } 2338 2339 void 2340 detach_port_device(portid_t port_id) 2341 { 2342 struct rte_device *dev; 2343 portid_t sibling; 2344 2345 printf("Removing a device...\n"); 2346 2347 dev = rte_eth_devices[port_id].device; 2348 if (dev == NULL) { 2349 printf("Device already removed\n"); 2350 return; 2351 } 2352 2353 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2354 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2355 printf("Port not stopped\n"); 2356 return; 2357 } 2358 printf("Port was not closed\n"); 2359 if (ports[port_id].flow_list) 2360 port_flow_flush(port_id); 2361 } 2362 2363 if (rte_dev_remove(dev) != 0) { 2364 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2365 return; 2366 } 2367 2368 for (sibling = 0; sibling < RTE_MAX_ETHPORTS; sibling++) { 2369 if (rte_eth_devices[sibling].device != dev) 2370 continue; 2371 /* reset mapping between old ports and removed device */ 2372 rte_eth_devices[sibling].device = NULL; 2373 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2374 /* sibling ports are forced to be closed */ 2375 ports[sibling].port_status = RTE_PORT_CLOSED; 2376 printf("Port %u is closed\n", sibling); 2377 } 2378 } 2379 2380 remove_invalid_ports(); 2381 2382 printf("Device of port %u is detached\n", port_id); 2383 printf("Now total ports is %d\n", nb_ports); 2384 printf("Done\n"); 2385 return; 2386 } 2387 2388 void 2389 pmd_test_exit(void) 2390 { 2391 struct rte_device *device; 2392 portid_t pt_id; 2393 int ret; 2394 int i; 2395 2396 if (test_done == 0) 2397 stop_packet_forwarding(); 2398 2399 if (ports != NULL) { 2400 no_link_check = 1; 2401 RTE_ETH_FOREACH_DEV(pt_id) { 2402 printf("\nStopping port %d...\n", pt_id); 2403 fflush(stdout); 2404 stop_port(pt_id); 2405 } 2406 RTE_ETH_FOREACH_DEV(pt_id) { 2407 printf("\nShutting down port %d...\n", pt_id); 2408 fflush(stdout); 2409 close_port(pt_id); 2410 2411 /* 2412 * This is a workaround to fix a virtio-user issue that 2413 * requires to call clean-up routine to remove existing 2414 * socket. 2415 * This workaround valid only for testpmd, needs a fix 2416 * valid for all applications. 2417 * TODO: Implement proper resource cleanup 2418 */ 2419 device = rte_eth_devices[pt_id].device; 2420 if (device && !strcmp(device->driver->name, "net_virtio_user")) 2421 detach_port_device(pt_id); 2422 } 2423 } 2424 2425 if (hot_plug) { 2426 ret = rte_dev_event_monitor_stop(); 2427 if (ret) { 2428 RTE_LOG(ERR, EAL, 2429 "fail to stop device event monitor."); 2430 return; 2431 } 2432 2433 ret = rte_dev_event_callback_unregister(NULL, 2434 dev_event_callback, NULL); 2435 if (ret < 0) { 2436 RTE_LOG(ERR, EAL, 2437 "fail to unregister device event callback.\n"); 2438 return; 2439 } 2440 2441 ret = rte_dev_hotplug_handle_disable(); 2442 if (ret) { 2443 RTE_LOG(ERR, EAL, 2444 "fail to disable hotplug handling.\n"); 2445 return; 2446 } 2447 } 2448 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2449 if (mempools[i]) 2450 rte_mempool_free(mempools[i]); 2451 } 2452 2453 printf("\nBye...\n"); 2454 } 2455 2456 typedef void (*cmd_func_t)(void); 2457 struct pmd_test_command { 2458 const char *cmd_name; 2459 cmd_func_t cmd_func; 2460 }; 2461 2462 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2463 2464 /* Check the link status of all ports in up to 9s, and print them finally */ 2465 static void 2466 check_all_ports_link_status(uint32_t port_mask) 2467 { 2468 #define CHECK_INTERVAL 100 /* 100ms */ 2469 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2470 portid_t portid; 2471 uint8_t count, all_ports_up, print_flag = 0; 2472 struct rte_eth_link link; 2473 2474 printf("Checking link statuses...\n"); 2475 fflush(stdout); 2476 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2477 all_ports_up = 1; 2478 RTE_ETH_FOREACH_DEV(portid) { 2479 if ((port_mask & (1 << portid)) == 0) 2480 continue; 2481 memset(&link, 0, sizeof(link)); 2482 rte_eth_link_get_nowait(portid, &link); 2483 /* print link status if flag set */ 2484 if (print_flag == 1) { 2485 if (link.link_status) 2486 printf( 2487 "Port%d Link Up. speed %u Mbps- %s\n", 2488 portid, link.link_speed, 2489 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2490 ("full-duplex") : ("half-duplex\n")); 2491 else 2492 printf("Port %d Link Down\n", portid); 2493 continue; 2494 } 2495 /* clear all_ports_up flag if any link down */ 2496 if (link.link_status == ETH_LINK_DOWN) { 2497 all_ports_up = 0; 2498 break; 2499 } 2500 } 2501 /* after finally printing all link status, get out */ 2502 if (print_flag == 1) 2503 break; 2504 2505 if (all_ports_up == 0) { 2506 fflush(stdout); 2507 rte_delay_ms(CHECK_INTERVAL); 2508 } 2509 2510 /* set the print_flag if all ports up or timeout */ 2511 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2512 print_flag = 1; 2513 } 2514 2515 if (lsc_interrupt) 2516 break; 2517 } 2518 } 2519 2520 /* 2521 * This callback is for remove a port for a device. It has limitation because 2522 * it is not for multiple port removal for a device. 2523 * TODO: the device detach invoke will plan to be removed from user side to 2524 * eal. And convert all PMDs to free port resources on ether device closing. 2525 */ 2526 static void 2527 rmv_port_callback(void *arg) 2528 { 2529 int need_to_start = 0; 2530 int org_no_link_check = no_link_check; 2531 portid_t port_id = (intptr_t)arg; 2532 2533 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2534 2535 if (!test_done && port_is_forwarding(port_id)) { 2536 need_to_start = 1; 2537 stop_packet_forwarding(); 2538 } 2539 no_link_check = 1; 2540 stop_port(port_id); 2541 no_link_check = org_no_link_check; 2542 close_port(port_id); 2543 detach_port_device(port_id); 2544 if (need_to_start) 2545 start_packet_forwarding(0); 2546 } 2547 2548 /* This function is used by the interrupt thread */ 2549 static int 2550 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2551 void *ret_param) 2552 { 2553 RTE_SET_USED(param); 2554 RTE_SET_USED(ret_param); 2555 2556 if (type >= RTE_ETH_EVENT_MAX) { 2557 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2558 port_id, __func__, type); 2559 fflush(stderr); 2560 } else if (event_print_mask & (UINT32_C(1) << type)) { 2561 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2562 eth_event_desc[type]); 2563 fflush(stdout); 2564 } 2565 2566 switch (type) { 2567 case RTE_ETH_EVENT_NEW: 2568 ports[port_id].need_setup = 1; 2569 ports[port_id].port_status = RTE_PORT_HANDLING; 2570 break; 2571 case RTE_ETH_EVENT_INTR_RMV: 2572 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2573 break; 2574 if (rte_eal_alarm_set(100000, 2575 rmv_port_callback, (void *)(intptr_t)port_id)) 2576 fprintf(stderr, "Could not set up deferred device removal\n"); 2577 break; 2578 default: 2579 break; 2580 } 2581 return 0; 2582 } 2583 2584 static int 2585 register_eth_event_callback(void) 2586 { 2587 int ret; 2588 enum rte_eth_event_type event; 2589 2590 for (event = RTE_ETH_EVENT_UNKNOWN; 2591 event < RTE_ETH_EVENT_MAX; event++) { 2592 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2593 event, 2594 eth_event_callback, 2595 NULL); 2596 if (ret != 0) { 2597 TESTPMD_LOG(ERR, "Failed to register callback for " 2598 "%s event\n", eth_event_desc[event]); 2599 return -1; 2600 } 2601 } 2602 2603 return 0; 2604 } 2605 2606 /* This function is used by the interrupt thread */ 2607 static void 2608 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2609 __rte_unused void *arg) 2610 { 2611 uint16_t port_id; 2612 int ret; 2613 2614 if (type >= RTE_DEV_EVENT_MAX) { 2615 fprintf(stderr, "%s called upon invalid event %d\n", 2616 __func__, type); 2617 fflush(stderr); 2618 } 2619 2620 switch (type) { 2621 case RTE_DEV_EVENT_REMOVE: 2622 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2623 device_name); 2624 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2625 if (ret) { 2626 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2627 device_name); 2628 return; 2629 } 2630 /* 2631 * Because the user's callback is invoked in eal interrupt 2632 * callback, the interrupt callback need to be finished before 2633 * it can be unregistered when detaching device. So finish 2634 * callback soon and use a deferred removal to detach device 2635 * is need. It is a workaround, once the device detaching be 2636 * moved into the eal in the future, the deferred removal could 2637 * be deleted. 2638 */ 2639 if (rte_eal_alarm_set(100000, 2640 rmv_port_callback, (void *)(intptr_t)port_id)) 2641 RTE_LOG(ERR, EAL, 2642 "Could not set up deferred device removal\n"); 2643 break; 2644 case RTE_DEV_EVENT_ADD: 2645 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2646 device_name); 2647 /* TODO: After finish kernel driver binding, 2648 * begin to attach port. 2649 */ 2650 break; 2651 default: 2652 break; 2653 } 2654 } 2655 2656 static int 2657 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2658 { 2659 uint16_t i; 2660 int diag; 2661 uint8_t mapping_found = 0; 2662 2663 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2664 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2665 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2666 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2667 tx_queue_stats_mappings[i].queue_id, 2668 tx_queue_stats_mappings[i].stats_counter_id); 2669 if (diag != 0) 2670 return diag; 2671 mapping_found = 1; 2672 } 2673 } 2674 if (mapping_found) 2675 port->tx_queue_stats_mapping_enabled = 1; 2676 return 0; 2677 } 2678 2679 static int 2680 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2681 { 2682 uint16_t i; 2683 int diag; 2684 uint8_t mapping_found = 0; 2685 2686 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2687 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2688 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2689 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2690 rx_queue_stats_mappings[i].queue_id, 2691 rx_queue_stats_mappings[i].stats_counter_id); 2692 if (diag != 0) 2693 return diag; 2694 mapping_found = 1; 2695 } 2696 } 2697 if (mapping_found) 2698 port->rx_queue_stats_mapping_enabled = 1; 2699 return 0; 2700 } 2701 2702 static void 2703 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2704 { 2705 int diag = 0; 2706 2707 diag = set_tx_queue_stats_mapping_registers(pi, port); 2708 if (diag != 0) { 2709 if (diag == -ENOTSUP) { 2710 port->tx_queue_stats_mapping_enabled = 0; 2711 printf("TX queue stats mapping not supported port id=%d\n", pi); 2712 } 2713 else 2714 rte_exit(EXIT_FAILURE, 2715 "set_tx_queue_stats_mapping_registers " 2716 "failed for port id=%d diag=%d\n", 2717 pi, diag); 2718 } 2719 2720 diag = set_rx_queue_stats_mapping_registers(pi, port); 2721 if (diag != 0) { 2722 if (diag == -ENOTSUP) { 2723 port->rx_queue_stats_mapping_enabled = 0; 2724 printf("RX queue stats mapping not supported port id=%d\n", pi); 2725 } 2726 else 2727 rte_exit(EXIT_FAILURE, 2728 "set_rx_queue_stats_mapping_registers " 2729 "failed for port id=%d diag=%d\n", 2730 pi, diag); 2731 } 2732 } 2733 2734 static void 2735 rxtx_port_config(struct rte_port *port) 2736 { 2737 uint16_t qid; 2738 2739 for (qid = 0; qid < nb_rxq; qid++) { 2740 port->rx_conf[qid] = port->dev_info.default_rxconf; 2741 2742 /* Check if any Rx parameters have been passed */ 2743 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 2744 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 2745 2746 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 2747 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 2748 2749 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 2750 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 2751 2752 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 2753 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 2754 2755 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 2756 port->rx_conf[qid].rx_drop_en = rx_drop_en; 2757 2758 port->nb_rx_desc[qid] = nb_rxd; 2759 } 2760 2761 for (qid = 0; qid < nb_txq; qid++) { 2762 port->tx_conf[qid] = port->dev_info.default_txconf; 2763 2764 /* Check if any Tx parameters have been passed */ 2765 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 2766 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 2767 2768 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 2769 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 2770 2771 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 2772 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 2773 2774 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 2775 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 2776 2777 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 2778 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 2779 2780 port->nb_tx_desc[qid] = nb_txd; 2781 } 2782 } 2783 2784 void 2785 init_port_config(void) 2786 { 2787 portid_t pid; 2788 struct rte_port *port; 2789 2790 RTE_ETH_FOREACH_DEV(pid) { 2791 port = &ports[pid]; 2792 port->dev_conf.fdir_conf = fdir_conf; 2793 rte_eth_dev_info_get(pid, &port->dev_info); 2794 if (nb_rxq > 1) { 2795 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2796 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 2797 rss_hf & port->dev_info.flow_type_rss_offloads; 2798 } else { 2799 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 2800 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 2801 } 2802 2803 if (port->dcb_flag == 0) { 2804 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 2805 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 2806 else 2807 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 2808 } 2809 2810 rxtx_port_config(port); 2811 2812 rte_eth_macaddr_get(pid, &port->eth_addr); 2813 2814 map_port_queue_stats_mapping_registers(pid, port); 2815 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 2816 rte_pmd_ixgbe_bypass_init(pid); 2817 #endif 2818 2819 if (lsc_interrupt && 2820 (rte_eth_devices[pid].data->dev_flags & 2821 RTE_ETH_DEV_INTR_LSC)) 2822 port->dev_conf.intr_conf.lsc = 1; 2823 if (rmv_interrupt && 2824 (rte_eth_devices[pid].data->dev_flags & 2825 RTE_ETH_DEV_INTR_RMV)) 2826 port->dev_conf.intr_conf.rmv = 1; 2827 } 2828 } 2829 2830 void set_port_slave_flag(portid_t slave_pid) 2831 { 2832 struct rte_port *port; 2833 2834 port = &ports[slave_pid]; 2835 port->slave_flag = 1; 2836 } 2837 2838 void clear_port_slave_flag(portid_t slave_pid) 2839 { 2840 struct rte_port *port; 2841 2842 port = &ports[slave_pid]; 2843 port->slave_flag = 0; 2844 } 2845 2846 uint8_t port_is_bonding_slave(portid_t slave_pid) 2847 { 2848 struct rte_port *port; 2849 2850 port = &ports[slave_pid]; 2851 if ((rte_eth_devices[slave_pid].data->dev_flags & 2852 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 2853 return 1; 2854 return 0; 2855 } 2856 2857 const uint16_t vlan_tags[] = { 2858 0, 1, 2, 3, 4, 5, 6, 7, 2859 8, 9, 10, 11, 12, 13, 14, 15, 2860 16, 17, 18, 19, 20, 21, 22, 23, 2861 24, 25, 26, 27, 28, 29, 30, 31 2862 }; 2863 2864 static int 2865 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 2866 enum dcb_mode_enable dcb_mode, 2867 enum rte_eth_nb_tcs num_tcs, 2868 uint8_t pfc_en) 2869 { 2870 uint8_t i; 2871 int32_t rc; 2872 struct rte_eth_rss_conf rss_conf; 2873 2874 /* 2875 * Builds up the correct configuration for dcb+vt based on the vlan tags array 2876 * given above, and the number of traffic classes available for use. 2877 */ 2878 if (dcb_mode == DCB_VT_ENABLED) { 2879 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 2880 ð_conf->rx_adv_conf.vmdq_dcb_conf; 2881 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 2882 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 2883 2884 /* VMDQ+DCB RX and TX configurations */ 2885 vmdq_rx_conf->enable_default_pool = 0; 2886 vmdq_rx_conf->default_pool = 0; 2887 vmdq_rx_conf->nb_queue_pools = 2888 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2889 vmdq_tx_conf->nb_queue_pools = 2890 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 2891 2892 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 2893 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 2894 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 2895 vmdq_rx_conf->pool_map[i].pools = 2896 1 << (i % vmdq_rx_conf->nb_queue_pools); 2897 } 2898 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2899 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 2900 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 2901 } 2902 2903 /* set DCB mode of RX and TX of multiple queues */ 2904 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 2905 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 2906 } else { 2907 struct rte_eth_dcb_rx_conf *rx_conf = 2908 ð_conf->rx_adv_conf.dcb_rx_conf; 2909 struct rte_eth_dcb_tx_conf *tx_conf = 2910 ð_conf->tx_adv_conf.dcb_tx_conf; 2911 2912 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 2913 if (rc != 0) 2914 return rc; 2915 2916 rx_conf->nb_tcs = num_tcs; 2917 tx_conf->nb_tcs = num_tcs; 2918 2919 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 2920 rx_conf->dcb_tc[i] = i % num_tcs; 2921 tx_conf->dcb_tc[i] = i % num_tcs; 2922 } 2923 2924 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 2925 eth_conf->rx_adv_conf.rss_conf = rss_conf; 2926 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 2927 } 2928 2929 if (pfc_en) 2930 eth_conf->dcb_capability_en = 2931 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 2932 else 2933 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 2934 2935 return 0; 2936 } 2937 2938 int 2939 init_port_dcb_config(portid_t pid, 2940 enum dcb_mode_enable dcb_mode, 2941 enum rte_eth_nb_tcs num_tcs, 2942 uint8_t pfc_en) 2943 { 2944 struct rte_eth_conf port_conf; 2945 struct rte_port *rte_port; 2946 int retval; 2947 uint16_t i; 2948 2949 rte_port = &ports[pid]; 2950 2951 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 2952 /* Enter DCB configuration status */ 2953 dcb_config = 1; 2954 2955 port_conf.rxmode = rte_port->dev_conf.rxmode; 2956 port_conf.txmode = rte_port->dev_conf.txmode; 2957 2958 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 2959 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 2960 if (retval < 0) 2961 return retval; 2962 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 2963 2964 /* re-configure the device . */ 2965 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 2966 if (retval < 0) 2967 return retval; 2968 rte_eth_dev_info_get(pid, &rte_port->dev_info); 2969 2970 /* If dev_info.vmdq_pool_base is greater than 0, 2971 * the queue id of vmdq pools is started after pf queues. 2972 */ 2973 if (dcb_mode == DCB_VT_ENABLED && 2974 rte_port->dev_info.vmdq_pool_base > 0) { 2975 printf("VMDQ_DCB multi-queue mode is nonsensical" 2976 " for port %d.", pid); 2977 return -1; 2978 } 2979 2980 /* Assume the ports in testpmd have the same dcb capability 2981 * and has the same number of rxq and txq in dcb mode 2982 */ 2983 if (dcb_mode == DCB_VT_ENABLED) { 2984 if (rte_port->dev_info.max_vfs > 0) { 2985 nb_rxq = rte_port->dev_info.nb_rx_queues; 2986 nb_txq = rte_port->dev_info.nb_tx_queues; 2987 } else { 2988 nb_rxq = rte_port->dev_info.max_rx_queues; 2989 nb_txq = rte_port->dev_info.max_tx_queues; 2990 } 2991 } else { 2992 /*if vt is disabled, use all pf queues */ 2993 if (rte_port->dev_info.vmdq_pool_base == 0) { 2994 nb_rxq = rte_port->dev_info.max_rx_queues; 2995 nb_txq = rte_port->dev_info.max_tx_queues; 2996 } else { 2997 nb_rxq = (queueid_t)num_tcs; 2998 nb_txq = (queueid_t)num_tcs; 2999 3000 } 3001 } 3002 rx_free_thresh = 64; 3003 3004 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3005 3006 rxtx_port_config(rte_port); 3007 /* VLAN filter */ 3008 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3009 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3010 rx_vft_set(pid, vlan_tags[i], 1); 3011 3012 rte_eth_macaddr_get(pid, &rte_port->eth_addr); 3013 map_port_queue_stats_mapping_registers(pid, rte_port); 3014 3015 rte_port->dcb_flag = 1; 3016 3017 return 0; 3018 } 3019 3020 static void 3021 init_port(void) 3022 { 3023 /* Configuration of Ethernet ports. */ 3024 ports = rte_zmalloc("testpmd: ports", 3025 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3026 RTE_CACHE_LINE_SIZE); 3027 if (ports == NULL) { 3028 rte_exit(EXIT_FAILURE, 3029 "rte_zmalloc(%d struct rte_port) failed\n", 3030 RTE_MAX_ETHPORTS); 3031 } 3032 3033 /* Initialize ports NUMA structures */ 3034 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3035 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3036 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3037 } 3038 3039 static void 3040 force_quit(void) 3041 { 3042 pmd_test_exit(); 3043 prompt_exit(); 3044 } 3045 3046 static void 3047 print_stats(void) 3048 { 3049 uint8_t i; 3050 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3051 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3052 3053 /* Clear screen and move to top left */ 3054 printf("%s%s", clr, top_left); 3055 3056 printf("\nPort statistics ===================================="); 3057 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3058 nic_stats_display(fwd_ports_ids[i]); 3059 3060 fflush(stdout); 3061 } 3062 3063 static void 3064 signal_handler(int signum) 3065 { 3066 if (signum == SIGINT || signum == SIGTERM) { 3067 printf("\nSignal %d received, preparing to exit...\n", 3068 signum); 3069 #ifdef RTE_LIBRTE_PDUMP 3070 /* uninitialize packet capture framework */ 3071 rte_pdump_uninit(); 3072 #endif 3073 #ifdef RTE_LIBRTE_LATENCY_STATS 3074 rte_latencystats_uninit(); 3075 #endif 3076 force_quit(); 3077 /* Set flag to indicate the force termination. */ 3078 f_quit = 1; 3079 /* exit with the expected status */ 3080 signal(signum, SIG_DFL); 3081 kill(getpid(), signum); 3082 } 3083 } 3084 3085 int 3086 main(int argc, char** argv) 3087 { 3088 int diag; 3089 portid_t port_id; 3090 uint16_t count; 3091 int ret; 3092 3093 signal(SIGINT, signal_handler); 3094 signal(SIGTERM, signal_handler); 3095 3096 diag = rte_eal_init(argc, argv); 3097 if (diag < 0) 3098 rte_panic("Cannot init EAL\n"); 3099 3100 testpmd_logtype = rte_log_register("testpmd"); 3101 if (testpmd_logtype < 0) 3102 rte_panic("Cannot register log type"); 3103 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3104 3105 ret = register_eth_event_callback(); 3106 if (ret != 0) 3107 rte_panic("Cannot register for ethdev events"); 3108 3109 #ifdef RTE_LIBRTE_PDUMP 3110 /* initialize packet capture framework */ 3111 rte_pdump_init(NULL); 3112 #endif 3113 3114 count = 0; 3115 RTE_ETH_FOREACH_DEV(port_id) { 3116 ports_ids[count] = port_id; 3117 count++; 3118 } 3119 nb_ports = (portid_t) count; 3120 if (nb_ports == 0) 3121 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3122 3123 /* allocate port structures, and init them */ 3124 init_port(); 3125 3126 set_def_fwd_config(); 3127 if (nb_lcores == 0) 3128 rte_panic("Empty set of forwarding logical cores - check the " 3129 "core mask supplied in the command parameters\n"); 3130 3131 /* Bitrate/latency stats disabled by default */ 3132 #ifdef RTE_LIBRTE_BITRATE 3133 bitrate_enabled = 0; 3134 #endif 3135 #ifdef RTE_LIBRTE_LATENCY_STATS 3136 latencystats_enabled = 0; 3137 #endif 3138 3139 /* on FreeBSD, mlockall() is disabled by default */ 3140 #ifdef RTE_EXEC_ENV_BSDAPP 3141 do_mlockall = 0; 3142 #else 3143 do_mlockall = 1; 3144 #endif 3145 3146 argc -= diag; 3147 argv += diag; 3148 if (argc > 1) 3149 launch_args_parse(argc, argv); 3150 3151 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3152 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3153 strerror(errno)); 3154 } 3155 3156 if (tx_first && interactive) 3157 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3158 "interactive mode.\n"); 3159 3160 if (tx_first && lsc_interrupt) { 3161 printf("Warning: lsc_interrupt needs to be off when " 3162 " using tx_first. Disabling.\n"); 3163 lsc_interrupt = 0; 3164 } 3165 3166 if (!nb_rxq && !nb_txq) 3167 printf("Warning: Either rx or tx queues should be non-zero\n"); 3168 3169 if (nb_rxq > 1 && nb_rxq > nb_txq) 3170 printf("Warning: nb_rxq=%d enables RSS configuration, " 3171 "but nb_txq=%d will prevent to fully test it.\n", 3172 nb_rxq, nb_txq); 3173 3174 init_config(); 3175 3176 if (hot_plug) { 3177 ret = rte_dev_hotplug_handle_enable(); 3178 if (ret) { 3179 RTE_LOG(ERR, EAL, 3180 "fail to enable hotplug handling."); 3181 return -1; 3182 } 3183 3184 ret = rte_dev_event_monitor_start(); 3185 if (ret) { 3186 RTE_LOG(ERR, EAL, 3187 "fail to start device event monitoring."); 3188 return -1; 3189 } 3190 3191 ret = rte_dev_event_callback_register(NULL, 3192 dev_event_callback, NULL); 3193 if (ret) { 3194 RTE_LOG(ERR, EAL, 3195 "fail to register device event callback\n"); 3196 return -1; 3197 } 3198 } 3199 3200 if (start_port(RTE_PORT_ALL) != 0) 3201 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3202 3203 /* set all ports to promiscuous mode by default */ 3204 RTE_ETH_FOREACH_DEV(port_id) 3205 rte_eth_promiscuous_enable(port_id); 3206 3207 /* Init metrics library */ 3208 rte_metrics_init(rte_socket_id()); 3209 3210 #ifdef RTE_LIBRTE_LATENCY_STATS 3211 if (latencystats_enabled != 0) { 3212 int ret = rte_latencystats_init(1, NULL); 3213 if (ret) 3214 printf("Warning: latencystats init()" 3215 " returned error %d\n", ret); 3216 printf("Latencystats running on lcore %d\n", 3217 latencystats_lcore_id); 3218 } 3219 #endif 3220 3221 /* Setup bitrate stats */ 3222 #ifdef RTE_LIBRTE_BITRATE 3223 if (bitrate_enabled != 0) { 3224 bitrate_data = rte_stats_bitrate_create(); 3225 if (bitrate_data == NULL) 3226 rte_exit(EXIT_FAILURE, 3227 "Could not allocate bitrate data.\n"); 3228 rte_stats_bitrate_reg(bitrate_data); 3229 } 3230 #endif 3231 3232 #ifdef RTE_LIBRTE_CMDLINE 3233 if (strlen(cmdline_filename) != 0) 3234 cmdline_read_from_file(cmdline_filename); 3235 3236 if (interactive == 1) { 3237 if (auto_start) { 3238 printf("Start automatic packet forwarding\n"); 3239 start_packet_forwarding(0); 3240 } 3241 prompt(); 3242 pmd_test_exit(); 3243 } else 3244 #endif 3245 { 3246 char c; 3247 int rc; 3248 3249 f_quit = 0; 3250 3251 printf("No commandline core given, start packet forwarding\n"); 3252 start_packet_forwarding(tx_first); 3253 if (stats_period != 0) { 3254 uint64_t prev_time = 0, cur_time, diff_time = 0; 3255 uint64_t timer_period; 3256 3257 /* Convert to number of cycles */ 3258 timer_period = stats_period * rte_get_timer_hz(); 3259 3260 while (f_quit == 0) { 3261 cur_time = rte_get_timer_cycles(); 3262 diff_time += cur_time - prev_time; 3263 3264 if (diff_time >= timer_period) { 3265 print_stats(); 3266 /* Reset the timer */ 3267 diff_time = 0; 3268 } 3269 /* Sleep to avoid unnecessary checks */ 3270 prev_time = cur_time; 3271 sleep(1); 3272 } 3273 } 3274 3275 printf("Press enter to exit\n"); 3276 rc = read(0, &c, 1); 3277 pmd_test_exit(); 3278 if (rc < 0) 3279 return 1; 3280 } 3281 3282 return 0; 3283 } 3284