1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M 82 83 uint16_t verbose_level = 0; /**< Silent by default. */ 84 int testpmd_logtype; /**< Log type for testpmd logs */ 85 86 /* use master core for command line ? */ 87 uint8_t interactive = 0; 88 uint8_t auto_start = 0; 89 uint8_t tx_first; 90 char cmdline_filename[PATH_MAX] = {0}; 91 92 /* 93 * NUMA support configuration. 94 * When set, the NUMA support attempts to dispatch the allocation of the 95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 96 * probed ports among the CPU sockets 0 and 1. 97 * Otherwise, all memory is allocated from CPU socket 0. 98 */ 99 uint8_t numa_support = 1; /**< numa enabled by default */ 100 101 /* 102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 103 * not configured. 104 */ 105 uint8_t socket_num = UMA_NO_CONFIG; 106 107 /* 108 * Select mempool allocation type: 109 * - native: use regular DPDK memory 110 * - anon: use regular DPDK memory to create mempool, but populate using 111 * anonymous memory (may not be IOVA-contiguous) 112 * - xmem: use externally allocated hugepage memory 113 */ 114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 115 116 /* 117 * Store specified sockets on which memory pool to be used by ports 118 * is allocated. 119 */ 120 uint8_t port_numa[RTE_MAX_ETHPORTS]; 121 122 /* 123 * Store specified sockets on which RX ring to be used by ports 124 * is allocated. 125 */ 126 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 127 128 /* 129 * Store specified sockets on which TX ring to be used by ports 130 * is allocated. 131 */ 132 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 133 134 /* 135 * Record the Ethernet address of peer target ports to which packets are 136 * forwarded. 137 * Must be instantiated with the ethernet addresses of peer traffic generator 138 * ports. 139 */ 140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 141 portid_t nb_peer_eth_addrs = 0; 142 143 /* 144 * Probed Target Environment. 145 */ 146 struct rte_port *ports; /**< For all probed ethernet ports. */ 147 portid_t nb_ports; /**< Number of probed ethernet ports. */ 148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 150 151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 152 153 /* 154 * Test Forwarding Configuration. 155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 157 */ 158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 160 portid_t nb_cfg_ports; /**< Number of configured ports. */ 161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 162 163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 165 166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 168 169 /* 170 * Forwarding engines. 171 */ 172 struct fwd_engine * fwd_engines[] = { 173 &io_fwd_engine, 174 &mac_fwd_engine, 175 &mac_swap_engine, 176 &flow_gen_engine, 177 &rx_only_engine, 178 &tx_only_engine, 179 &csum_fwd_engine, 180 &icmp_echo_engine, 181 &noisy_vnf_engine, 182 #if defined RTE_LIBRTE_PMD_SOFTNIC 183 &softnic_fwd_engine, 184 #endif 185 #ifdef RTE_LIBRTE_IEEE1588 186 &ieee1588_fwd_engine, 187 #endif 188 NULL, 189 }; 190 191 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 192 uint16_t mempool_flags; 193 194 struct fwd_config cur_fwd_config; 195 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 196 uint32_t retry_enabled; 197 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 198 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 199 200 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 201 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 202 * specified on command-line. */ 203 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 204 205 /* 206 * In container, it cannot terminate the process which running with 'stats-period' 207 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 208 */ 209 uint8_t f_quit; 210 211 /* 212 * Configuration of packet segments used by the "txonly" processing engine. 213 */ 214 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 215 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 216 TXONLY_DEF_PACKET_LEN, 217 }; 218 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 219 220 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 221 /**< Split policy for packets to TX. */ 222 223 uint8_t txonly_multi_flow; 224 /**< Whether multiple flows are generated in TXONLY mode. */ 225 226 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 227 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 228 229 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 230 uint8_t dcb_config = 0; 231 232 /* Whether the dcb is in testing status */ 233 uint8_t dcb_test = 0; 234 235 /* 236 * Configurable number of RX/TX queues. 237 */ 238 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 239 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 240 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 241 242 /* 243 * Configurable number of RX/TX ring descriptors. 244 * Defaults are supplied by drivers via ethdev. 245 */ 246 #define RTE_TEST_RX_DESC_DEFAULT 0 247 #define RTE_TEST_TX_DESC_DEFAULT 0 248 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 249 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 250 251 #define RTE_PMD_PARAM_UNSET -1 252 /* 253 * Configurable values of RX and TX ring threshold registers. 254 */ 255 256 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 258 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 259 260 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 262 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 263 264 /* 265 * Configurable value of RX free threshold. 266 */ 267 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 268 269 /* 270 * Configurable value of RX drop enable. 271 */ 272 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 273 274 /* 275 * Configurable value of TX free threshold. 276 */ 277 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 278 279 /* 280 * Configurable value of TX RS bit threshold. 281 */ 282 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 283 284 /* 285 * Configurable value of buffered packets before sending. 286 */ 287 uint16_t noisy_tx_sw_bufsz; 288 289 /* 290 * Configurable value of packet buffer timeout. 291 */ 292 uint16_t noisy_tx_sw_buf_flush_time; 293 294 /* 295 * Configurable value for size of VNF internal memory area 296 * used for simulating noisy neighbour behaviour 297 */ 298 uint64_t noisy_lkup_mem_sz; 299 300 /* 301 * Configurable value of number of random writes done in 302 * VNF simulation memory area. 303 */ 304 uint64_t noisy_lkup_num_writes; 305 306 /* 307 * Configurable value of number of random reads done in 308 * VNF simulation memory area. 309 */ 310 uint64_t noisy_lkup_num_reads; 311 312 /* 313 * Configurable value of number of random reads/writes done in 314 * VNF simulation memory area. 315 */ 316 uint64_t noisy_lkup_num_reads_writes; 317 318 /* 319 * Receive Side Scaling (RSS) configuration. 320 */ 321 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 322 323 /* 324 * Port topology configuration 325 */ 326 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 327 328 /* 329 * Avoids to flush all the RX streams before starts forwarding. 330 */ 331 uint8_t no_flush_rx = 0; /* flush by default */ 332 333 /* 334 * Flow API isolated mode. 335 */ 336 uint8_t flow_isolate_all; 337 338 /* 339 * Avoids to check link status when starting/stopping a port. 340 */ 341 uint8_t no_link_check = 0; /* check by default */ 342 343 /* 344 * Don't automatically start all ports in interactive mode. 345 */ 346 uint8_t no_device_start = 0; 347 348 /* 349 * Enable link status change notification 350 */ 351 uint8_t lsc_interrupt = 1; /* enabled by default */ 352 353 /* 354 * Enable device removal notification. 355 */ 356 uint8_t rmv_interrupt = 1; /* enabled by default */ 357 358 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 359 360 /* After attach, port setup is called on event or by iterator */ 361 bool setup_on_probe_event = true; 362 363 /* Clear ptypes on port initialization. */ 364 uint8_t clear_ptypes = true; 365 366 /* Pretty printing of ethdev events */ 367 static const char * const eth_event_desc[] = { 368 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 369 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 370 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 371 [RTE_ETH_EVENT_INTR_RESET] = "reset", 372 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 373 [RTE_ETH_EVENT_IPSEC] = "IPsec", 374 [RTE_ETH_EVENT_MACSEC] = "MACsec", 375 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 376 [RTE_ETH_EVENT_NEW] = "device probed", 377 [RTE_ETH_EVENT_DESTROY] = "device released", 378 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged", 379 [RTE_ETH_EVENT_MAX] = NULL, 380 }; 381 382 /* 383 * Display or mask ether events 384 * Default to all events except VF_MBOX 385 */ 386 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 387 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 388 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 389 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 390 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 391 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 392 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) | 393 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED); 394 /* 395 * Decide if all memory are locked for performance. 396 */ 397 int do_mlockall = 0; 398 399 /* 400 * NIC bypass mode configuration options. 401 */ 402 403 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 404 /* The NIC bypass watchdog timeout. */ 405 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 406 #endif 407 408 409 #ifdef RTE_LIBRTE_LATENCY_STATS 410 411 /* 412 * Set when latency stats is enabled in the commandline 413 */ 414 uint8_t latencystats_enabled; 415 416 /* 417 * Lcore ID to serive latency statistics. 418 */ 419 lcoreid_t latencystats_lcore_id = -1; 420 421 #endif 422 423 /* 424 * Ethernet device configuration. 425 */ 426 struct rte_eth_rxmode rx_mode = { 427 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 428 /**< Default maximum frame length. */ 429 }; 430 431 struct rte_eth_txmode tx_mode = { 432 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 433 }; 434 435 struct rte_fdir_conf fdir_conf = { 436 .mode = RTE_FDIR_MODE_NONE, 437 .pballoc = RTE_FDIR_PBALLOC_64K, 438 .status = RTE_FDIR_REPORT_STATUS, 439 .mask = { 440 .vlan_tci_mask = 0xFFEF, 441 .ipv4_mask = { 442 .src_ip = 0xFFFFFFFF, 443 .dst_ip = 0xFFFFFFFF, 444 }, 445 .ipv6_mask = { 446 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 447 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 448 }, 449 .src_port_mask = 0xFFFF, 450 .dst_port_mask = 0xFFFF, 451 .mac_addr_byte_mask = 0xFF, 452 .tunnel_type_mask = 1, 453 .tunnel_id_mask = 0xFFFFFFFF, 454 }, 455 .drop_queue = 127, 456 }; 457 458 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 459 460 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 461 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 462 463 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 464 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 465 466 uint16_t nb_tx_queue_stats_mappings = 0; 467 uint16_t nb_rx_queue_stats_mappings = 0; 468 469 /* 470 * Display zero values by default for xstats 471 */ 472 uint8_t xstats_hide_zero; 473 474 unsigned int num_sockets = 0; 475 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 476 477 #ifdef RTE_LIBRTE_BITRATE 478 /* Bitrate statistics */ 479 struct rte_stats_bitrates *bitrate_data; 480 lcoreid_t bitrate_lcore_id; 481 uint8_t bitrate_enabled; 482 #endif 483 484 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 485 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 486 487 /* 488 * hexadecimal bitmask of RX mq mode can be enabled. 489 */ 490 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS; 491 492 /* Forward function declarations */ 493 static void setup_attached_port(portid_t pi); 494 static void map_port_queue_stats_mapping_registers(portid_t pi, 495 struct rte_port *port); 496 static void check_all_ports_link_status(uint32_t port_mask); 497 static int eth_event_callback(portid_t port_id, 498 enum rte_eth_event_type type, 499 void *param, void *ret_param); 500 static void dev_event_callback(const char *device_name, 501 enum rte_dev_event_type type, 502 void *param); 503 504 /* 505 * Check if all the ports are started. 506 * If yes, return positive value. If not, return zero. 507 */ 508 static int all_ports_started(void); 509 510 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 511 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 512 513 /* Holds the registered mbuf dynamic flags names. */ 514 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 515 516 /* 517 * Helper function to check if socket is already discovered. 518 * If yes, return positive value. If not, return zero. 519 */ 520 int 521 new_socket_id(unsigned int socket_id) 522 { 523 unsigned int i; 524 525 for (i = 0; i < num_sockets; i++) { 526 if (socket_ids[i] == socket_id) 527 return 0; 528 } 529 return 1; 530 } 531 532 /* 533 * Setup default configuration. 534 */ 535 static void 536 set_default_fwd_lcores_config(void) 537 { 538 unsigned int i; 539 unsigned int nb_lc; 540 unsigned int sock_num; 541 542 nb_lc = 0; 543 for (i = 0; i < RTE_MAX_LCORE; i++) { 544 if (!rte_lcore_is_enabled(i)) 545 continue; 546 sock_num = rte_lcore_to_socket_id(i); 547 if (new_socket_id(sock_num)) { 548 if (num_sockets >= RTE_MAX_NUMA_NODES) { 549 rte_exit(EXIT_FAILURE, 550 "Total sockets greater than %u\n", 551 RTE_MAX_NUMA_NODES); 552 } 553 socket_ids[num_sockets++] = sock_num; 554 } 555 if (i == rte_get_master_lcore()) 556 continue; 557 fwd_lcores_cpuids[nb_lc++] = i; 558 } 559 nb_lcores = (lcoreid_t) nb_lc; 560 nb_cfg_lcores = nb_lcores; 561 nb_fwd_lcores = 1; 562 } 563 564 static void 565 set_def_peer_eth_addrs(void) 566 { 567 portid_t i; 568 569 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 570 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 571 peer_eth_addrs[i].addr_bytes[5] = i; 572 } 573 } 574 575 static void 576 set_default_fwd_ports_config(void) 577 { 578 portid_t pt_id; 579 int i = 0; 580 581 RTE_ETH_FOREACH_DEV(pt_id) { 582 fwd_ports_ids[i++] = pt_id; 583 584 /* Update sockets info according to the attached device */ 585 int socket_id = rte_eth_dev_socket_id(pt_id); 586 if (socket_id >= 0 && new_socket_id(socket_id)) { 587 if (num_sockets >= RTE_MAX_NUMA_NODES) { 588 rte_exit(EXIT_FAILURE, 589 "Total sockets greater than %u\n", 590 RTE_MAX_NUMA_NODES); 591 } 592 socket_ids[num_sockets++] = socket_id; 593 } 594 } 595 596 nb_cfg_ports = nb_ports; 597 nb_fwd_ports = nb_ports; 598 } 599 600 void 601 set_def_fwd_config(void) 602 { 603 set_default_fwd_lcores_config(); 604 set_def_peer_eth_addrs(); 605 set_default_fwd_ports_config(); 606 } 607 608 /* extremely pessimistic estimation of memory required to create a mempool */ 609 static int 610 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 611 { 612 unsigned int n_pages, mbuf_per_pg, leftover; 613 uint64_t total_mem, mbuf_mem, obj_sz; 614 615 /* there is no good way to predict how much space the mempool will 616 * occupy because it will allocate chunks on the fly, and some of those 617 * will come from default DPDK memory while some will come from our 618 * external memory, so just assume 128MB will be enough for everyone. 619 */ 620 uint64_t hdr_mem = 128 << 20; 621 622 /* account for possible non-contiguousness */ 623 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 624 if (obj_sz > pgsz) { 625 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 626 return -1; 627 } 628 629 mbuf_per_pg = pgsz / obj_sz; 630 leftover = (nb_mbufs % mbuf_per_pg) > 0; 631 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 632 633 mbuf_mem = n_pages * pgsz; 634 635 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 636 637 if (total_mem > SIZE_MAX) { 638 TESTPMD_LOG(ERR, "Memory size too big\n"); 639 return -1; 640 } 641 *out = (size_t)total_mem; 642 643 return 0; 644 } 645 646 static int 647 pagesz_flags(uint64_t page_sz) 648 { 649 /* as per mmap() manpage, all page sizes are log2 of page size 650 * shifted by MAP_HUGE_SHIFT 651 */ 652 int log2 = rte_log2_u64(page_sz); 653 654 return (log2 << HUGE_SHIFT); 655 } 656 657 static void * 658 alloc_mem(size_t memsz, size_t pgsz, bool huge) 659 { 660 void *addr; 661 int flags; 662 663 /* allocate anonymous hugepages */ 664 flags = MAP_ANONYMOUS | MAP_PRIVATE; 665 if (huge) 666 flags |= HUGE_FLAG | pagesz_flags(pgsz); 667 668 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 669 if (addr == MAP_FAILED) 670 return NULL; 671 672 return addr; 673 } 674 675 struct extmem_param { 676 void *addr; 677 size_t len; 678 size_t pgsz; 679 rte_iova_t *iova_table; 680 unsigned int iova_table_len; 681 }; 682 683 static int 684 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 685 bool huge) 686 { 687 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 688 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 689 unsigned int cur_page, n_pages, pgsz_idx; 690 size_t mem_sz, cur_pgsz; 691 rte_iova_t *iovas = NULL; 692 void *addr; 693 int ret; 694 695 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 696 /* skip anything that is too big */ 697 if (pgsizes[pgsz_idx] > SIZE_MAX) 698 continue; 699 700 cur_pgsz = pgsizes[pgsz_idx]; 701 702 /* if we were told not to allocate hugepages, override */ 703 if (!huge) 704 cur_pgsz = sysconf(_SC_PAGESIZE); 705 706 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 707 if (ret < 0) { 708 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 709 return -1; 710 } 711 712 /* allocate our memory */ 713 addr = alloc_mem(mem_sz, cur_pgsz, huge); 714 715 /* if we couldn't allocate memory with a specified page size, 716 * that doesn't mean we can't do it with other page sizes, so 717 * try another one. 718 */ 719 if (addr == NULL) 720 continue; 721 722 /* store IOVA addresses for every page in this memory area */ 723 n_pages = mem_sz / cur_pgsz; 724 725 iovas = malloc(sizeof(*iovas) * n_pages); 726 727 if (iovas == NULL) { 728 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 729 goto fail; 730 } 731 /* lock memory if it's not huge pages */ 732 if (!huge) 733 mlock(addr, mem_sz); 734 735 /* populate IOVA addresses */ 736 for (cur_page = 0; cur_page < n_pages; cur_page++) { 737 rte_iova_t iova; 738 size_t offset; 739 void *cur; 740 741 offset = cur_pgsz * cur_page; 742 cur = RTE_PTR_ADD(addr, offset); 743 744 /* touch the page before getting its IOVA */ 745 *(volatile char *)cur = 0; 746 747 iova = rte_mem_virt2iova(cur); 748 749 iovas[cur_page] = iova; 750 } 751 752 break; 753 } 754 /* if we couldn't allocate anything */ 755 if (iovas == NULL) 756 return -1; 757 758 param->addr = addr; 759 param->len = mem_sz; 760 param->pgsz = cur_pgsz; 761 param->iova_table = iovas; 762 param->iova_table_len = n_pages; 763 764 return 0; 765 fail: 766 if (iovas) 767 free(iovas); 768 if (addr) 769 munmap(addr, mem_sz); 770 771 return -1; 772 } 773 774 static int 775 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 776 { 777 struct extmem_param param; 778 int socket_id, ret; 779 780 memset(¶m, 0, sizeof(param)); 781 782 /* check if our heap exists */ 783 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 784 if (socket_id < 0) { 785 /* create our heap */ 786 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 787 if (ret < 0) { 788 TESTPMD_LOG(ERR, "Cannot create heap\n"); 789 return -1; 790 } 791 } 792 793 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 794 if (ret < 0) { 795 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 796 return -1; 797 } 798 799 /* we now have a valid memory area, so add it to heap */ 800 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 801 param.addr, param.len, param.iova_table, 802 param.iova_table_len, param.pgsz); 803 804 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 805 806 /* not needed any more */ 807 free(param.iova_table); 808 809 if (ret < 0) { 810 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 811 munmap(param.addr, param.len); 812 return -1; 813 } 814 815 /* success */ 816 817 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 818 param.len >> 20); 819 820 return 0; 821 } 822 static void 823 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 824 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 825 { 826 uint16_t pid = 0; 827 int ret; 828 829 RTE_ETH_FOREACH_DEV(pid) { 830 struct rte_eth_dev *dev = 831 &rte_eth_devices[pid]; 832 833 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 834 memhdr->len); 835 if (ret) { 836 TESTPMD_LOG(DEBUG, 837 "unable to DMA unmap addr 0x%p " 838 "for device %s\n", 839 memhdr->addr, dev->data->name); 840 } 841 } 842 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 843 if (ret) { 844 TESTPMD_LOG(DEBUG, 845 "unable to un-register addr 0x%p\n", memhdr->addr); 846 } 847 } 848 849 static void 850 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 851 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 852 { 853 uint16_t pid = 0; 854 size_t page_size = sysconf(_SC_PAGESIZE); 855 int ret; 856 857 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 858 page_size); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to register addr 0x%p\n", memhdr->addr); 862 return; 863 } 864 RTE_ETH_FOREACH_DEV(pid) { 865 struct rte_eth_dev *dev = 866 &rte_eth_devices[pid]; 867 868 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 869 memhdr->len); 870 if (ret) { 871 TESTPMD_LOG(DEBUG, 872 "unable to DMA map addr 0x%p " 873 "for device %s\n", 874 memhdr->addr, dev->data->name); 875 } 876 } 877 } 878 879 static unsigned int 880 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id, 881 char *pool_name, struct rte_pktmbuf_extmem **ext_mem) 882 { 883 struct rte_pktmbuf_extmem *xmem; 884 unsigned int ext_num, zone_num, elt_num; 885 uint16_t elt_size; 886 887 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE); 888 elt_num = EXTBUF_ZONE_SIZE / elt_size; 889 zone_num = (nb_mbufs + elt_num - 1) / elt_num; 890 891 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num); 892 if (xmem == NULL) { 893 TESTPMD_LOG(ERR, "Cannot allocate memory for " 894 "external buffer descriptors\n"); 895 *ext_mem = NULL; 896 return 0; 897 } 898 for (ext_num = 0; ext_num < zone_num; ext_num++) { 899 struct rte_pktmbuf_extmem *xseg = xmem + ext_num; 900 const struct rte_memzone *mz; 901 char mz_name[RTE_MEMZONE_NAMESIZE]; 902 int ret; 903 904 ret = snprintf(mz_name, sizeof(mz_name), 905 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num); 906 if (ret < 0 || ret >= (int)sizeof(mz_name)) { 907 errno = ENAMETOOLONG; 908 ext_num = 0; 909 break; 910 } 911 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE, 912 socket_id, 913 RTE_MEMZONE_IOVA_CONTIG | 914 RTE_MEMZONE_1GB | 915 RTE_MEMZONE_SIZE_HINT_ONLY, 916 EXTBUF_ZONE_SIZE); 917 if (mz == NULL) { 918 /* 919 * The caller exits on external buffer creation 920 * error, so there is no need to free memzones. 921 */ 922 errno = ENOMEM; 923 ext_num = 0; 924 break; 925 } 926 xseg->buf_ptr = mz->addr; 927 xseg->buf_iova = mz->iova; 928 xseg->buf_len = EXTBUF_ZONE_SIZE; 929 xseg->elt_size = elt_size; 930 } 931 if (ext_num == 0 && xmem != NULL) { 932 free(xmem); 933 xmem = NULL; 934 } 935 *ext_mem = xmem; 936 return ext_num; 937 } 938 939 /* 940 * Configuration initialisation done once at init time. 941 */ 942 static struct rte_mempool * 943 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 944 unsigned int socket_id) 945 { 946 char pool_name[RTE_MEMPOOL_NAMESIZE]; 947 struct rte_mempool *rte_mp = NULL; 948 uint32_t mb_size; 949 950 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 951 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 952 953 TESTPMD_LOG(INFO, 954 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 955 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 956 957 switch (mp_alloc_type) { 958 case MP_ALLOC_NATIVE: 959 { 960 /* wrapper to rte_mempool_create() */ 961 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 962 rte_mbuf_best_mempool_ops()); 963 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 964 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 965 break; 966 } 967 case MP_ALLOC_ANON: 968 { 969 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 970 mb_size, (unsigned int) mb_mempool_cache, 971 sizeof(struct rte_pktmbuf_pool_private), 972 socket_id, mempool_flags); 973 if (rte_mp == NULL) 974 goto err; 975 976 if (rte_mempool_populate_anon(rte_mp) == 0) { 977 rte_mempool_free(rte_mp); 978 rte_mp = NULL; 979 goto err; 980 } 981 rte_pktmbuf_pool_init(rte_mp, NULL); 982 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 983 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 984 break; 985 } 986 case MP_ALLOC_XMEM: 987 case MP_ALLOC_XMEM_HUGE: 988 { 989 int heap_socket; 990 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 991 992 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 993 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 994 995 heap_socket = 996 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 997 if (heap_socket < 0) 998 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 999 1000 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1001 rte_mbuf_best_mempool_ops()); 1002 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 1003 mb_mempool_cache, 0, mbuf_seg_size, 1004 heap_socket); 1005 break; 1006 } 1007 case MP_ALLOC_XBUF: 1008 { 1009 struct rte_pktmbuf_extmem *ext_mem; 1010 unsigned int ext_num; 1011 1012 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size, 1013 socket_id, pool_name, &ext_mem); 1014 if (ext_num == 0) 1015 rte_exit(EXIT_FAILURE, 1016 "Can't create pinned data buffers\n"); 1017 1018 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 1019 rte_mbuf_best_mempool_ops()); 1020 rte_mp = rte_pktmbuf_pool_create_extbuf 1021 (pool_name, nb_mbuf, mb_mempool_cache, 1022 0, mbuf_seg_size, socket_id, 1023 ext_mem, ext_num); 1024 free(ext_mem); 1025 break; 1026 } 1027 default: 1028 { 1029 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 1030 } 1031 } 1032 1033 err: 1034 if (rte_mp == NULL) { 1035 rte_exit(EXIT_FAILURE, 1036 "Creation of mbuf pool for socket %u failed: %s\n", 1037 socket_id, rte_strerror(rte_errno)); 1038 } else if (verbose_level > 0) { 1039 rte_mempool_dump(stdout, rte_mp); 1040 } 1041 return rte_mp; 1042 } 1043 1044 /* 1045 * Check given socket id is valid or not with NUMA mode, 1046 * if valid, return 0, else return -1 1047 */ 1048 static int 1049 check_socket_id(const unsigned int socket_id) 1050 { 1051 static int warning_once = 0; 1052 1053 if (new_socket_id(socket_id)) { 1054 if (!warning_once && numa_support) 1055 printf("Warning: NUMA should be configured manually by" 1056 " using --port-numa-config and" 1057 " --ring-numa-config parameters along with" 1058 " --numa.\n"); 1059 warning_once = 1; 1060 return -1; 1061 } 1062 return 0; 1063 } 1064 1065 /* 1066 * Get the allowed maximum number of RX queues. 1067 * *pid return the port id which has minimal value of 1068 * max_rx_queues in all ports. 1069 */ 1070 queueid_t 1071 get_allowed_max_nb_rxq(portid_t *pid) 1072 { 1073 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 1074 bool max_rxq_valid = false; 1075 portid_t pi; 1076 struct rte_eth_dev_info dev_info; 1077 1078 RTE_ETH_FOREACH_DEV(pi) { 1079 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1080 continue; 1081 1082 max_rxq_valid = true; 1083 if (dev_info.max_rx_queues < allowed_max_rxq) { 1084 allowed_max_rxq = dev_info.max_rx_queues; 1085 *pid = pi; 1086 } 1087 } 1088 return max_rxq_valid ? allowed_max_rxq : 0; 1089 } 1090 1091 /* 1092 * Check input rxq is valid or not. 1093 * If input rxq is not greater than any of maximum number 1094 * of RX queues of all ports, it is valid. 1095 * if valid, return 0, else return -1 1096 */ 1097 int 1098 check_nb_rxq(queueid_t rxq) 1099 { 1100 queueid_t allowed_max_rxq; 1101 portid_t pid = 0; 1102 1103 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1104 if (rxq > allowed_max_rxq) { 1105 printf("Fail: input rxq (%u) can't be greater " 1106 "than max_rx_queues (%u) of port %u\n", 1107 rxq, 1108 allowed_max_rxq, 1109 pid); 1110 return -1; 1111 } 1112 return 0; 1113 } 1114 1115 /* 1116 * Get the allowed maximum number of TX queues. 1117 * *pid return the port id which has minimal value of 1118 * max_tx_queues in all ports. 1119 */ 1120 queueid_t 1121 get_allowed_max_nb_txq(portid_t *pid) 1122 { 1123 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1124 bool max_txq_valid = false; 1125 portid_t pi; 1126 struct rte_eth_dev_info dev_info; 1127 1128 RTE_ETH_FOREACH_DEV(pi) { 1129 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1130 continue; 1131 1132 max_txq_valid = true; 1133 if (dev_info.max_tx_queues < allowed_max_txq) { 1134 allowed_max_txq = dev_info.max_tx_queues; 1135 *pid = pi; 1136 } 1137 } 1138 return max_txq_valid ? allowed_max_txq : 0; 1139 } 1140 1141 /* 1142 * Check input txq is valid or not. 1143 * If input txq is not greater than any of maximum number 1144 * of TX queues of all ports, it is valid. 1145 * if valid, return 0, else return -1 1146 */ 1147 int 1148 check_nb_txq(queueid_t txq) 1149 { 1150 queueid_t allowed_max_txq; 1151 portid_t pid = 0; 1152 1153 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1154 if (txq > allowed_max_txq) { 1155 printf("Fail: input txq (%u) can't be greater " 1156 "than max_tx_queues (%u) of port %u\n", 1157 txq, 1158 allowed_max_txq, 1159 pid); 1160 return -1; 1161 } 1162 return 0; 1163 } 1164 1165 /* 1166 * Get the allowed maximum number of RXDs of every rx queue. 1167 * *pid return the port id which has minimal value of 1168 * max_rxd in all queues of all ports. 1169 */ 1170 static uint16_t 1171 get_allowed_max_nb_rxd(portid_t *pid) 1172 { 1173 uint16_t allowed_max_rxd = UINT16_MAX; 1174 portid_t pi; 1175 struct rte_eth_dev_info dev_info; 1176 1177 RTE_ETH_FOREACH_DEV(pi) { 1178 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1179 continue; 1180 1181 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) { 1182 allowed_max_rxd = dev_info.rx_desc_lim.nb_max; 1183 *pid = pi; 1184 } 1185 } 1186 return allowed_max_rxd; 1187 } 1188 1189 /* 1190 * Get the allowed minimal number of RXDs of every rx queue. 1191 * *pid return the port id which has minimal value of 1192 * min_rxd in all queues of all ports. 1193 */ 1194 static uint16_t 1195 get_allowed_min_nb_rxd(portid_t *pid) 1196 { 1197 uint16_t allowed_min_rxd = 0; 1198 portid_t pi; 1199 struct rte_eth_dev_info dev_info; 1200 1201 RTE_ETH_FOREACH_DEV(pi) { 1202 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1203 continue; 1204 1205 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) { 1206 allowed_min_rxd = dev_info.rx_desc_lim.nb_min; 1207 *pid = pi; 1208 } 1209 } 1210 1211 return allowed_min_rxd; 1212 } 1213 1214 /* 1215 * Check input rxd is valid or not. 1216 * If input rxd is not greater than any of maximum number 1217 * of RXDs of every Rx queues and is not less than any of 1218 * minimal number of RXDs of every Rx queues, it is valid. 1219 * if valid, return 0, else return -1 1220 */ 1221 int 1222 check_nb_rxd(queueid_t rxd) 1223 { 1224 uint16_t allowed_max_rxd; 1225 uint16_t allowed_min_rxd; 1226 portid_t pid = 0; 1227 1228 allowed_max_rxd = get_allowed_max_nb_rxd(&pid); 1229 if (rxd > allowed_max_rxd) { 1230 printf("Fail: input rxd (%u) can't be greater " 1231 "than max_rxds (%u) of port %u\n", 1232 rxd, 1233 allowed_max_rxd, 1234 pid); 1235 return -1; 1236 } 1237 1238 allowed_min_rxd = get_allowed_min_nb_rxd(&pid); 1239 if (rxd < allowed_min_rxd) { 1240 printf("Fail: input rxd (%u) can't be less " 1241 "than min_rxds (%u) of port %u\n", 1242 rxd, 1243 allowed_min_rxd, 1244 pid); 1245 return -1; 1246 } 1247 1248 return 0; 1249 } 1250 1251 /* 1252 * Get the allowed maximum number of TXDs of every rx queues. 1253 * *pid return the port id which has minimal value of 1254 * max_txd in every tx queue. 1255 */ 1256 static uint16_t 1257 get_allowed_max_nb_txd(portid_t *pid) 1258 { 1259 uint16_t allowed_max_txd = UINT16_MAX; 1260 portid_t pi; 1261 struct rte_eth_dev_info dev_info; 1262 1263 RTE_ETH_FOREACH_DEV(pi) { 1264 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1265 continue; 1266 1267 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) { 1268 allowed_max_txd = dev_info.tx_desc_lim.nb_max; 1269 *pid = pi; 1270 } 1271 } 1272 return allowed_max_txd; 1273 } 1274 1275 /* 1276 * Get the allowed maximum number of TXDs of every tx queues. 1277 * *pid return the port id which has minimal value of 1278 * min_txd in every tx queue. 1279 */ 1280 static uint16_t 1281 get_allowed_min_nb_txd(portid_t *pid) 1282 { 1283 uint16_t allowed_min_txd = 0; 1284 portid_t pi; 1285 struct rte_eth_dev_info dev_info; 1286 1287 RTE_ETH_FOREACH_DEV(pi) { 1288 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1289 continue; 1290 1291 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) { 1292 allowed_min_txd = dev_info.tx_desc_lim.nb_min; 1293 *pid = pi; 1294 } 1295 } 1296 1297 return allowed_min_txd; 1298 } 1299 1300 /* 1301 * Check input txd is valid or not. 1302 * If input txd is not greater than any of maximum number 1303 * of TXDs of every Rx queues, it is valid. 1304 * if valid, return 0, else return -1 1305 */ 1306 int 1307 check_nb_txd(queueid_t txd) 1308 { 1309 uint16_t allowed_max_txd; 1310 uint16_t allowed_min_txd; 1311 portid_t pid = 0; 1312 1313 allowed_max_txd = get_allowed_max_nb_txd(&pid); 1314 if (txd > allowed_max_txd) { 1315 printf("Fail: input txd (%u) can't be greater " 1316 "than max_txds (%u) of port %u\n", 1317 txd, 1318 allowed_max_txd, 1319 pid); 1320 return -1; 1321 } 1322 1323 allowed_min_txd = get_allowed_min_nb_txd(&pid); 1324 if (txd < allowed_min_txd) { 1325 printf("Fail: input txd (%u) can't be less " 1326 "than min_txds (%u) of port %u\n", 1327 txd, 1328 allowed_min_txd, 1329 pid); 1330 return -1; 1331 } 1332 return 0; 1333 } 1334 1335 1336 /* 1337 * Get the allowed maximum number of hairpin queues. 1338 * *pid return the port id which has minimal value of 1339 * max_hairpin_queues in all ports. 1340 */ 1341 queueid_t 1342 get_allowed_max_nb_hairpinq(portid_t *pid) 1343 { 1344 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1345 portid_t pi; 1346 struct rte_eth_hairpin_cap cap; 1347 1348 RTE_ETH_FOREACH_DEV(pi) { 1349 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1350 *pid = pi; 1351 return 0; 1352 } 1353 if (cap.max_nb_queues < allowed_max_hairpinq) { 1354 allowed_max_hairpinq = cap.max_nb_queues; 1355 *pid = pi; 1356 } 1357 } 1358 return allowed_max_hairpinq; 1359 } 1360 1361 /* 1362 * Check input hairpin is valid or not. 1363 * If input hairpin is not greater than any of maximum number 1364 * of hairpin queues of all ports, it is valid. 1365 * if valid, return 0, else return -1 1366 */ 1367 int 1368 check_nb_hairpinq(queueid_t hairpinq) 1369 { 1370 queueid_t allowed_max_hairpinq; 1371 portid_t pid = 0; 1372 1373 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1374 if (hairpinq > allowed_max_hairpinq) { 1375 printf("Fail: input hairpin (%u) can't be greater " 1376 "than max_hairpin_queues (%u) of port %u\n", 1377 hairpinq, allowed_max_hairpinq, pid); 1378 return -1; 1379 } 1380 return 0; 1381 } 1382 1383 static void 1384 init_config(void) 1385 { 1386 portid_t pid; 1387 struct rte_port *port; 1388 struct rte_mempool *mbp; 1389 unsigned int nb_mbuf_per_pool; 1390 lcoreid_t lc_id; 1391 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1392 struct rte_gro_param gro_param; 1393 uint32_t gso_types; 1394 uint16_t data_size; 1395 bool warning = 0; 1396 int k; 1397 int ret; 1398 1399 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1400 1401 /* Configuration of logical cores. */ 1402 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1403 sizeof(struct fwd_lcore *) * nb_lcores, 1404 RTE_CACHE_LINE_SIZE); 1405 if (fwd_lcores == NULL) { 1406 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1407 "failed\n", nb_lcores); 1408 } 1409 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1410 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1411 sizeof(struct fwd_lcore), 1412 RTE_CACHE_LINE_SIZE); 1413 if (fwd_lcores[lc_id] == NULL) { 1414 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1415 "failed\n"); 1416 } 1417 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1418 } 1419 1420 RTE_ETH_FOREACH_DEV(pid) { 1421 port = &ports[pid]; 1422 /* Apply default TxRx configuration for all ports */ 1423 port->dev_conf.txmode = tx_mode; 1424 port->dev_conf.rxmode = rx_mode; 1425 1426 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1427 if (ret != 0) 1428 rte_exit(EXIT_FAILURE, 1429 "rte_eth_dev_info_get() failed\n"); 1430 1431 if (!(port->dev_info.tx_offload_capa & 1432 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1433 port->dev_conf.txmode.offloads &= 1434 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1435 if (numa_support) { 1436 if (port_numa[pid] != NUMA_NO_CONFIG) 1437 port_per_socket[port_numa[pid]]++; 1438 else { 1439 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1440 1441 /* 1442 * if socket_id is invalid, 1443 * set to the first available socket. 1444 */ 1445 if (check_socket_id(socket_id) < 0) 1446 socket_id = socket_ids[0]; 1447 port_per_socket[socket_id]++; 1448 } 1449 } 1450 1451 /* Apply Rx offloads configuration */ 1452 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1453 port->rx_conf[k].offloads = 1454 port->dev_conf.rxmode.offloads; 1455 /* Apply Tx offloads configuration */ 1456 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1457 port->tx_conf[k].offloads = 1458 port->dev_conf.txmode.offloads; 1459 1460 /* set flag to initialize port/queue */ 1461 port->need_reconfig = 1; 1462 port->need_reconfig_queues = 1; 1463 port->tx_metadata = 0; 1464 1465 /* Check for maximum number of segments per MTU. Accordingly 1466 * update the mbuf data size. 1467 */ 1468 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1469 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1470 data_size = rx_mode.max_rx_pkt_len / 1471 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1472 1473 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1474 mbuf_data_size) { 1475 mbuf_data_size = data_size + 1476 RTE_PKTMBUF_HEADROOM; 1477 warning = 1; 1478 } 1479 } 1480 } 1481 1482 if (warning) 1483 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1484 mbuf_data_size); 1485 1486 /* 1487 * Create pools of mbuf. 1488 * If NUMA support is disabled, create a single pool of mbuf in 1489 * socket 0 memory by default. 1490 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1491 * 1492 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1493 * nb_txd can be configured at run time. 1494 */ 1495 if (param_total_num_mbufs) 1496 nb_mbuf_per_pool = param_total_num_mbufs; 1497 else { 1498 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1499 (nb_lcores * mb_mempool_cache) + 1500 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1501 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1502 } 1503 1504 if (numa_support) { 1505 uint8_t i; 1506 1507 for (i = 0; i < num_sockets; i++) 1508 mempools[i] = mbuf_pool_create(mbuf_data_size, 1509 nb_mbuf_per_pool, 1510 socket_ids[i]); 1511 } else { 1512 if (socket_num == UMA_NO_CONFIG) 1513 mempools[0] = mbuf_pool_create(mbuf_data_size, 1514 nb_mbuf_per_pool, 0); 1515 else 1516 mempools[socket_num] = mbuf_pool_create 1517 (mbuf_data_size, 1518 nb_mbuf_per_pool, 1519 socket_num); 1520 } 1521 1522 init_port_config(); 1523 1524 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1525 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1526 /* 1527 * Records which Mbuf pool to use by each logical core, if needed. 1528 */ 1529 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1530 mbp = mbuf_pool_find( 1531 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1532 1533 if (mbp == NULL) 1534 mbp = mbuf_pool_find(0); 1535 fwd_lcores[lc_id]->mbp = mbp; 1536 /* initialize GSO context */ 1537 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1538 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1539 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1540 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1541 RTE_ETHER_CRC_LEN; 1542 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1543 } 1544 1545 /* Configuration of packet forwarding streams. */ 1546 if (init_fwd_streams() < 0) 1547 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1548 1549 fwd_config_setup(); 1550 1551 /* create a gro context for each lcore */ 1552 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1553 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1554 gro_param.max_item_per_flow = MAX_PKT_BURST; 1555 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1556 gro_param.socket_id = rte_lcore_to_socket_id( 1557 fwd_lcores_cpuids[lc_id]); 1558 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1559 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1560 rte_exit(EXIT_FAILURE, 1561 "rte_gro_ctx_create() failed\n"); 1562 } 1563 } 1564 1565 #if defined RTE_LIBRTE_PMD_SOFTNIC 1566 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1567 RTE_ETH_FOREACH_DEV(pid) { 1568 port = &ports[pid]; 1569 const char *driver = port->dev_info.driver_name; 1570 1571 if (strcmp(driver, "net_softnic") == 0) 1572 port->softport.fwd_lcore_arg = fwd_lcores; 1573 } 1574 } 1575 #endif 1576 1577 } 1578 1579 1580 void 1581 reconfig(portid_t new_port_id, unsigned socket_id) 1582 { 1583 struct rte_port *port; 1584 int ret; 1585 1586 /* Reconfiguration of Ethernet ports. */ 1587 port = &ports[new_port_id]; 1588 1589 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1590 if (ret != 0) 1591 return; 1592 1593 /* set flag to initialize port/queue */ 1594 port->need_reconfig = 1; 1595 port->need_reconfig_queues = 1; 1596 port->socket_id = socket_id; 1597 1598 init_port_config(); 1599 } 1600 1601 1602 int 1603 init_fwd_streams(void) 1604 { 1605 portid_t pid; 1606 struct rte_port *port; 1607 streamid_t sm_id, nb_fwd_streams_new; 1608 queueid_t q; 1609 1610 /* set socket id according to numa or not */ 1611 RTE_ETH_FOREACH_DEV(pid) { 1612 port = &ports[pid]; 1613 if (nb_rxq > port->dev_info.max_rx_queues) { 1614 printf("Fail: nb_rxq(%d) is greater than " 1615 "max_rx_queues(%d)\n", nb_rxq, 1616 port->dev_info.max_rx_queues); 1617 return -1; 1618 } 1619 if (nb_txq > port->dev_info.max_tx_queues) { 1620 printf("Fail: nb_txq(%d) is greater than " 1621 "max_tx_queues(%d)\n", nb_txq, 1622 port->dev_info.max_tx_queues); 1623 return -1; 1624 } 1625 if (numa_support) { 1626 if (port_numa[pid] != NUMA_NO_CONFIG) 1627 port->socket_id = port_numa[pid]; 1628 else { 1629 port->socket_id = rte_eth_dev_socket_id(pid); 1630 1631 /* 1632 * if socket_id is invalid, 1633 * set to the first available socket. 1634 */ 1635 if (check_socket_id(port->socket_id) < 0) 1636 port->socket_id = socket_ids[0]; 1637 } 1638 } 1639 else { 1640 if (socket_num == UMA_NO_CONFIG) 1641 port->socket_id = 0; 1642 else 1643 port->socket_id = socket_num; 1644 } 1645 } 1646 1647 q = RTE_MAX(nb_rxq, nb_txq); 1648 if (q == 0) { 1649 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1650 return -1; 1651 } 1652 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1653 if (nb_fwd_streams_new == nb_fwd_streams) 1654 return 0; 1655 /* clear the old */ 1656 if (fwd_streams != NULL) { 1657 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1658 if (fwd_streams[sm_id] == NULL) 1659 continue; 1660 rte_free(fwd_streams[sm_id]); 1661 fwd_streams[sm_id] = NULL; 1662 } 1663 rte_free(fwd_streams); 1664 fwd_streams = NULL; 1665 } 1666 1667 /* init new */ 1668 nb_fwd_streams = nb_fwd_streams_new; 1669 if (nb_fwd_streams) { 1670 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1671 sizeof(struct fwd_stream *) * nb_fwd_streams, 1672 RTE_CACHE_LINE_SIZE); 1673 if (fwd_streams == NULL) 1674 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1675 " (struct fwd_stream *)) failed\n", 1676 nb_fwd_streams); 1677 1678 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1679 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1680 " struct fwd_stream", sizeof(struct fwd_stream), 1681 RTE_CACHE_LINE_SIZE); 1682 if (fwd_streams[sm_id] == NULL) 1683 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1684 "(struct fwd_stream) failed\n"); 1685 } 1686 } 1687 1688 return 0; 1689 } 1690 1691 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1692 static void 1693 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1694 { 1695 uint64_t total_burst, sburst; 1696 uint64_t nb_burst; 1697 uint64_t burst_stats[4]; 1698 uint16_t pktnb_stats[4]; 1699 uint16_t nb_pkt; 1700 int burst_percent[4], sburstp; 1701 int i; 1702 1703 /* 1704 * First compute the total number of packet bursts and the 1705 * two highest numbers of bursts of the same number of packets. 1706 */ 1707 memset(&burst_stats, 0x0, sizeof(burst_stats)); 1708 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats)); 1709 1710 /* Show stats for 0 burst size always */ 1711 total_burst = pbs->pkt_burst_spread[0]; 1712 burst_stats[0] = pbs->pkt_burst_spread[0]; 1713 pktnb_stats[0] = 0; 1714 1715 /* Find the next 2 burst sizes with highest occurrences. */ 1716 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1717 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1718 1719 if (nb_burst == 0) 1720 continue; 1721 1722 total_burst += nb_burst; 1723 1724 if (nb_burst > burst_stats[1]) { 1725 burst_stats[2] = burst_stats[1]; 1726 pktnb_stats[2] = pktnb_stats[1]; 1727 burst_stats[1] = nb_burst; 1728 pktnb_stats[1] = nb_pkt; 1729 } else if (nb_burst > burst_stats[2]) { 1730 burst_stats[2] = nb_burst; 1731 pktnb_stats[2] = nb_pkt; 1732 } 1733 } 1734 if (total_burst == 0) 1735 return; 1736 1737 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst); 1738 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) { 1739 if (i == 3) { 1740 printf("%d%% of other]\n", 100 - sburstp); 1741 return; 1742 } 1743 1744 sburst += burst_stats[i]; 1745 if (sburst == total_burst) { 1746 printf("%d%% of %d pkts]\n", 1747 100 - sburstp, (int) pktnb_stats[i]); 1748 return; 1749 } 1750 1751 burst_percent[i] = 1752 (double)burst_stats[i] / total_burst * 100; 1753 printf("%d%% of %d pkts + ", 1754 burst_percent[i], (int) pktnb_stats[i]); 1755 sburstp += burst_percent[i]; 1756 } 1757 } 1758 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1759 1760 static void 1761 fwd_stream_stats_display(streamid_t stream_id) 1762 { 1763 struct fwd_stream *fs; 1764 static const char *fwd_top_stats_border = "-------"; 1765 1766 fs = fwd_streams[stream_id]; 1767 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1768 (fs->fwd_dropped == 0)) 1769 return; 1770 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1771 "TX Port=%2d/Queue=%2d %s\n", 1772 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1773 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1774 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1775 " TX-dropped: %-14"PRIu64, 1776 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1777 1778 /* if checksum mode */ 1779 if (cur_fwd_eng == &csum_fwd_engine) { 1780 printf(" RX- bad IP checksum: %-14"PRIu64 1781 " Rx- bad L4 checksum: %-14"PRIu64 1782 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1783 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1784 fs->rx_bad_outer_l4_csum); 1785 } else { 1786 printf("\n"); 1787 } 1788 1789 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1790 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1791 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1792 #endif 1793 } 1794 1795 void 1796 fwd_stats_display(void) 1797 { 1798 static const char *fwd_stats_border = "----------------------"; 1799 static const char *acc_stats_border = "+++++++++++++++"; 1800 struct { 1801 struct fwd_stream *rx_stream; 1802 struct fwd_stream *tx_stream; 1803 uint64_t tx_dropped; 1804 uint64_t rx_bad_ip_csum; 1805 uint64_t rx_bad_l4_csum; 1806 uint64_t rx_bad_outer_l4_csum; 1807 } ports_stats[RTE_MAX_ETHPORTS]; 1808 uint64_t total_rx_dropped = 0; 1809 uint64_t total_tx_dropped = 0; 1810 uint64_t total_rx_nombuf = 0; 1811 struct rte_eth_stats stats; 1812 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1813 uint64_t fwd_cycles = 0; 1814 #endif 1815 uint64_t total_recv = 0; 1816 uint64_t total_xmit = 0; 1817 struct rte_port *port; 1818 streamid_t sm_id; 1819 portid_t pt_id; 1820 int i; 1821 1822 memset(ports_stats, 0, sizeof(ports_stats)); 1823 1824 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1825 struct fwd_stream *fs = fwd_streams[sm_id]; 1826 1827 if (cur_fwd_config.nb_fwd_streams > 1828 cur_fwd_config.nb_fwd_ports) { 1829 fwd_stream_stats_display(sm_id); 1830 } else { 1831 ports_stats[fs->tx_port].tx_stream = fs; 1832 ports_stats[fs->rx_port].rx_stream = fs; 1833 } 1834 1835 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1836 1837 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1838 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1839 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1840 fs->rx_bad_outer_l4_csum; 1841 1842 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1843 fwd_cycles += fs->core_cycles; 1844 #endif 1845 } 1846 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1847 uint8_t j; 1848 1849 pt_id = fwd_ports_ids[i]; 1850 port = &ports[pt_id]; 1851 1852 rte_eth_stats_get(pt_id, &stats); 1853 stats.ipackets -= port->stats.ipackets; 1854 stats.opackets -= port->stats.opackets; 1855 stats.ibytes -= port->stats.ibytes; 1856 stats.obytes -= port->stats.obytes; 1857 stats.imissed -= port->stats.imissed; 1858 stats.oerrors -= port->stats.oerrors; 1859 stats.rx_nombuf -= port->stats.rx_nombuf; 1860 1861 total_recv += stats.ipackets; 1862 total_xmit += stats.opackets; 1863 total_rx_dropped += stats.imissed; 1864 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1865 total_tx_dropped += stats.oerrors; 1866 total_rx_nombuf += stats.rx_nombuf; 1867 1868 printf("\n %s Forward statistics for port %-2d %s\n", 1869 fwd_stats_border, pt_id, fwd_stats_border); 1870 1871 if (!port->rx_queue_stats_mapping_enabled && 1872 !port->tx_queue_stats_mapping_enabled) { 1873 printf(" RX-packets: %-14"PRIu64 1874 " RX-dropped: %-14"PRIu64 1875 "RX-total: %-"PRIu64"\n", 1876 stats.ipackets, stats.imissed, 1877 stats.ipackets + stats.imissed); 1878 1879 if (cur_fwd_eng == &csum_fwd_engine) 1880 printf(" Bad-ipcsum: %-14"PRIu64 1881 " Bad-l4csum: %-14"PRIu64 1882 "Bad-outer-l4csum: %-14"PRIu64"\n", 1883 ports_stats[pt_id].rx_bad_ip_csum, 1884 ports_stats[pt_id].rx_bad_l4_csum, 1885 ports_stats[pt_id].rx_bad_outer_l4_csum); 1886 if (stats.ierrors + stats.rx_nombuf > 0) { 1887 printf(" RX-error: %-"PRIu64"\n", 1888 stats.ierrors); 1889 printf(" RX-nombufs: %-14"PRIu64"\n", 1890 stats.rx_nombuf); 1891 } 1892 1893 printf(" TX-packets: %-14"PRIu64 1894 " TX-dropped: %-14"PRIu64 1895 "TX-total: %-"PRIu64"\n", 1896 stats.opackets, ports_stats[pt_id].tx_dropped, 1897 stats.opackets + ports_stats[pt_id].tx_dropped); 1898 } else { 1899 printf(" RX-packets: %14"PRIu64 1900 " RX-dropped:%14"PRIu64 1901 " RX-total:%14"PRIu64"\n", 1902 stats.ipackets, stats.imissed, 1903 stats.ipackets + stats.imissed); 1904 1905 if (cur_fwd_eng == &csum_fwd_engine) 1906 printf(" Bad-ipcsum:%14"PRIu64 1907 " Bad-l4csum:%14"PRIu64 1908 " Bad-outer-l4csum: %-14"PRIu64"\n", 1909 ports_stats[pt_id].rx_bad_ip_csum, 1910 ports_stats[pt_id].rx_bad_l4_csum, 1911 ports_stats[pt_id].rx_bad_outer_l4_csum); 1912 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1913 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1914 printf(" RX-nombufs: %14"PRIu64"\n", 1915 stats.rx_nombuf); 1916 } 1917 1918 printf(" TX-packets: %14"PRIu64 1919 " TX-dropped:%14"PRIu64 1920 " TX-total:%14"PRIu64"\n", 1921 stats.opackets, ports_stats[pt_id].tx_dropped, 1922 stats.opackets + ports_stats[pt_id].tx_dropped); 1923 } 1924 1925 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1926 if (ports_stats[pt_id].rx_stream) 1927 pkt_burst_stats_display("RX", 1928 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1929 if (ports_stats[pt_id].tx_stream) 1930 pkt_burst_stats_display("TX", 1931 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1932 #endif 1933 1934 if (port->rx_queue_stats_mapping_enabled) { 1935 printf("\n"); 1936 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1937 printf(" Stats reg %2d RX-packets:%14"PRIu64 1938 " RX-errors:%14"PRIu64 1939 " RX-bytes:%14"PRIu64"\n", 1940 j, stats.q_ipackets[j], 1941 stats.q_errors[j], stats.q_ibytes[j]); 1942 } 1943 printf("\n"); 1944 } 1945 if (port->tx_queue_stats_mapping_enabled) { 1946 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1947 printf(" Stats reg %2d TX-packets:%14"PRIu64 1948 " TX-bytes:%14" 1949 PRIu64"\n", 1950 j, stats.q_opackets[j], 1951 stats.q_obytes[j]); 1952 } 1953 } 1954 1955 printf(" %s--------------------------------%s\n", 1956 fwd_stats_border, fwd_stats_border); 1957 } 1958 1959 printf("\n %s Accumulated forward statistics for all ports" 1960 "%s\n", 1961 acc_stats_border, acc_stats_border); 1962 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1963 "%-"PRIu64"\n" 1964 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1965 "%-"PRIu64"\n", 1966 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1967 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1968 if (total_rx_nombuf > 0) 1969 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1970 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1971 "%s\n", 1972 acc_stats_border, acc_stats_border); 1973 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1974 #define CYC_PER_MHZ 1E6 1975 if (total_recv > 0) 1976 printf("\n CPU cycles/packet=%.2F (total cycles=" 1977 "%"PRIu64" / total RX packets=%"PRIu64") at %"PRIu64 1978 " MHz Clock\n", 1979 (double) fwd_cycles / total_recv, 1980 fwd_cycles, total_recv, 1981 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ)); 1982 #endif 1983 } 1984 1985 void 1986 fwd_stats_reset(void) 1987 { 1988 streamid_t sm_id; 1989 portid_t pt_id; 1990 int i; 1991 1992 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1993 pt_id = fwd_ports_ids[i]; 1994 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1995 } 1996 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1997 struct fwd_stream *fs = fwd_streams[sm_id]; 1998 1999 fs->rx_packets = 0; 2000 fs->tx_packets = 0; 2001 fs->fwd_dropped = 0; 2002 fs->rx_bad_ip_csum = 0; 2003 fs->rx_bad_l4_csum = 0; 2004 fs->rx_bad_outer_l4_csum = 0; 2005 2006 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 2007 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 2008 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 2009 #endif 2010 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 2011 fs->core_cycles = 0; 2012 #endif 2013 } 2014 } 2015 2016 static void 2017 flush_fwd_rx_queues(void) 2018 { 2019 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 2020 portid_t rxp; 2021 portid_t port_id; 2022 queueid_t rxq; 2023 uint16_t nb_rx; 2024 uint16_t i; 2025 uint8_t j; 2026 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 2027 uint64_t timer_period; 2028 2029 /* convert to number of cycles */ 2030 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 2031 2032 for (j = 0; j < 2; j++) { 2033 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 2034 for (rxq = 0; rxq < nb_rxq; rxq++) { 2035 port_id = fwd_ports_ids[rxp]; 2036 /** 2037 * testpmd can stuck in the below do while loop 2038 * if rte_eth_rx_burst() always returns nonzero 2039 * packets. So timer is added to exit this loop 2040 * after 1sec timer expiry. 2041 */ 2042 prev_tsc = rte_rdtsc(); 2043 do { 2044 nb_rx = rte_eth_rx_burst(port_id, rxq, 2045 pkts_burst, MAX_PKT_BURST); 2046 for (i = 0; i < nb_rx; i++) 2047 rte_pktmbuf_free(pkts_burst[i]); 2048 2049 cur_tsc = rte_rdtsc(); 2050 diff_tsc = cur_tsc - prev_tsc; 2051 timer_tsc += diff_tsc; 2052 } while ((nb_rx > 0) && 2053 (timer_tsc < timer_period)); 2054 timer_tsc = 0; 2055 } 2056 } 2057 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 2058 } 2059 } 2060 2061 static void 2062 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 2063 { 2064 struct fwd_stream **fsm; 2065 streamid_t nb_fs; 2066 streamid_t sm_id; 2067 #ifdef RTE_LIBRTE_BITRATE 2068 uint64_t tics_per_1sec; 2069 uint64_t tics_datum; 2070 uint64_t tics_current; 2071 uint16_t i, cnt_ports; 2072 2073 cnt_ports = nb_ports; 2074 tics_datum = rte_rdtsc(); 2075 tics_per_1sec = rte_get_timer_hz(); 2076 #endif 2077 fsm = &fwd_streams[fc->stream_idx]; 2078 nb_fs = fc->stream_nb; 2079 do { 2080 for (sm_id = 0; sm_id < nb_fs; sm_id++) 2081 (*pkt_fwd)(fsm[sm_id]); 2082 #ifdef RTE_LIBRTE_BITRATE 2083 if (bitrate_enabled != 0 && 2084 bitrate_lcore_id == rte_lcore_id()) { 2085 tics_current = rte_rdtsc(); 2086 if (tics_current - tics_datum >= tics_per_1sec) { 2087 /* Periodic bitrate calculation */ 2088 for (i = 0; i < cnt_ports; i++) 2089 rte_stats_bitrate_calc(bitrate_data, 2090 ports_ids[i]); 2091 tics_datum = tics_current; 2092 } 2093 } 2094 #endif 2095 #ifdef RTE_LIBRTE_LATENCY_STATS 2096 if (latencystats_enabled != 0 && 2097 latencystats_lcore_id == rte_lcore_id()) 2098 rte_latencystats_update(); 2099 #endif 2100 2101 } while (! fc->stopped); 2102 } 2103 2104 static int 2105 start_pkt_forward_on_core(void *fwd_arg) 2106 { 2107 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 2108 cur_fwd_config.fwd_eng->packet_fwd); 2109 return 0; 2110 } 2111 2112 /* 2113 * Run the TXONLY packet forwarding engine to send a single burst of packets. 2114 * Used to start communication flows in network loopback test configurations. 2115 */ 2116 static int 2117 run_one_txonly_burst_on_core(void *fwd_arg) 2118 { 2119 struct fwd_lcore *fwd_lc; 2120 struct fwd_lcore tmp_lcore; 2121 2122 fwd_lc = (struct fwd_lcore *) fwd_arg; 2123 tmp_lcore = *fwd_lc; 2124 tmp_lcore.stopped = 1; 2125 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 2126 return 0; 2127 } 2128 2129 /* 2130 * Launch packet forwarding: 2131 * - Setup per-port forwarding context. 2132 * - launch logical cores with their forwarding configuration. 2133 */ 2134 static void 2135 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 2136 { 2137 port_fwd_begin_t port_fwd_begin; 2138 unsigned int i; 2139 unsigned int lc_id; 2140 int diag; 2141 2142 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 2143 if (port_fwd_begin != NULL) { 2144 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2145 (*port_fwd_begin)(fwd_ports_ids[i]); 2146 } 2147 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 2148 lc_id = fwd_lcores_cpuids[i]; 2149 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 2150 fwd_lcores[i]->stopped = 0; 2151 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 2152 fwd_lcores[i], lc_id); 2153 if (diag != 0) 2154 printf("launch lcore %u failed - diag=%d\n", 2155 lc_id, diag); 2156 } 2157 } 2158 } 2159 2160 /* 2161 * Launch packet forwarding configuration. 2162 */ 2163 void 2164 start_packet_forwarding(int with_tx_first) 2165 { 2166 port_fwd_begin_t port_fwd_begin; 2167 port_fwd_end_t port_fwd_end; 2168 struct rte_port *port; 2169 unsigned int i; 2170 portid_t pt_id; 2171 2172 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 2173 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 2174 2175 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 2176 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 2177 2178 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 2179 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 2180 (!nb_rxq || !nb_txq)) 2181 rte_exit(EXIT_FAILURE, 2182 "Either rxq or txq are 0, cannot use %s fwd mode\n", 2183 cur_fwd_eng->fwd_mode_name); 2184 2185 if (all_ports_started() == 0) { 2186 printf("Not all ports were started\n"); 2187 return; 2188 } 2189 if (test_done == 0) { 2190 printf("Packet forwarding already started\n"); 2191 return; 2192 } 2193 2194 2195 if(dcb_test) { 2196 for (i = 0; i < nb_fwd_ports; i++) { 2197 pt_id = fwd_ports_ids[i]; 2198 port = &ports[pt_id]; 2199 if (!port->dcb_flag) { 2200 printf("In DCB mode, all forwarding ports must " 2201 "be configured in this mode.\n"); 2202 return; 2203 } 2204 } 2205 if (nb_fwd_lcores == 1) { 2206 printf("In DCB mode,the nb forwarding cores " 2207 "should be larger than 1.\n"); 2208 return; 2209 } 2210 } 2211 test_done = 0; 2212 2213 fwd_config_setup(); 2214 2215 if(!no_flush_rx) 2216 flush_fwd_rx_queues(); 2217 2218 pkt_fwd_config_display(&cur_fwd_config); 2219 rxtx_config_display(); 2220 2221 fwd_stats_reset(); 2222 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2223 pt_id = fwd_ports_ids[i]; 2224 port = &ports[pt_id]; 2225 map_port_queue_stats_mapping_registers(pt_id, port); 2226 } 2227 if (with_tx_first) { 2228 port_fwd_begin = tx_only_engine.port_fwd_begin; 2229 if (port_fwd_begin != NULL) { 2230 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2231 (*port_fwd_begin)(fwd_ports_ids[i]); 2232 } 2233 while (with_tx_first--) { 2234 launch_packet_forwarding( 2235 run_one_txonly_burst_on_core); 2236 rte_eal_mp_wait_lcore(); 2237 } 2238 port_fwd_end = tx_only_engine.port_fwd_end; 2239 if (port_fwd_end != NULL) { 2240 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 2241 (*port_fwd_end)(fwd_ports_ids[i]); 2242 } 2243 } 2244 launch_packet_forwarding(start_pkt_forward_on_core); 2245 } 2246 2247 void 2248 stop_packet_forwarding(void) 2249 { 2250 port_fwd_end_t port_fwd_end; 2251 lcoreid_t lc_id; 2252 portid_t pt_id; 2253 int i; 2254 2255 if (test_done) { 2256 printf("Packet forwarding not started\n"); 2257 return; 2258 } 2259 printf("Telling cores to stop..."); 2260 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 2261 fwd_lcores[lc_id]->stopped = 1; 2262 printf("\nWaiting for lcores to finish...\n"); 2263 rte_eal_mp_wait_lcore(); 2264 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 2265 if (port_fwd_end != NULL) { 2266 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 2267 pt_id = fwd_ports_ids[i]; 2268 (*port_fwd_end)(pt_id); 2269 } 2270 } 2271 2272 fwd_stats_display(); 2273 2274 printf("\nDone.\n"); 2275 test_done = 1; 2276 } 2277 2278 void 2279 dev_set_link_up(portid_t pid) 2280 { 2281 if (rte_eth_dev_set_link_up(pid) < 0) 2282 printf("\nSet link up fail.\n"); 2283 } 2284 2285 void 2286 dev_set_link_down(portid_t pid) 2287 { 2288 if (rte_eth_dev_set_link_down(pid) < 0) 2289 printf("\nSet link down fail.\n"); 2290 } 2291 2292 static int 2293 all_ports_started(void) 2294 { 2295 portid_t pi; 2296 struct rte_port *port; 2297 2298 RTE_ETH_FOREACH_DEV(pi) { 2299 port = &ports[pi]; 2300 /* Check if there is a port which is not started */ 2301 if ((port->port_status != RTE_PORT_STARTED) && 2302 (port->slave_flag == 0)) 2303 return 0; 2304 } 2305 2306 /* No port is not started */ 2307 return 1; 2308 } 2309 2310 int 2311 port_is_stopped(portid_t port_id) 2312 { 2313 struct rte_port *port = &ports[port_id]; 2314 2315 if ((port->port_status != RTE_PORT_STOPPED) && 2316 (port->slave_flag == 0)) 2317 return 0; 2318 return 1; 2319 } 2320 2321 int 2322 all_ports_stopped(void) 2323 { 2324 portid_t pi; 2325 2326 RTE_ETH_FOREACH_DEV(pi) { 2327 if (!port_is_stopped(pi)) 2328 return 0; 2329 } 2330 2331 return 1; 2332 } 2333 2334 int 2335 port_is_started(portid_t port_id) 2336 { 2337 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2338 return 0; 2339 2340 if (ports[port_id].port_status != RTE_PORT_STARTED) 2341 return 0; 2342 2343 return 1; 2344 } 2345 2346 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2347 static int 2348 setup_hairpin_queues(portid_t pi) 2349 { 2350 queueid_t qi; 2351 struct rte_eth_hairpin_conf hairpin_conf = { 2352 .peer_count = 1, 2353 }; 2354 int i; 2355 int diag; 2356 struct rte_port *port = &ports[pi]; 2357 2358 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2359 hairpin_conf.peers[0].port = pi; 2360 hairpin_conf.peers[0].queue = i + nb_rxq; 2361 diag = rte_eth_tx_hairpin_queue_setup 2362 (pi, qi, nb_txd, &hairpin_conf); 2363 i++; 2364 if (diag == 0) 2365 continue; 2366 2367 /* Fail to setup rx queue, return */ 2368 if (rte_atomic16_cmpset(&(port->port_status), 2369 RTE_PORT_HANDLING, 2370 RTE_PORT_STOPPED) == 0) 2371 printf("Port %d can not be set back " 2372 "to stopped\n", pi); 2373 printf("Fail to configure port %d hairpin " 2374 "queues\n", pi); 2375 /* try to reconfigure queues next time */ 2376 port->need_reconfig_queues = 1; 2377 return -1; 2378 } 2379 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2380 hairpin_conf.peers[0].port = pi; 2381 hairpin_conf.peers[0].queue = i + nb_txq; 2382 diag = rte_eth_rx_hairpin_queue_setup 2383 (pi, qi, nb_rxd, &hairpin_conf); 2384 i++; 2385 if (diag == 0) 2386 continue; 2387 2388 /* Fail to setup rx queue, return */ 2389 if (rte_atomic16_cmpset(&(port->port_status), 2390 RTE_PORT_HANDLING, 2391 RTE_PORT_STOPPED) == 0) 2392 printf("Port %d can not be set back " 2393 "to stopped\n", pi); 2394 printf("Fail to configure port %d hairpin " 2395 "queues\n", pi); 2396 /* try to reconfigure queues next time */ 2397 port->need_reconfig_queues = 1; 2398 return -1; 2399 } 2400 return 0; 2401 } 2402 2403 int 2404 start_port(portid_t pid) 2405 { 2406 int diag, need_check_link_status = -1; 2407 portid_t pi; 2408 queueid_t qi; 2409 struct rte_port *port; 2410 struct rte_ether_addr mac_addr; 2411 struct rte_eth_hairpin_cap cap; 2412 2413 if (port_id_is_invalid(pid, ENABLED_WARN)) 2414 return 0; 2415 2416 if(dcb_config) 2417 dcb_test = 1; 2418 RTE_ETH_FOREACH_DEV(pi) { 2419 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2420 continue; 2421 2422 need_check_link_status = 0; 2423 port = &ports[pi]; 2424 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2425 RTE_PORT_HANDLING) == 0) { 2426 printf("Port %d is now not stopped\n", pi); 2427 continue; 2428 } 2429 2430 if (port->need_reconfig > 0) { 2431 port->need_reconfig = 0; 2432 2433 if (flow_isolate_all) { 2434 int ret = port_flow_isolate(pi, 1); 2435 if (ret) { 2436 printf("Failed to apply isolated" 2437 " mode on port %d\n", pi); 2438 return -1; 2439 } 2440 } 2441 configure_rxtx_dump_callbacks(0); 2442 printf("Configuring Port %d (socket %u)\n", pi, 2443 port->socket_id); 2444 if (nb_hairpinq > 0 && 2445 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2446 printf("Port %d doesn't support hairpin " 2447 "queues\n", pi); 2448 return -1; 2449 } 2450 /* configure port */ 2451 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2452 nb_txq + nb_hairpinq, 2453 &(port->dev_conf)); 2454 if (diag != 0) { 2455 if (rte_atomic16_cmpset(&(port->port_status), 2456 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2457 printf("Port %d can not be set back " 2458 "to stopped\n", pi); 2459 printf("Fail to configure port %d\n", pi); 2460 /* try to reconfigure port next time */ 2461 port->need_reconfig = 1; 2462 return -1; 2463 } 2464 } 2465 if (port->need_reconfig_queues > 0) { 2466 port->need_reconfig_queues = 0; 2467 /* setup tx queues */ 2468 for (qi = 0; qi < nb_txq; qi++) { 2469 if ((numa_support) && 2470 (txring_numa[pi] != NUMA_NO_CONFIG)) 2471 diag = rte_eth_tx_queue_setup(pi, qi, 2472 port->nb_tx_desc[qi], 2473 txring_numa[pi], 2474 &(port->tx_conf[qi])); 2475 else 2476 diag = rte_eth_tx_queue_setup(pi, qi, 2477 port->nb_tx_desc[qi], 2478 port->socket_id, 2479 &(port->tx_conf[qi])); 2480 2481 if (diag == 0) 2482 continue; 2483 2484 /* Fail to setup tx queue, return */ 2485 if (rte_atomic16_cmpset(&(port->port_status), 2486 RTE_PORT_HANDLING, 2487 RTE_PORT_STOPPED) == 0) 2488 printf("Port %d can not be set back " 2489 "to stopped\n", pi); 2490 printf("Fail to configure port %d tx queues\n", 2491 pi); 2492 /* try to reconfigure queues next time */ 2493 port->need_reconfig_queues = 1; 2494 return -1; 2495 } 2496 for (qi = 0; qi < nb_rxq; qi++) { 2497 /* setup rx queues */ 2498 if ((numa_support) && 2499 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2500 struct rte_mempool * mp = 2501 mbuf_pool_find(rxring_numa[pi]); 2502 if (mp == NULL) { 2503 printf("Failed to setup RX queue:" 2504 "No mempool allocation" 2505 " on the socket %d\n", 2506 rxring_numa[pi]); 2507 return -1; 2508 } 2509 2510 diag = rte_eth_rx_queue_setup(pi, qi, 2511 port->nb_rx_desc[qi], 2512 rxring_numa[pi], 2513 &(port->rx_conf[qi]), 2514 mp); 2515 } else { 2516 struct rte_mempool *mp = 2517 mbuf_pool_find(port->socket_id); 2518 if (mp == NULL) { 2519 printf("Failed to setup RX queue:" 2520 "No mempool allocation" 2521 " on the socket %d\n", 2522 port->socket_id); 2523 return -1; 2524 } 2525 diag = rte_eth_rx_queue_setup(pi, qi, 2526 port->nb_rx_desc[qi], 2527 port->socket_id, 2528 &(port->rx_conf[qi]), 2529 mp); 2530 } 2531 if (diag == 0) 2532 continue; 2533 2534 /* Fail to setup rx queue, return */ 2535 if (rte_atomic16_cmpset(&(port->port_status), 2536 RTE_PORT_HANDLING, 2537 RTE_PORT_STOPPED) == 0) 2538 printf("Port %d can not be set back " 2539 "to stopped\n", pi); 2540 printf("Fail to configure port %d rx queues\n", 2541 pi); 2542 /* try to reconfigure queues next time */ 2543 port->need_reconfig_queues = 1; 2544 return -1; 2545 } 2546 /* setup hairpin queues */ 2547 if (setup_hairpin_queues(pi) != 0) 2548 return -1; 2549 } 2550 configure_rxtx_dump_callbacks(verbose_level); 2551 if (clear_ptypes) { 2552 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2553 NULL, 0); 2554 if (diag < 0) 2555 printf( 2556 "Port %d: Failed to disable Ptype parsing\n", 2557 pi); 2558 } 2559 2560 /* start port */ 2561 if (rte_eth_dev_start(pi) < 0) { 2562 printf("Fail to start port %d\n", pi); 2563 2564 /* Fail to setup rx queue, return */ 2565 if (rte_atomic16_cmpset(&(port->port_status), 2566 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2567 printf("Port %d can not be set back to " 2568 "stopped\n", pi); 2569 continue; 2570 } 2571 2572 if (rte_atomic16_cmpset(&(port->port_status), 2573 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2574 printf("Port %d can not be set into started\n", pi); 2575 2576 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2577 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2578 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2579 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2580 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2581 2582 /* at least one port started, need checking link status */ 2583 need_check_link_status = 1; 2584 } 2585 2586 if (need_check_link_status == 1 && !no_link_check) 2587 check_all_ports_link_status(RTE_PORT_ALL); 2588 else if (need_check_link_status == 0) 2589 printf("Please stop the ports first\n"); 2590 2591 printf("Done\n"); 2592 return 0; 2593 } 2594 2595 void 2596 stop_port(portid_t pid) 2597 { 2598 portid_t pi; 2599 struct rte_port *port; 2600 int need_check_link_status = 0; 2601 2602 if (dcb_test) { 2603 dcb_test = 0; 2604 dcb_config = 0; 2605 } 2606 2607 if (port_id_is_invalid(pid, ENABLED_WARN)) 2608 return; 2609 2610 printf("Stopping ports...\n"); 2611 2612 RTE_ETH_FOREACH_DEV(pi) { 2613 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2614 continue; 2615 2616 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2617 printf("Please remove port %d from forwarding configuration.\n", pi); 2618 continue; 2619 } 2620 2621 if (port_is_bonding_slave(pi)) { 2622 printf("Please remove port %d from bonded device.\n", pi); 2623 continue; 2624 } 2625 2626 port = &ports[pi]; 2627 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2628 RTE_PORT_HANDLING) == 0) 2629 continue; 2630 2631 rte_eth_dev_stop(pi); 2632 2633 if (rte_atomic16_cmpset(&(port->port_status), 2634 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2635 printf("Port %d can not be set into stopped\n", pi); 2636 need_check_link_status = 1; 2637 } 2638 if (need_check_link_status && !no_link_check) 2639 check_all_ports_link_status(RTE_PORT_ALL); 2640 2641 printf("Done\n"); 2642 } 2643 2644 static void 2645 remove_invalid_ports_in(portid_t *array, portid_t *total) 2646 { 2647 portid_t i; 2648 portid_t new_total = 0; 2649 2650 for (i = 0; i < *total; i++) 2651 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2652 array[new_total] = array[i]; 2653 new_total++; 2654 } 2655 *total = new_total; 2656 } 2657 2658 static void 2659 remove_invalid_ports(void) 2660 { 2661 remove_invalid_ports_in(ports_ids, &nb_ports); 2662 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2663 nb_cfg_ports = nb_fwd_ports; 2664 } 2665 2666 void 2667 close_port(portid_t pid) 2668 { 2669 portid_t pi; 2670 struct rte_port *port; 2671 2672 if (port_id_is_invalid(pid, ENABLED_WARN)) 2673 return; 2674 2675 printf("Closing ports...\n"); 2676 2677 RTE_ETH_FOREACH_DEV(pi) { 2678 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2679 continue; 2680 2681 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2682 printf("Please remove port %d from forwarding configuration.\n", pi); 2683 continue; 2684 } 2685 2686 if (port_is_bonding_slave(pi)) { 2687 printf("Please remove port %d from bonded device.\n", pi); 2688 continue; 2689 } 2690 2691 port = &ports[pi]; 2692 if (rte_atomic16_cmpset(&(port->port_status), 2693 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2694 printf("Port %d is already closed\n", pi); 2695 continue; 2696 } 2697 2698 if (rte_atomic16_cmpset(&(port->port_status), 2699 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2700 printf("Port %d is now not stopped\n", pi); 2701 continue; 2702 } 2703 2704 if (port->flow_list) 2705 port_flow_flush(pi); 2706 rte_eth_dev_close(pi); 2707 2708 remove_invalid_ports(); 2709 2710 if (rte_atomic16_cmpset(&(port->port_status), 2711 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2712 printf("Port %d cannot be set to closed\n", pi); 2713 } 2714 2715 printf("Done\n"); 2716 } 2717 2718 void 2719 reset_port(portid_t pid) 2720 { 2721 int diag; 2722 portid_t pi; 2723 struct rte_port *port; 2724 2725 if (port_id_is_invalid(pid, ENABLED_WARN)) 2726 return; 2727 2728 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2729 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2730 printf("Can not reset port(s), please stop port(s) first.\n"); 2731 return; 2732 } 2733 2734 printf("Resetting ports...\n"); 2735 2736 RTE_ETH_FOREACH_DEV(pi) { 2737 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2738 continue; 2739 2740 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2741 printf("Please remove port %d from forwarding " 2742 "configuration.\n", pi); 2743 continue; 2744 } 2745 2746 if (port_is_bonding_slave(pi)) { 2747 printf("Please remove port %d from bonded device.\n", 2748 pi); 2749 continue; 2750 } 2751 2752 diag = rte_eth_dev_reset(pi); 2753 if (diag == 0) { 2754 port = &ports[pi]; 2755 port->need_reconfig = 1; 2756 port->need_reconfig_queues = 1; 2757 } else { 2758 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2759 } 2760 } 2761 2762 printf("Done\n"); 2763 } 2764 2765 void 2766 attach_port(char *identifier) 2767 { 2768 portid_t pi; 2769 struct rte_dev_iterator iterator; 2770 2771 printf("Attaching a new port...\n"); 2772 2773 if (identifier == NULL) { 2774 printf("Invalid parameters are specified\n"); 2775 return; 2776 } 2777 2778 if (rte_dev_probe(identifier) < 0) { 2779 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2780 return; 2781 } 2782 2783 /* first attach mode: event */ 2784 if (setup_on_probe_event) { 2785 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2786 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2787 if (ports[pi].port_status == RTE_PORT_HANDLING && 2788 ports[pi].need_setup != 0) 2789 setup_attached_port(pi); 2790 return; 2791 } 2792 2793 /* second attach mode: iterator */ 2794 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2795 /* setup ports matching the devargs used for probing */ 2796 if (port_is_forwarding(pi)) 2797 continue; /* port was already attached before */ 2798 setup_attached_port(pi); 2799 } 2800 } 2801 2802 static void 2803 setup_attached_port(portid_t pi) 2804 { 2805 unsigned int socket_id; 2806 int ret; 2807 2808 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2809 /* if socket_id is invalid, set to the first available socket. */ 2810 if (check_socket_id(socket_id) < 0) 2811 socket_id = socket_ids[0]; 2812 reconfig(pi, socket_id); 2813 ret = rte_eth_promiscuous_enable(pi); 2814 if (ret != 0) 2815 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2816 pi, rte_strerror(-ret)); 2817 2818 ports_ids[nb_ports++] = pi; 2819 fwd_ports_ids[nb_fwd_ports++] = pi; 2820 nb_cfg_ports = nb_fwd_ports; 2821 ports[pi].need_setup = 0; 2822 ports[pi].port_status = RTE_PORT_STOPPED; 2823 2824 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2825 printf("Done\n"); 2826 } 2827 2828 static void 2829 detach_device(struct rte_device *dev) 2830 { 2831 portid_t sibling; 2832 2833 if (dev == NULL) { 2834 printf("Device already removed\n"); 2835 return; 2836 } 2837 2838 printf("Removing a device...\n"); 2839 2840 if (rte_dev_remove(dev) < 0) { 2841 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2842 return; 2843 } 2844 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2845 /* reset mapping between old ports and removed device */ 2846 rte_eth_devices[sibling].device = NULL; 2847 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2848 /* sibling ports are forced to be closed */ 2849 ports[sibling].port_status = RTE_PORT_CLOSED; 2850 printf("Port %u is closed\n", sibling); 2851 } 2852 } 2853 2854 remove_invalid_ports(); 2855 2856 printf("Device is detached\n"); 2857 printf("Now total ports is %d\n", nb_ports); 2858 printf("Done\n"); 2859 return; 2860 } 2861 2862 void 2863 detach_port_device(portid_t port_id) 2864 { 2865 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2866 return; 2867 2868 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2869 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2870 printf("Port not stopped\n"); 2871 return; 2872 } 2873 printf("Port was not closed\n"); 2874 if (ports[port_id].flow_list) 2875 port_flow_flush(port_id); 2876 } 2877 2878 detach_device(rte_eth_devices[port_id].device); 2879 } 2880 2881 void 2882 detach_devargs(char *identifier) 2883 { 2884 struct rte_dev_iterator iterator; 2885 struct rte_devargs da; 2886 portid_t port_id; 2887 2888 printf("Removing a device...\n"); 2889 2890 memset(&da, 0, sizeof(da)); 2891 if (rte_devargs_parsef(&da, "%s", identifier)) { 2892 printf("cannot parse identifier\n"); 2893 if (da.args) 2894 free(da.args); 2895 return; 2896 } 2897 2898 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2899 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2900 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2901 printf("Port %u not stopped\n", port_id); 2902 rte_eth_iterator_cleanup(&iterator); 2903 return; 2904 } 2905 2906 /* sibling ports are forced to be closed */ 2907 if (ports[port_id].flow_list) 2908 port_flow_flush(port_id); 2909 ports[port_id].port_status = RTE_PORT_CLOSED; 2910 printf("Port %u is now closed\n", port_id); 2911 } 2912 } 2913 2914 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2915 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2916 da.name, da.bus->name); 2917 return; 2918 } 2919 2920 remove_invalid_ports(); 2921 2922 printf("Device %s is detached\n", identifier); 2923 printf("Now total ports is %d\n", nb_ports); 2924 printf("Done\n"); 2925 } 2926 2927 void 2928 pmd_test_exit(void) 2929 { 2930 portid_t pt_id; 2931 int ret; 2932 int i; 2933 2934 if (test_done == 0) 2935 stop_packet_forwarding(); 2936 2937 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2938 if (mempools[i]) { 2939 if (mp_alloc_type == MP_ALLOC_ANON) 2940 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2941 NULL); 2942 } 2943 } 2944 if (ports != NULL) { 2945 no_link_check = 1; 2946 RTE_ETH_FOREACH_DEV(pt_id) { 2947 printf("\nStopping port %d...\n", pt_id); 2948 fflush(stdout); 2949 stop_port(pt_id); 2950 } 2951 RTE_ETH_FOREACH_DEV(pt_id) { 2952 printf("\nShutting down port %d...\n", pt_id); 2953 fflush(stdout); 2954 close_port(pt_id); 2955 } 2956 } 2957 2958 if (hot_plug) { 2959 ret = rte_dev_event_monitor_stop(); 2960 if (ret) { 2961 RTE_LOG(ERR, EAL, 2962 "fail to stop device event monitor."); 2963 return; 2964 } 2965 2966 ret = rte_dev_event_callback_unregister(NULL, 2967 dev_event_callback, NULL); 2968 if (ret < 0) { 2969 RTE_LOG(ERR, EAL, 2970 "fail to unregister device event callback.\n"); 2971 return; 2972 } 2973 2974 ret = rte_dev_hotplug_handle_disable(); 2975 if (ret) { 2976 RTE_LOG(ERR, EAL, 2977 "fail to disable hotplug handling.\n"); 2978 return; 2979 } 2980 } 2981 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2982 if (mempools[i]) 2983 rte_mempool_free(mempools[i]); 2984 } 2985 2986 printf("\nBye...\n"); 2987 } 2988 2989 typedef void (*cmd_func_t)(void); 2990 struct pmd_test_command { 2991 const char *cmd_name; 2992 cmd_func_t cmd_func; 2993 }; 2994 2995 /* Check the link status of all ports in up to 9s, and print them finally */ 2996 static void 2997 check_all_ports_link_status(uint32_t port_mask) 2998 { 2999 #define CHECK_INTERVAL 100 /* 100ms */ 3000 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 3001 portid_t portid; 3002 uint8_t count, all_ports_up, print_flag = 0; 3003 struct rte_eth_link link; 3004 int ret; 3005 3006 printf("Checking link statuses...\n"); 3007 fflush(stdout); 3008 for (count = 0; count <= MAX_CHECK_TIME; count++) { 3009 all_ports_up = 1; 3010 RTE_ETH_FOREACH_DEV(portid) { 3011 if ((port_mask & (1 << portid)) == 0) 3012 continue; 3013 memset(&link, 0, sizeof(link)); 3014 ret = rte_eth_link_get_nowait(portid, &link); 3015 if (ret < 0) { 3016 all_ports_up = 0; 3017 if (print_flag == 1) 3018 printf("Port %u link get failed: %s\n", 3019 portid, rte_strerror(-ret)); 3020 continue; 3021 } 3022 /* print link status if flag set */ 3023 if (print_flag == 1) { 3024 if (link.link_status) 3025 printf( 3026 "Port%d Link Up. speed %u Mbps- %s\n", 3027 portid, link.link_speed, 3028 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 3029 ("full-duplex") : ("half-duplex")); 3030 else 3031 printf("Port %d Link Down\n", portid); 3032 continue; 3033 } 3034 /* clear all_ports_up flag if any link down */ 3035 if (link.link_status == ETH_LINK_DOWN) { 3036 all_ports_up = 0; 3037 break; 3038 } 3039 } 3040 /* after finally printing all link status, get out */ 3041 if (print_flag == 1) 3042 break; 3043 3044 if (all_ports_up == 0) { 3045 fflush(stdout); 3046 rte_delay_ms(CHECK_INTERVAL); 3047 } 3048 3049 /* set the print_flag if all ports up or timeout */ 3050 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 3051 print_flag = 1; 3052 } 3053 3054 if (lsc_interrupt) 3055 break; 3056 } 3057 } 3058 3059 /* 3060 * This callback is for remove a port for a device. It has limitation because 3061 * it is not for multiple port removal for a device. 3062 * TODO: the device detach invoke will plan to be removed from user side to 3063 * eal. And convert all PMDs to free port resources on ether device closing. 3064 */ 3065 static void 3066 rmv_port_callback(void *arg) 3067 { 3068 int need_to_start = 0; 3069 int org_no_link_check = no_link_check; 3070 portid_t port_id = (intptr_t)arg; 3071 struct rte_device *dev; 3072 3073 RTE_ETH_VALID_PORTID_OR_RET(port_id); 3074 3075 if (!test_done && port_is_forwarding(port_id)) { 3076 need_to_start = 1; 3077 stop_packet_forwarding(); 3078 } 3079 no_link_check = 1; 3080 stop_port(port_id); 3081 no_link_check = org_no_link_check; 3082 3083 /* Save rte_device pointer before closing ethdev port */ 3084 dev = rte_eth_devices[port_id].device; 3085 close_port(port_id); 3086 detach_device(dev); /* might be already removed or have more ports */ 3087 3088 if (need_to_start) 3089 start_packet_forwarding(0); 3090 } 3091 3092 /* This function is used by the interrupt thread */ 3093 static int 3094 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 3095 void *ret_param) 3096 { 3097 RTE_SET_USED(param); 3098 RTE_SET_USED(ret_param); 3099 3100 if (type >= RTE_ETH_EVENT_MAX) { 3101 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 3102 port_id, __func__, type); 3103 fflush(stderr); 3104 } else if (event_print_mask & (UINT32_C(1) << type)) { 3105 printf("\nPort %" PRIu16 ": %s event\n", port_id, 3106 eth_event_desc[type]); 3107 fflush(stdout); 3108 } 3109 3110 switch (type) { 3111 case RTE_ETH_EVENT_NEW: 3112 ports[port_id].need_setup = 1; 3113 ports[port_id].port_status = RTE_PORT_HANDLING; 3114 break; 3115 case RTE_ETH_EVENT_INTR_RMV: 3116 if (port_id_is_invalid(port_id, DISABLED_WARN)) 3117 break; 3118 if (rte_eal_alarm_set(100000, 3119 rmv_port_callback, (void *)(intptr_t)port_id)) 3120 fprintf(stderr, "Could not set up deferred device removal\n"); 3121 break; 3122 default: 3123 break; 3124 } 3125 return 0; 3126 } 3127 3128 static int 3129 register_eth_event_callback(void) 3130 { 3131 int ret; 3132 enum rte_eth_event_type event; 3133 3134 for (event = RTE_ETH_EVENT_UNKNOWN; 3135 event < RTE_ETH_EVENT_MAX; event++) { 3136 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 3137 event, 3138 eth_event_callback, 3139 NULL); 3140 if (ret != 0) { 3141 TESTPMD_LOG(ERR, "Failed to register callback for " 3142 "%s event\n", eth_event_desc[event]); 3143 return -1; 3144 } 3145 } 3146 3147 return 0; 3148 } 3149 3150 /* This function is used by the interrupt thread */ 3151 static void 3152 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 3153 __rte_unused void *arg) 3154 { 3155 uint16_t port_id; 3156 int ret; 3157 3158 if (type >= RTE_DEV_EVENT_MAX) { 3159 fprintf(stderr, "%s called upon invalid event %d\n", 3160 __func__, type); 3161 fflush(stderr); 3162 } 3163 3164 switch (type) { 3165 case RTE_DEV_EVENT_REMOVE: 3166 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 3167 device_name); 3168 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 3169 if (ret) { 3170 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 3171 device_name); 3172 return; 3173 } 3174 /* 3175 * Because the user's callback is invoked in eal interrupt 3176 * callback, the interrupt callback need to be finished before 3177 * it can be unregistered when detaching device. So finish 3178 * callback soon and use a deferred removal to detach device 3179 * is need. It is a workaround, once the device detaching be 3180 * moved into the eal in the future, the deferred removal could 3181 * be deleted. 3182 */ 3183 if (rte_eal_alarm_set(100000, 3184 rmv_port_callback, (void *)(intptr_t)port_id)) 3185 RTE_LOG(ERR, EAL, 3186 "Could not set up deferred device removal\n"); 3187 break; 3188 case RTE_DEV_EVENT_ADD: 3189 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 3190 device_name); 3191 /* TODO: After finish kernel driver binding, 3192 * begin to attach port. 3193 */ 3194 break; 3195 default: 3196 break; 3197 } 3198 } 3199 3200 static int 3201 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3202 { 3203 uint16_t i; 3204 int diag; 3205 uint8_t mapping_found = 0; 3206 3207 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 3208 if ((tx_queue_stats_mappings[i].port_id == port_id) && 3209 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 3210 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 3211 tx_queue_stats_mappings[i].queue_id, 3212 tx_queue_stats_mappings[i].stats_counter_id); 3213 if (diag != 0) 3214 return diag; 3215 mapping_found = 1; 3216 } 3217 } 3218 if (mapping_found) 3219 port->tx_queue_stats_mapping_enabled = 1; 3220 return 0; 3221 } 3222 3223 static int 3224 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 3225 { 3226 uint16_t i; 3227 int diag; 3228 uint8_t mapping_found = 0; 3229 3230 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 3231 if ((rx_queue_stats_mappings[i].port_id == port_id) && 3232 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 3233 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 3234 rx_queue_stats_mappings[i].queue_id, 3235 rx_queue_stats_mappings[i].stats_counter_id); 3236 if (diag != 0) 3237 return diag; 3238 mapping_found = 1; 3239 } 3240 } 3241 if (mapping_found) 3242 port->rx_queue_stats_mapping_enabled = 1; 3243 return 0; 3244 } 3245 3246 static void 3247 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 3248 { 3249 int diag = 0; 3250 3251 diag = set_tx_queue_stats_mapping_registers(pi, port); 3252 if (diag != 0) { 3253 if (diag == -ENOTSUP) { 3254 port->tx_queue_stats_mapping_enabled = 0; 3255 printf("TX queue stats mapping not supported port id=%d\n", pi); 3256 } 3257 else 3258 rte_exit(EXIT_FAILURE, 3259 "set_tx_queue_stats_mapping_registers " 3260 "failed for port id=%d diag=%d\n", 3261 pi, diag); 3262 } 3263 3264 diag = set_rx_queue_stats_mapping_registers(pi, port); 3265 if (diag != 0) { 3266 if (diag == -ENOTSUP) { 3267 port->rx_queue_stats_mapping_enabled = 0; 3268 printf("RX queue stats mapping not supported port id=%d\n", pi); 3269 } 3270 else 3271 rte_exit(EXIT_FAILURE, 3272 "set_rx_queue_stats_mapping_registers " 3273 "failed for port id=%d diag=%d\n", 3274 pi, diag); 3275 } 3276 } 3277 3278 static void 3279 rxtx_port_config(struct rte_port *port) 3280 { 3281 uint16_t qid; 3282 uint64_t offloads; 3283 3284 for (qid = 0; qid < nb_rxq; qid++) { 3285 offloads = port->rx_conf[qid].offloads; 3286 port->rx_conf[qid] = port->dev_info.default_rxconf; 3287 if (offloads != 0) 3288 port->rx_conf[qid].offloads = offloads; 3289 3290 /* Check if any Rx parameters have been passed */ 3291 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3292 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3293 3294 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3295 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3296 3297 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3298 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3299 3300 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3301 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3302 3303 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3304 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3305 3306 port->nb_rx_desc[qid] = nb_rxd; 3307 } 3308 3309 for (qid = 0; qid < nb_txq; qid++) { 3310 offloads = port->tx_conf[qid].offloads; 3311 port->tx_conf[qid] = port->dev_info.default_txconf; 3312 if (offloads != 0) 3313 port->tx_conf[qid].offloads = offloads; 3314 3315 /* Check if any Tx parameters have been passed */ 3316 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3317 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3318 3319 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3320 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3321 3322 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3323 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3324 3325 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3326 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3327 3328 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3329 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3330 3331 port->nb_tx_desc[qid] = nb_txd; 3332 } 3333 } 3334 3335 void 3336 init_port_config(void) 3337 { 3338 portid_t pid; 3339 struct rte_port *port; 3340 int ret; 3341 3342 RTE_ETH_FOREACH_DEV(pid) { 3343 port = &ports[pid]; 3344 port->dev_conf.fdir_conf = fdir_conf; 3345 3346 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3347 if (ret != 0) 3348 return; 3349 3350 if (nb_rxq > 1) { 3351 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3352 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3353 rss_hf & port->dev_info.flow_type_rss_offloads; 3354 } else { 3355 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3356 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3357 } 3358 3359 if (port->dcb_flag == 0) { 3360 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3361 port->dev_conf.rxmode.mq_mode = 3362 (enum rte_eth_rx_mq_mode) 3363 (rx_mq_mode & ETH_MQ_RX_RSS); 3364 else 3365 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3366 } 3367 3368 rxtx_port_config(port); 3369 3370 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3371 if (ret != 0) 3372 return; 3373 3374 map_port_queue_stats_mapping_registers(pid, port); 3375 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3376 rte_pmd_ixgbe_bypass_init(pid); 3377 #endif 3378 3379 if (lsc_interrupt && 3380 (rte_eth_devices[pid].data->dev_flags & 3381 RTE_ETH_DEV_INTR_LSC)) 3382 port->dev_conf.intr_conf.lsc = 1; 3383 if (rmv_interrupt && 3384 (rte_eth_devices[pid].data->dev_flags & 3385 RTE_ETH_DEV_INTR_RMV)) 3386 port->dev_conf.intr_conf.rmv = 1; 3387 } 3388 } 3389 3390 void set_port_slave_flag(portid_t slave_pid) 3391 { 3392 struct rte_port *port; 3393 3394 port = &ports[slave_pid]; 3395 port->slave_flag = 1; 3396 } 3397 3398 void clear_port_slave_flag(portid_t slave_pid) 3399 { 3400 struct rte_port *port; 3401 3402 port = &ports[slave_pid]; 3403 port->slave_flag = 0; 3404 } 3405 3406 uint8_t port_is_bonding_slave(portid_t slave_pid) 3407 { 3408 struct rte_port *port; 3409 3410 port = &ports[slave_pid]; 3411 if ((rte_eth_devices[slave_pid].data->dev_flags & 3412 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3413 return 1; 3414 return 0; 3415 } 3416 3417 const uint16_t vlan_tags[] = { 3418 0, 1, 2, 3, 4, 5, 6, 7, 3419 8, 9, 10, 11, 12, 13, 14, 15, 3420 16, 17, 18, 19, 20, 21, 22, 23, 3421 24, 25, 26, 27, 28, 29, 30, 31 3422 }; 3423 3424 static int 3425 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3426 enum dcb_mode_enable dcb_mode, 3427 enum rte_eth_nb_tcs num_tcs, 3428 uint8_t pfc_en) 3429 { 3430 uint8_t i; 3431 int32_t rc; 3432 struct rte_eth_rss_conf rss_conf; 3433 3434 /* 3435 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3436 * given above, and the number of traffic classes available for use. 3437 */ 3438 if (dcb_mode == DCB_VT_ENABLED) { 3439 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3440 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3441 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3442 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3443 3444 /* VMDQ+DCB RX and TX configurations */ 3445 vmdq_rx_conf->enable_default_pool = 0; 3446 vmdq_rx_conf->default_pool = 0; 3447 vmdq_rx_conf->nb_queue_pools = 3448 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3449 vmdq_tx_conf->nb_queue_pools = 3450 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3451 3452 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3453 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3454 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3455 vmdq_rx_conf->pool_map[i].pools = 3456 1 << (i % vmdq_rx_conf->nb_queue_pools); 3457 } 3458 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3459 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3460 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3461 } 3462 3463 /* set DCB mode of RX and TX of multiple queues */ 3464 eth_conf->rxmode.mq_mode = 3465 (enum rte_eth_rx_mq_mode) 3466 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB); 3467 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3468 } else { 3469 struct rte_eth_dcb_rx_conf *rx_conf = 3470 ð_conf->rx_adv_conf.dcb_rx_conf; 3471 struct rte_eth_dcb_tx_conf *tx_conf = 3472 ð_conf->tx_adv_conf.dcb_tx_conf; 3473 3474 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf)); 3475 3476 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3477 if (rc != 0) 3478 return rc; 3479 3480 rx_conf->nb_tcs = num_tcs; 3481 tx_conf->nb_tcs = num_tcs; 3482 3483 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3484 rx_conf->dcb_tc[i] = i % num_tcs; 3485 tx_conf->dcb_tc[i] = i % num_tcs; 3486 } 3487 3488 eth_conf->rxmode.mq_mode = 3489 (enum rte_eth_rx_mq_mode) 3490 (rx_mq_mode & ETH_MQ_RX_DCB_RSS); 3491 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3492 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3493 } 3494 3495 if (pfc_en) 3496 eth_conf->dcb_capability_en = 3497 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3498 else 3499 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3500 3501 return 0; 3502 } 3503 3504 int 3505 init_port_dcb_config(portid_t pid, 3506 enum dcb_mode_enable dcb_mode, 3507 enum rte_eth_nb_tcs num_tcs, 3508 uint8_t pfc_en) 3509 { 3510 struct rte_eth_conf port_conf; 3511 struct rte_port *rte_port; 3512 int retval; 3513 uint16_t i; 3514 3515 rte_port = &ports[pid]; 3516 3517 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3518 /* Enter DCB configuration status */ 3519 dcb_config = 1; 3520 3521 port_conf.rxmode = rte_port->dev_conf.rxmode; 3522 port_conf.txmode = rte_port->dev_conf.txmode; 3523 3524 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3525 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3526 if (retval < 0) 3527 return retval; 3528 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3529 3530 /* re-configure the device . */ 3531 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3532 if (retval < 0) 3533 return retval; 3534 3535 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3536 if (retval != 0) 3537 return retval; 3538 3539 /* If dev_info.vmdq_pool_base is greater than 0, 3540 * the queue id of vmdq pools is started after pf queues. 3541 */ 3542 if (dcb_mode == DCB_VT_ENABLED && 3543 rte_port->dev_info.vmdq_pool_base > 0) { 3544 printf("VMDQ_DCB multi-queue mode is nonsensical" 3545 " for port %d.", pid); 3546 return -1; 3547 } 3548 3549 /* Assume the ports in testpmd have the same dcb capability 3550 * and has the same number of rxq and txq in dcb mode 3551 */ 3552 if (dcb_mode == DCB_VT_ENABLED) { 3553 if (rte_port->dev_info.max_vfs > 0) { 3554 nb_rxq = rte_port->dev_info.nb_rx_queues; 3555 nb_txq = rte_port->dev_info.nb_tx_queues; 3556 } else { 3557 nb_rxq = rte_port->dev_info.max_rx_queues; 3558 nb_txq = rte_port->dev_info.max_tx_queues; 3559 } 3560 } else { 3561 /*if vt is disabled, use all pf queues */ 3562 if (rte_port->dev_info.vmdq_pool_base == 0) { 3563 nb_rxq = rte_port->dev_info.max_rx_queues; 3564 nb_txq = rte_port->dev_info.max_tx_queues; 3565 } else { 3566 nb_rxq = (queueid_t)num_tcs; 3567 nb_txq = (queueid_t)num_tcs; 3568 3569 } 3570 } 3571 rx_free_thresh = 64; 3572 3573 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3574 3575 rxtx_port_config(rte_port); 3576 /* VLAN filter */ 3577 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3578 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3579 rx_vft_set(pid, vlan_tags[i], 1); 3580 3581 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3582 if (retval != 0) 3583 return retval; 3584 3585 map_port_queue_stats_mapping_registers(pid, rte_port); 3586 3587 rte_port->dcb_flag = 1; 3588 3589 return 0; 3590 } 3591 3592 static void 3593 init_port(void) 3594 { 3595 /* Configuration of Ethernet ports. */ 3596 ports = rte_zmalloc("testpmd: ports", 3597 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3598 RTE_CACHE_LINE_SIZE); 3599 if (ports == NULL) { 3600 rte_exit(EXIT_FAILURE, 3601 "rte_zmalloc(%d struct rte_port) failed\n", 3602 RTE_MAX_ETHPORTS); 3603 } 3604 3605 /* Initialize ports NUMA structures */ 3606 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3607 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3608 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3609 } 3610 3611 static void 3612 force_quit(void) 3613 { 3614 pmd_test_exit(); 3615 prompt_exit(); 3616 } 3617 3618 static void 3619 print_stats(void) 3620 { 3621 uint8_t i; 3622 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3623 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3624 3625 /* Clear screen and move to top left */ 3626 printf("%s%s", clr, top_left); 3627 3628 printf("\nPort statistics ===================================="); 3629 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3630 nic_stats_display(fwd_ports_ids[i]); 3631 3632 fflush(stdout); 3633 } 3634 3635 static void 3636 signal_handler(int signum) 3637 { 3638 if (signum == SIGINT || signum == SIGTERM) { 3639 printf("\nSignal %d received, preparing to exit...\n", 3640 signum); 3641 #ifdef RTE_LIBRTE_PDUMP 3642 /* uninitialize packet capture framework */ 3643 rte_pdump_uninit(); 3644 #endif 3645 #ifdef RTE_LIBRTE_LATENCY_STATS 3646 if (latencystats_enabled != 0) 3647 rte_latencystats_uninit(); 3648 #endif 3649 force_quit(); 3650 /* Set flag to indicate the force termination. */ 3651 f_quit = 1; 3652 /* exit with the expected status */ 3653 signal(signum, SIG_DFL); 3654 kill(getpid(), signum); 3655 } 3656 } 3657 3658 int 3659 main(int argc, char** argv) 3660 { 3661 int diag; 3662 portid_t port_id; 3663 uint16_t count; 3664 int ret; 3665 3666 signal(SIGINT, signal_handler); 3667 signal(SIGTERM, signal_handler); 3668 3669 testpmd_logtype = rte_log_register("testpmd"); 3670 if (testpmd_logtype < 0) 3671 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3672 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3673 3674 diag = rte_eal_init(argc, argv); 3675 if (diag < 0) 3676 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3677 rte_strerror(rte_errno)); 3678 3679 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3680 rte_exit(EXIT_FAILURE, 3681 "Secondary process type not supported.\n"); 3682 3683 ret = register_eth_event_callback(); 3684 if (ret != 0) 3685 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3686 3687 #ifdef RTE_LIBRTE_PDUMP 3688 /* initialize packet capture framework */ 3689 rte_pdump_init(); 3690 #endif 3691 3692 count = 0; 3693 RTE_ETH_FOREACH_DEV(port_id) { 3694 ports_ids[count] = port_id; 3695 count++; 3696 } 3697 nb_ports = (portid_t) count; 3698 if (nb_ports == 0) 3699 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3700 3701 /* allocate port structures, and init them */ 3702 init_port(); 3703 3704 set_def_fwd_config(); 3705 if (nb_lcores == 0) 3706 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3707 "Check the core mask argument\n"); 3708 3709 /* Bitrate/latency stats disabled by default */ 3710 #ifdef RTE_LIBRTE_BITRATE 3711 bitrate_enabled = 0; 3712 #endif 3713 #ifdef RTE_LIBRTE_LATENCY_STATS 3714 latencystats_enabled = 0; 3715 #endif 3716 3717 /* on FreeBSD, mlockall() is disabled by default */ 3718 #ifdef RTE_EXEC_ENV_FREEBSD 3719 do_mlockall = 0; 3720 #else 3721 do_mlockall = 1; 3722 #endif 3723 3724 argc -= diag; 3725 argv += diag; 3726 if (argc > 1) 3727 launch_args_parse(argc, argv); 3728 3729 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3730 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3731 strerror(errno)); 3732 } 3733 3734 if (tx_first && interactive) 3735 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3736 "interactive mode.\n"); 3737 3738 if (tx_first && lsc_interrupt) { 3739 printf("Warning: lsc_interrupt needs to be off when " 3740 " using tx_first. Disabling.\n"); 3741 lsc_interrupt = 0; 3742 } 3743 3744 if (!nb_rxq && !nb_txq) 3745 printf("Warning: Either rx or tx queues should be non-zero\n"); 3746 3747 if (nb_rxq > 1 && nb_rxq > nb_txq) 3748 printf("Warning: nb_rxq=%d enables RSS configuration, " 3749 "but nb_txq=%d will prevent to fully test it.\n", 3750 nb_rxq, nb_txq); 3751 3752 init_config(); 3753 3754 if (hot_plug) { 3755 ret = rte_dev_hotplug_handle_enable(); 3756 if (ret) { 3757 RTE_LOG(ERR, EAL, 3758 "fail to enable hotplug handling."); 3759 return -1; 3760 } 3761 3762 ret = rte_dev_event_monitor_start(); 3763 if (ret) { 3764 RTE_LOG(ERR, EAL, 3765 "fail to start device event monitoring."); 3766 return -1; 3767 } 3768 3769 ret = rte_dev_event_callback_register(NULL, 3770 dev_event_callback, NULL); 3771 if (ret) { 3772 RTE_LOG(ERR, EAL, 3773 "fail to register device event callback\n"); 3774 return -1; 3775 } 3776 } 3777 3778 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3779 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3780 3781 /* set all ports to promiscuous mode by default */ 3782 RTE_ETH_FOREACH_DEV(port_id) { 3783 ret = rte_eth_promiscuous_enable(port_id); 3784 if (ret != 0) 3785 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3786 port_id, rte_strerror(-ret)); 3787 } 3788 3789 /* Init metrics library */ 3790 rte_metrics_init(rte_socket_id()); 3791 3792 #ifdef RTE_LIBRTE_LATENCY_STATS 3793 if (latencystats_enabled != 0) { 3794 int ret = rte_latencystats_init(1, NULL); 3795 if (ret) 3796 printf("Warning: latencystats init()" 3797 " returned error %d\n", ret); 3798 printf("Latencystats running on lcore %d\n", 3799 latencystats_lcore_id); 3800 } 3801 #endif 3802 3803 /* Setup bitrate stats */ 3804 #ifdef RTE_LIBRTE_BITRATE 3805 if (bitrate_enabled != 0) { 3806 bitrate_data = rte_stats_bitrate_create(); 3807 if (bitrate_data == NULL) 3808 rte_exit(EXIT_FAILURE, 3809 "Could not allocate bitrate data.\n"); 3810 rte_stats_bitrate_reg(bitrate_data); 3811 } 3812 #endif 3813 3814 #ifdef RTE_LIBRTE_CMDLINE 3815 if (strlen(cmdline_filename) != 0) 3816 cmdline_read_from_file(cmdline_filename); 3817 3818 if (interactive == 1) { 3819 if (auto_start) { 3820 printf("Start automatic packet forwarding\n"); 3821 start_packet_forwarding(0); 3822 } 3823 prompt(); 3824 pmd_test_exit(); 3825 } else 3826 #endif 3827 { 3828 char c; 3829 int rc; 3830 3831 f_quit = 0; 3832 3833 printf("No commandline core given, start packet forwarding\n"); 3834 start_packet_forwarding(tx_first); 3835 if (stats_period != 0) { 3836 uint64_t prev_time = 0, cur_time, diff_time = 0; 3837 uint64_t timer_period; 3838 3839 /* Convert to number of cycles */ 3840 timer_period = stats_period * rte_get_timer_hz(); 3841 3842 while (f_quit == 0) { 3843 cur_time = rte_get_timer_cycles(); 3844 diff_time += cur_time - prev_time; 3845 3846 if (diff_time >= timer_period) { 3847 print_stats(); 3848 /* Reset the timer */ 3849 diff_time = 0; 3850 } 3851 /* Sleep to avoid unnecessary checks */ 3852 prev_time = cur_time; 3853 sleep(1); 3854 } 3855 } 3856 3857 printf("Press enter to exit\n"); 3858 rc = read(0, &c, 1); 3859 pmd_test_exit(); 3860 if (rc < 0) 3861 return 1; 3862 } 3863 3864 ret = rte_eal_cleanup(); 3865 if (ret != 0) 3866 rte_exit(EXIT_FAILURE, 3867 "EAL cleanup failed: %s\n", strerror(-ret)); 3868 3869 return EXIT_SUCCESS; 3870 } 3871