1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 82 uint16_t verbose_level = 0; /**< Silent by default. */ 83 int testpmd_logtype; /**< Log type for testpmd logs */ 84 85 /* use master core for command line ? */ 86 uint8_t interactive = 0; 87 uint8_t auto_start = 0; 88 uint8_t tx_first; 89 char cmdline_filename[PATH_MAX] = {0}; 90 91 /* 92 * NUMA support configuration. 93 * When set, the NUMA support attempts to dispatch the allocation of the 94 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 95 * probed ports among the CPU sockets 0 and 1. 96 * Otherwise, all memory is allocated from CPU socket 0. 97 */ 98 uint8_t numa_support = 1; /**< numa enabled by default */ 99 100 /* 101 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 102 * not configured. 103 */ 104 uint8_t socket_num = UMA_NO_CONFIG; 105 106 /* 107 * Select mempool allocation type: 108 * - native: use regular DPDK memory 109 * - anon: use regular DPDK memory to create mempool, but populate using 110 * anonymous memory (may not be IOVA-contiguous) 111 * - xmem: use externally allocated hugepage memory 112 */ 113 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 114 115 /* 116 * Store specified sockets on which memory pool to be used by ports 117 * is allocated. 118 */ 119 uint8_t port_numa[RTE_MAX_ETHPORTS]; 120 121 /* 122 * Store specified sockets on which RX ring to be used by ports 123 * is allocated. 124 */ 125 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 126 127 /* 128 * Store specified sockets on which TX ring to be used by ports 129 * is allocated. 130 */ 131 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 132 133 /* 134 * Record the Ethernet address of peer target ports to which packets are 135 * forwarded. 136 * Must be instantiated with the ethernet addresses of peer traffic generator 137 * ports. 138 */ 139 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 140 portid_t nb_peer_eth_addrs = 0; 141 142 /* 143 * Probed Target Environment. 144 */ 145 struct rte_port *ports; /**< For all probed ethernet ports. */ 146 portid_t nb_ports; /**< Number of probed ethernet ports. */ 147 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 148 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 149 150 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 151 152 /* 153 * Test Forwarding Configuration. 154 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 155 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 156 */ 157 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 158 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 159 portid_t nb_cfg_ports; /**< Number of configured ports. */ 160 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 161 162 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 163 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 164 165 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 166 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 167 168 /* 169 * Forwarding engines. 170 */ 171 struct fwd_engine * fwd_engines[] = { 172 &io_fwd_engine, 173 &mac_fwd_engine, 174 &mac_swap_engine, 175 &flow_gen_engine, 176 &rx_only_engine, 177 &tx_only_engine, 178 &csum_fwd_engine, 179 &icmp_echo_engine, 180 &noisy_vnf_engine, 181 #if defined RTE_LIBRTE_PMD_SOFTNIC 182 &softnic_fwd_engine, 183 #endif 184 #ifdef RTE_LIBRTE_IEEE1588 185 &ieee1588_fwd_engine, 186 #endif 187 NULL, 188 }; 189 190 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 191 uint16_t mempool_flags; 192 193 struct fwd_config cur_fwd_config; 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 195 uint32_t retry_enabled; 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 198 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 200 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 201 * specified on command-line. */ 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 203 204 /* 205 * In container, it cannot terminate the process which running with 'stats-period' 206 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 207 */ 208 uint8_t f_quit; 209 210 /* 211 * Configuration of packet segments used by the "txonly" processing engine. 212 */ 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 215 TXONLY_DEF_PACKET_LEN, 216 }; 217 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 218 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 220 /**< Split policy for packets to TX. */ 221 222 uint8_t txonly_multi_flow; 223 /**< Whether multiple flows are generated in TXONLY mode. */ 224 225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 227 228 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 229 uint8_t dcb_config = 0; 230 231 /* Whether the dcb is in testing status */ 232 uint8_t dcb_test = 0; 233 234 /* 235 * Configurable number of RX/TX queues. 236 */ 237 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Don't automatically start all ports in interactive mode. 344 */ 345 uint8_t no_device_start = 0; 346 347 /* 348 * Enable link status change notification 349 */ 350 uint8_t lsc_interrupt = 1; /* enabled by default */ 351 352 /* 353 * Enable device removal notification. 354 */ 355 uint8_t rmv_interrupt = 1; /* enabled by default */ 356 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 358 359 /* After attach, port setup is called on event or by iterator */ 360 bool setup_on_probe_event = true; 361 362 /* Clear ptypes on port initialization. */ 363 uint8_t clear_ptypes = true; 364 365 /* Pretty printing of ethdev events */ 366 static const char * const eth_event_desc[] = { 367 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 368 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 369 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 370 [RTE_ETH_EVENT_INTR_RESET] = "reset", 371 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 372 [RTE_ETH_EVENT_IPSEC] = "IPsec", 373 [RTE_ETH_EVENT_MACSEC] = "MACsec", 374 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 375 [RTE_ETH_EVENT_NEW] = "device probed", 376 [RTE_ETH_EVENT_DESTROY] = "device released", 377 [RTE_ETH_EVENT_MAX] = NULL, 378 }; 379 380 /* 381 * Display or mask ether events 382 * Default to all events except VF_MBOX 383 */ 384 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 385 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 386 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 387 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 388 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 389 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 390 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 391 /* 392 * Decide if all memory are locked for performance. 393 */ 394 int do_mlockall = 0; 395 396 /* 397 * NIC bypass mode configuration options. 398 */ 399 400 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 401 /* The NIC bypass watchdog timeout. */ 402 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 403 #endif 404 405 406 #ifdef RTE_LIBRTE_LATENCY_STATS 407 408 /* 409 * Set when latency stats is enabled in the commandline 410 */ 411 uint8_t latencystats_enabled; 412 413 /* 414 * Lcore ID to serive latency statistics. 415 */ 416 lcoreid_t latencystats_lcore_id = -1; 417 418 #endif 419 420 /* 421 * Ethernet device configuration. 422 */ 423 struct rte_eth_rxmode rx_mode = { 424 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 425 /**< Default maximum frame length. */ 426 }; 427 428 struct rte_eth_txmode tx_mode = { 429 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 430 }; 431 432 struct rte_fdir_conf fdir_conf = { 433 .mode = RTE_FDIR_MODE_NONE, 434 .pballoc = RTE_FDIR_PBALLOC_64K, 435 .status = RTE_FDIR_REPORT_STATUS, 436 .mask = { 437 .vlan_tci_mask = 0xFFEF, 438 .ipv4_mask = { 439 .src_ip = 0xFFFFFFFF, 440 .dst_ip = 0xFFFFFFFF, 441 }, 442 .ipv6_mask = { 443 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 444 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 445 }, 446 .src_port_mask = 0xFFFF, 447 .dst_port_mask = 0xFFFF, 448 .mac_addr_byte_mask = 0xFF, 449 .tunnel_type_mask = 1, 450 .tunnel_id_mask = 0xFFFFFFFF, 451 }, 452 .drop_queue = 127, 453 }; 454 455 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 456 457 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 458 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 459 460 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 461 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 462 463 uint16_t nb_tx_queue_stats_mappings = 0; 464 uint16_t nb_rx_queue_stats_mappings = 0; 465 466 /* 467 * Display zero values by default for xstats 468 */ 469 uint8_t xstats_hide_zero; 470 471 unsigned int num_sockets = 0; 472 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 473 474 #ifdef RTE_LIBRTE_BITRATE 475 /* Bitrate statistics */ 476 struct rte_stats_bitrates *bitrate_data; 477 lcoreid_t bitrate_lcore_id; 478 uint8_t bitrate_enabled; 479 #endif 480 481 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 482 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 483 484 /* Forward function declarations */ 485 static void setup_attached_port(portid_t pi); 486 static void map_port_queue_stats_mapping_registers(portid_t pi, 487 struct rte_port *port); 488 static void check_all_ports_link_status(uint32_t port_mask); 489 static int eth_event_callback(portid_t port_id, 490 enum rte_eth_event_type type, 491 void *param, void *ret_param); 492 static void dev_event_callback(const char *device_name, 493 enum rte_dev_event_type type, 494 void *param); 495 496 /* 497 * Check if all the ports are started. 498 * If yes, return positive value. If not, return zero. 499 */ 500 static int all_ports_started(void); 501 502 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 503 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 504 505 /* 506 * Helper function to check if socket is already discovered. 507 * If yes, return positive value. If not, return zero. 508 */ 509 int 510 new_socket_id(unsigned int socket_id) 511 { 512 unsigned int i; 513 514 for (i = 0; i < num_sockets; i++) { 515 if (socket_ids[i] == socket_id) 516 return 0; 517 } 518 return 1; 519 } 520 521 /* 522 * Setup default configuration. 523 */ 524 static void 525 set_default_fwd_lcores_config(void) 526 { 527 unsigned int i; 528 unsigned int nb_lc; 529 unsigned int sock_num; 530 531 nb_lc = 0; 532 for (i = 0; i < RTE_MAX_LCORE; i++) { 533 if (!rte_lcore_is_enabled(i)) 534 continue; 535 sock_num = rte_lcore_to_socket_id(i); 536 if (new_socket_id(sock_num)) { 537 if (num_sockets >= RTE_MAX_NUMA_NODES) { 538 rte_exit(EXIT_FAILURE, 539 "Total sockets greater than %u\n", 540 RTE_MAX_NUMA_NODES); 541 } 542 socket_ids[num_sockets++] = sock_num; 543 } 544 if (i == rte_get_master_lcore()) 545 continue; 546 fwd_lcores_cpuids[nb_lc++] = i; 547 } 548 nb_lcores = (lcoreid_t) nb_lc; 549 nb_cfg_lcores = nb_lcores; 550 nb_fwd_lcores = 1; 551 } 552 553 static void 554 set_def_peer_eth_addrs(void) 555 { 556 portid_t i; 557 558 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 559 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 560 peer_eth_addrs[i].addr_bytes[5] = i; 561 } 562 } 563 564 static void 565 set_default_fwd_ports_config(void) 566 { 567 portid_t pt_id; 568 int i = 0; 569 570 RTE_ETH_FOREACH_DEV(pt_id) { 571 fwd_ports_ids[i++] = pt_id; 572 573 /* Update sockets info according to the attached device */ 574 int socket_id = rte_eth_dev_socket_id(pt_id); 575 if (socket_id >= 0 && new_socket_id(socket_id)) { 576 if (num_sockets >= RTE_MAX_NUMA_NODES) { 577 rte_exit(EXIT_FAILURE, 578 "Total sockets greater than %u\n", 579 RTE_MAX_NUMA_NODES); 580 } 581 socket_ids[num_sockets++] = socket_id; 582 } 583 } 584 585 nb_cfg_ports = nb_ports; 586 nb_fwd_ports = nb_ports; 587 } 588 589 void 590 set_def_fwd_config(void) 591 { 592 set_default_fwd_lcores_config(); 593 set_def_peer_eth_addrs(); 594 set_default_fwd_ports_config(); 595 } 596 597 /* extremely pessimistic estimation of memory required to create a mempool */ 598 static int 599 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 600 { 601 unsigned int n_pages, mbuf_per_pg, leftover; 602 uint64_t total_mem, mbuf_mem, obj_sz; 603 604 /* there is no good way to predict how much space the mempool will 605 * occupy because it will allocate chunks on the fly, and some of those 606 * will come from default DPDK memory while some will come from our 607 * external memory, so just assume 128MB will be enough for everyone. 608 */ 609 uint64_t hdr_mem = 128 << 20; 610 611 /* account for possible non-contiguousness */ 612 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 613 if (obj_sz > pgsz) { 614 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 615 return -1; 616 } 617 618 mbuf_per_pg = pgsz / obj_sz; 619 leftover = (nb_mbufs % mbuf_per_pg) > 0; 620 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 621 622 mbuf_mem = n_pages * pgsz; 623 624 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 625 626 if (total_mem > SIZE_MAX) { 627 TESTPMD_LOG(ERR, "Memory size too big\n"); 628 return -1; 629 } 630 *out = (size_t)total_mem; 631 632 return 0; 633 } 634 635 static int 636 pagesz_flags(uint64_t page_sz) 637 { 638 /* as per mmap() manpage, all page sizes are log2 of page size 639 * shifted by MAP_HUGE_SHIFT 640 */ 641 int log2 = rte_log2_u64(page_sz); 642 643 return (log2 << HUGE_SHIFT); 644 } 645 646 static void * 647 alloc_mem(size_t memsz, size_t pgsz, bool huge) 648 { 649 void *addr; 650 int flags; 651 652 /* allocate anonymous hugepages */ 653 flags = MAP_ANONYMOUS | MAP_PRIVATE; 654 if (huge) 655 flags |= HUGE_FLAG | pagesz_flags(pgsz); 656 657 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 658 if (addr == MAP_FAILED) 659 return NULL; 660 661 return addr; 662 } 663 664 struct extmem_param { 665 void *addr; 666 size_t len; 667 size_t pgsz; 668 rte_iova_t *iova_table; 669 unsigned int iova_table_len; 670 }; 671 672 static int 673 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 674 bool huge) 675 { 676 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 677 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 678 unsigned int cur_page, n_pages, pgsz_idx; 679 size_t mem_sz, cur_pgsz; 680 rte_iova_t *iovas = NULL; 681 void *addr; 682 int ret; 683 684 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 685 /* skip anything that is too big */ 686 if (pgsizes[pgsz_idx] > SIZE_MAX) 687 continue; 688 689 cur_pgsz = pgsizes[pgsz_idx]; 690 691 /* if we were told not to allocate hugepages, override */ 692 if (!huge) 693 cur_pgsz = sysconf(_SC_PAGESIZE); 694 695 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 696 if (ret < 0) { 697 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 698 return -1; 699 } 700 701 /* allocate our memory */ 702 addr = alloc_mem(mem_sz, cur_pgsz, huge); 703 704 /* if we couldn't allocate memory with a specified page size, 705 * that doesn't mean we can't do it with other page sizes, so 706 * try another one. 707 */ 708 if (addr == NULL) 709 continue; 710 711 /* store IOVA addresses for every page in this memory area */ 712 n_pages = mem_sz / cur_pgsz; 713 714 iovas = malloc(sizeof(*iovas) * n_pages); 715 716 if (iovas == NULL) { 717 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 718 goto fail; 719 } 720 /* lock memory if it's not huge pages */ 721 if (!huge) 722 mlock(addr, mem_sz); 723 724 /* populate IOVA addresses */ 725 for (cur_page = 0; cur_page < n_pages; cur_page++) { 726 rte_iova_t iova; 727 size_t offset; 728 void *cur; 729 730 offset = cur_pgsz * cur_page; 731 cur = RTE_PTR_ADD(addr, offset); 732 733 /* touch the page before getting its IOVA */ 734 *(volatile char *)cur = 0; 735 736 iova = rte_mem_virt2iova(cur); 737 738 iovas[cur_page] = iova; 739 } 740 741 break; 742 } 743 /* if we couldn't allocate anything */ 744 if (iovas == NULL) 745 return -1; 746 747 param->addr = addr; 748 param->len = mem_sz; 749 param->pgsz = cur_pgsz; 750 param->iova_table = iovas; 751 param->iova_table_len = n_pages; 752 753 return 0; 754 fail: 755 if (iovas) 756 free(iovas); 757 if (addr) 758 munmap(addr, mem_sz); 759 760 return -1; 761 } 762 763 static int 764 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 765 { 766 struct extmem_param param; 767 int socket_id, ret; 768 769 memset(¶m, 0, sizeof(param)); 770 771 /* check if our heap exists */ 772 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 773 if (socket_id < 0) { 774 /* create our heap */ 775 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 776 if (ret < 0) { 777 TESTPMD_LOG(ERR, "Cannot create heap\n"); 778 return -1; 779 } 780 } 781 782 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 783 if (ret < 0) { 784 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 785 return -1; 786 } 787 788 /* we now have a valid memory area, so add it to heap */ 789 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 790 param.addr, param.len, param.iova_table, 791 param.iova_table_len, param.pgsz); 792 793 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 794 795 /* not needed any more */ 796 free(param.iova_table); 797 798 if (ret < 0) { 799 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 800 munmap(param.addr, param.len); 801 return -1; 802 } 803 804 /* success */ 805 806 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 807 param.len >> 20); 808 809 return 0; 810 } 811 static void 812 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 813 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 814 { 815 uint16_t pid = 0; 816 int ret; 817 818 RTE_ETH_FOREACH_DEV(pid) { 819 struct rte_eth_dev *dev = 820 &rte_eth_devices[pid]; 821 822 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 823 memhdr->len); 824 if (ret) { 825 TESTPMD_LOG(DEBUG, 826 "unable to DMA unmap addr 0x%p " 827 "for device %s\n", 828 memhdr->addr, dev->data->name); 829 } 830 } 831 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 832 if (ret) { 833 TESTPMD_LOG(DEBUG, 834 "unable to un-register addr 0x%p\n", memhdr->addr); 835 } 836 } 837 838 static void 839 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 840 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 841 { 842 uint16_t pid = 0; 843 size_t page_size = sysconf(_SC_PAGESIZE); 844 int ret; 845 846 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 847 page_size); 848 if (ret) { 849 TESTPMD_LOG(DEBUG, 850 "unable to register addr 0x%p\n", memhdr->addr); 851 return; 852 } 853 RTE_ETH_FOREACH_DEV(pid) { 854 struct rte_eth_dev *dev = 855 &rte_eth_devices[pid]; 856 857 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 858 memhdr->len); 859 if (ret) { 860 TESTPMD_LOG(DEBUG, 861 "unable to DMA map addr 0x%p " 862 "for device %s\n", 863 memhdr->addr, dev->data->name); 864 } 865 } 866 } 867 868 /* 869 * Configuration initialisation done once at init time. 870 */ 871 static struct rte_mempool * 872 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 873 unsigned int socket_id) 874 { 875 char pool_name[RTE_MEMPOOL_NAMESIZE]; 876 struct rte_mempool *rte_mp = NULL; 877 uint32_t mb_size; 878 879 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 880 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 881 882 TESTPMD_LOG(INFO, 883 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 884 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 885 886 switch (mp_alloc_type) { 887 case MP_ALLOC_NATIVE: 888 { 889 /* wrapper to rte_mempool_create() */ 890 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 891 rte_mbuf_best_mempool_ops()); 892 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 893 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 894 break; 895 } 896 case MP_ALLOC_ANON: 897 { 898 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 899 mb_size, (unsigned int) mb_mempool_cache, 900 sizeof(struct rte_pktmbuf_pool_private), 901 socket_id, mempool_flags); 902 if (rte_mp == NULL) 903 goto err; 904 905 if (rte_mempool_populate_anon(rte_mp) == 0) { 906 rte_mempool_free(rte_mp); 907 rte_mp = NULL; 908 goto err; 909 } 910 rte_pktmbuf_pool_init(rte_mp, NULL); 911 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 912 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 913 break; 914 } 915 case MP_ALLOC_XMEM: 916 case MP_ALLOC_XMEM_HUGE: 917 { 918 int heap_socket; 919 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 920 921 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 922 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 923 924 heap_socket = 925 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 926 if (heap_socket < 0) 927 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 928 929 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 930 rte_mbuf_best_mempool_ops()); 931 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 932 mb_mempool_cache, 0, mbuf_seg_size, 933 heap_socket); 934 break; 935 } 936 default: 937 { 938 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 939 } 940 } 941 942 err: 943 if (rte_mp == NULL) { 944 rte_exit(EXIT_FAILURE, 945 "Creation of mbuf pool for socket %u failed: %s\n", 946 socket_id, rte_strerror(rte_errno)); 947 } else if (verbose_level > 0) { 948 rte_mempool_dump(stdout, rte_mp); 949 } 950 return rte_mp; 951 } 952 953 /* 954 * Check given socket id is valid or not with NUMA mode, 955 * if valid, return 0, else return -1 956 */ 957 static int 958 check_socket_id(const unsigned int socket_id) 959 { 960 static int warning_once = 0; 961 962 if (new_socket_id(socket_id)) { 963 if (!warning_once && numa_support) 964 printf("Warning: NUMA should be configured manually by" 965 " using --port-numa-config and" 966 " --ring-numa-config parameters along with" 967 " --numa.\n"); 968 warning_once = 1; 969 return -1; 970 } 971 return 0; 972 } 973 974 /* 975 * Get the allowed maximum number of RX queues. 976 * *pid return the port id which has minimal value of 977 * max_rx_queues in all ports. 978 */ 979 queueid_t 980 get_allowed_max_nb_rxq(portid_t *pid) 981 { 982 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 983 bool max_rxq_valid = false; 984 portid_t pi; 985 struct rte_eth_dev_info dev_info; 986 987 RTE_ETH_FOREACH_DEV(pi) { 988 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 989 continue; 990 991 max_rxq_valid = true; 992 if (dev_info.max_rx_queues < allowed_max_rxq) { 993 allowed_max_rxq = dev_info.max_rx_queues; 994 *pid = pi; 995 } 996 } 997 return max_rxq_valid ? allowed_max_rxq : 0; 998 } 999 1000 /* 1001 * Check input rxq is valid or not. 1002 * If input rxq is not greater than any of maximum number 1003 * of RX queues of all ports, it is valid. 1004 * if valid, return 0, else return -1 1005 */ 1006 int 1007 check_nb_rxq(queueid_t rxq) 1008 { 1009 queueid_t allowed_max_rxq; 1010 portid_t pid = 0; 1011 1012 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1013 if (rxq > allowed_max_rxq) { 1014 printf("Fail: input rxq (%u) can't be greater " 1015 "than max_rx_queues (%u) of port %u\n", 1016 rxq, 1017 allowed_max_rxq, 1018 pid); 1019 return -1; 1020 } 1021 return 0; 1022 } 1023 1024 /* 1025 * Get the allowed maximum number of TX queues. 1026 * *pid return the port id which has minimal value of 1027 * max_tx_queues in all ports. 1028 */ 1029 queueid_t 1030 get_allowed_max_nb_txq(portid_t *pid) 1031 { 1032 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1033 bool max_txq_valid = false; 1034 portid_t pi; 1035 struct rte_eth_dev_info dev_info; 1036 1037 RTE_ETH_FOREACH_DEV(pi) { 1038 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1039 continue; 1040 1041 max_txq_valid = true; 1042 if (dev_info.max_tx_queues < allowed_max_txq) { 1043 allowed_max_txq = dev_info.max_tx_queues; 1044 *pid = pi; 1045 } 1046 } 1047 return max_txq_valid ? allowed_max_txq : 0; 1048 } 1049 1050 /* 1051 * Check input txq is valid or not. 1052 * If input txq is not greater than any of maximum number 1053 * of TX queues of all ports, it is valid. 1054 * if valid, return 0, else return -1 1055 */ 1056 int 1057 check_nb_txq(queueid_t txq) 1058 { 1059 queueid_t allowed_max_txq; 1060 portid_t pid = 0; 1061 1062 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1063 if (txq > allowed_max_txq) { 1064 printf("Fail: input txq (%u) can't be greater " 1065 "than max_tx_queues (%u) of port %u\n", 1066 txq, 1067 allowed_max_txq, 1068 pid); 1069 return -1; 1070 } 1071 return 0; 1072 } 1073 1074 /* 1075 * Get the allowed maximum number of hairpin queues. 1076 * *pid return the port id which has minimal value of 1077 * max_hairpin_queues in all ports. 1078 */ 1079 queueid_t 1080 get_allowed_max_nb_hairpinq(portid_t *pid) 1081 { 1082 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1083 portid_t pi; 1084 struct rte_eth_hairpin_cap cap; 1085 1086 RTE_ETH_FOREACH_DEV(pi) { 1087 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1088 *pid = pi; 1089 return 0; 1090 } 1091 if (cap.max_nb_queues < allowed_max_hairpinq) { 1092 allowed_max_hairpinq = cap.max_nb_queues; 1093 *pid = pi; 1094 } 1095 } 1096 return allowed_max_hairpinq; 1097 } 1098 1099 /* 1100 * Check input hairpin is valid or not. 1101 * If input hairpin is not greater than any of maximum number 1102 * of hairpin queues of all ports, it is valid. 1103 * if valid, return 0, else return -1 1104 */ 1105 int 1106 check_nb_hairpinq(queueid_t hairpinq) 1107 { 1108 queueid_t allowed_max_hairpinq; 1109 portid_t pid = 0; 1110 1111 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1112 if (hairpinq > allowed_max_hairpinq) { 1113 printf("Fail: input hairpin (%u) can't be greater " 1114 "than max_hairpin_queues (%u) of port %u\n", 1115 hairpinq, allowed_max_hairpinq, pid); 1116 return -1; 1117 } 1118 return 0; 1119 } 1120 1121 static void 1122 init_config(void) 1123 { 1124 portid_t pid; 1125 struct rte_port *port; 1126 struct rte_mempool *mbp; 1127 unsigned int nb_mbuf_per_pool; 1128 lcoreid_t lc_id; 1129 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1130 struct rte_gro_param gro_param; 1131 uint32_t gso_types; 1132 uint16_t data_size; 1133 bool warning = 0; 1134 int k; 1135 int ret; 1136 1137 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1138 1139 /* Configuration of logical cores. */ 1140 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1141 sizeof(struct fwd_lcore *) * nb_lcores, 1142 RTE_CACHE_LINE_SIZE); 1143 if (fwd_lcores == NULL) { 1144 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1145 "failed\n", nb_lcores); 1146 } 1147 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1148 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1149 sizeof(struct fwd_lcore), 1150 RTE_CACHE_LINE_SIZE); 1151 if (fwd_lcores[lc_id] == NULL) { 1152 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1153 "failed\n"); 1154 } 1155 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1156 } 1157 1158 RTE_ETH_FOREACH_DEV(pid) { 1159 port = &ports[pid]; 1160 /* Apply default TxRx configuration for all ports */ 1161 port->dev_conf.txmode = tx_mode; 1162 port->dev_conf.rxmode = rx_mode; 1163 1164 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1165 if (ret != 0) 1166 rte_exit(EXIT_FAILURE, 1167 "rte_eth_dev_info_get() failed\n"); 1168 1169 if (!(port->dev_info.tx_offload_capa & 1170 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1171 port->dev_conf.txmode.offloads &= 1172 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1173 if (numa_support) { 1174 if (port_numa[pid] != NUMA_NO_CONFIG) 1175 port_per_socket[port_numa[pid]]++; 1176 else { 1177 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1178 1179 /* 1180 * if socket_id is invalid, 1181 * set to the first available socket. 1182 */ 1183 if (check_socket_id(socket_id) < 0) 1184 socket_id = socket_ids[0]; 1185 port_per_socket[socket_id]++; 1186 } 1187 } 1188 1189 /* Apply Rx offloads configuration */ 1190 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1191 port->rx_conf[k].offloads = 1192 port->dev_conf.rxmode.offloads; 1193 /* Apply Tx offloads configuration */ 1194 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1195 port->tx_conf[k].offloads = 1196 port->dev_conf.txmode.offloads; 1197 1198 /* set flag to initialize port/queue */ 1199 port->need_reconfig = 1; 1200 port->need_reconfig_queues = 1; 1201 port->tx_metadata = 0; 1202 1203 /* Check for maximum number of segments per MTU. Accordingly 1204 * update the mbuf data size. 1205 */ 1206 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1207 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1208 data_size = rx_mode.max_rx_pkt_len / 1209 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1210 1211 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1212 mbuf_data_size) { 1213 mbuf_data_size = data_size + 1214 RTE_PKTMBUF_HEADROOM; 1215 warning = 1; 1216 } 1217 } 1218 } 1219 1220 if (warning) 1221 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1222 mbuf_data_size); 1223 1224 /* 1225 * Create pools of mbuf. 1226 * If NUMA support is disabled, create a single pool of mbuf in 1227 * socket 0 memory by default. 1228 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1229 * 1230 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1231 * nb_txd can be configured at run time. 1232 */ 1233 if (param_total_num_mbufs) 1234 nb_mbuf_per_pool = param_total_num_mbufs; 1235 else { 1236 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1237 (nb_lcores * mb_mempool_cache) + 1238 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1239 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1240 } 1241 1242 if (numa_support) { 1243 uint8_t i; 1244 1245 for (i = 0; i < num_sockets; i++) 1246 mempools[i] = mbuf_pool_create(mbuf_data_size, 1247 nb_mbuf_per_pool, 1248 socket_ids[i]); 1249 } else { 1250 if (socket_num == UMA_NO_CONFIG) 1251 mempools[0] = mbuf_pool_create(mbuf_data_size, 1252 nb_mbuf_per_pool, 0); 1253 else 1254 mempools[socket_num] = mbuf_pool_create 1255 (mbuf_data_size, 1256 nb_mbuf_per_pool, 1257 socket_num); 1258 } 1259 1260 init_port_config(); 1261 1262 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1263 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1264 /* 1265 * Records which Mbuf pool to use by each logical core, if needed. 1266 */ 1267 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1268 mbp = mbuf_pool_find( 1269 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1270 1271 if (mbp == NULL) 1272 mbp = mbuf_pool_find(0); 1273 fwd_lcores[lc_id]->mbp = mbp; 1274 /* initialize GSO context */ 1275 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1276 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1277 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1278 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1279 RTE_ETHER_CRC_LEN; 1280 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1281 } 1282 1283 /* Configuration of packet forwarding streams. */ 1284 if (init_fwd_streams() < 0) 1285 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1286 1287 fwd_config_setup(); 1288 1289 /* create a gro context for each lcore */ 1290 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1291 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1292 gro_param.max_item_per_flow = MAX_PKT_BURST; 1293 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1294 gro_param.socket_id = rte_lcore_to_socket_id( 1295 fwd_lcores_cpuids[lc_id]); 1296 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1297 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1298 rte_exit(EXIT_FAILURE, 1299 "rte_gro_ctx_create() failed\n"); 1300 } 1301 } 1302 1303 #if defined RTE_LIBRTE_PMD_SOFTNIC 1304 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1305 RTE_ETH_FOREACH_DEV(pid) { 1306 port = &ports[pid]; 1307 const char *driver = port->dev_info.driver_name; 1308 1309 if (strcmp(driver, "net_softnic") == 0) 1310 port->softport.fwd_lcore_arg = fwd_lcores; 1311 } 1312 } 1313 #endif 1314 1315 } 1316 1317 1318 void 1319 reconfig(portid_t new_port_id, unsigned socket_id) 1320 { 1321 struct rte_port *port; 1322 int ret; 1323 1324 /* Reconfiguration of Ethernet ports. */ 1325 port = &ports[new_port_id]; 1326 1327 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1328 if (ret != 0) 1329 return; 1330 1331 /* set flag to initialize port/queue */ 1332 port->need_reconfig = 1; 1333 port->need_reconfig_queues = 1; 1334 port->socket_id = socket_id; 1335 1336 init_port_config(); 1337 } 1338 1339 1340 int 1341 init_fwd_streams(void) 1342 { 1343 portid_t pid; 1344 struct rte_port *port; 1345 streamid_t sm_id, nb_fwd_streams_new; 1346 queueid_t q; 1347 1348 /* set socket id according to numa or not */ 1349 RTE_ETH_FOREACH_DEV(pid) { 1350 port = &ports[pid]; 1351 if (nb_rxq > port->dev_info.max_rx_queues) { 1352 printf("Fail: nb_rxq(%d) is greater than " 1353 "max_rx_queues(%d)\n", nb_rxq, 1354 port->dev_info.max_rx_queues); 1355 return -1; 1356 } 1357 if (nb_txq > port->dev_info.max_tx_queues) { 1358 printf("Fail: nb_txq(%d) is greater than " 1359 "max_tx_queues(%d)\n", nb_txq, 1360 port->dev_info.max_tx_queues); 1361 return -1; 1362 } 1363 if (numa_support) { 1364 if (port_numa[pid] != NUMA_NO_CONFIG) 1365 port->socket_id = port_numa[pid]; 1366 else { 1367 port->socket_id = rte_eth_dev_socket_id(pid); 1368 1369 /* 1370 * if socket_id is invalid, 1371 * set to the first available socket. 1372 */ 1373 if (check_socket_id(port->socket_id) < 0) 1374 port->socket_id = socket_ids[0]; 1375 } 1376 } 1377 else { 1378 if (socket_num == UMA_NO_CONFIG) 1379 port->socket_id = 0; 1380 else 1381 port->socket_id = socket_num; 1382 } 1383 } 1384 1385 q = RTE_MAX(nb_rxq, nb_txq); 1386 if (q == 0) { 1387 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1388 return -1; 1389 } 1390 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1391 if (nb_fwd_streams_new == nb_fwd_streams) 1392 return 0; 1393 /* clear the old */ 1394 if (fwd_streams != NULL) { 1395 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1396 if (fwd_streams[sm_id] == NULL) 1397 continue; 1398 rte_free(fwd_streams[sm_id]); 1399 fwd_streams[sm_id] = NULL; 1400 } 1401 rte_free(fwd_streams); 1402 fwd_streams = NULL; 1403 } 1404 1405 /* init new */ 1406 nb_fwd_streams = nb_fwd_streams_new; 1407 if (nb_fwd_streams) { 1408 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1409 sizeof(struct fwd_stream *) * nb_fwd_streams, 1410 RTE_CACHE_LINE_SIZE); 1411 if (fwd_streams == NULL) 1412 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1413 " (struct fwd_stream *)) failed\n", 1414 nb_fwd_streams); 1415 1416 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1417 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1418 " struct fwd_stream", sizeof(struct fwd_stream), 1419 RTE_CACHE_LINE_SIZE); 1420 if (fwd_streams[sm_id] == NULL) 1421 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1422 "(struct fwd_stream) failed\n"); 1423 } 1424 } 1425 1426 return 0; 1427 } 1428 1429 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1430 static void 1431 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1432 { 1433 unsigned int total_burst; 1434 unsigned int nb_burst; 1435 unsigned int burst_stats[3]; 1436 uint16_t pktnb_stats[3]; 1437 uint16_t nb_pkt; 1438 int burst_percent[3]; 1439 1440 /* 1441 * First compute the total number of packet bursts and the 1442 * two highest numbers of bursts of the same number of packets. 1443 */ 1444 total_burst = 0; 1445 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1446 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1447 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1448 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1449 if (nb_burst == 0) 1450 continue; 1451 total_burst += nb_burst; 1452 if (nb_burst > burst_stats[0]) { 1453 burst_stats[1] = burst_stats[0]; 1454 pktnb_stats[1] = pktnb_stats[0]; 1455 burst_stats[0] = nb_burst; 1456 pktnb_stats[0] = nb_pkt; 1457 } else if (nb_burst > burst_stats[1]) { 1458 burst_stats[1] = nb_burst; 1459 pktnb_stats[1] = nb_pkt; 1460 } 1461 } 1462 if (total_burst == 0) 1463 return; 1464 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1465 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1466 burst_percent[0], (int) pktnb_stats[0]); 1467 if (burst_stats[0] == total_burst) { 1468 printf("]\n"); 1469 return; 1470 } 1471 if (burst_stats[0] + burst_stats[1] == total_burst) { 1472 printf(" + %d%% of %d pkts]\n", 1473 100 - burst_percent[0], pktnb_stats[1]); 1474 return; 1475 } 1476 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1477 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1478 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1479 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1480 return; 1481 } 1482 printf(" + %d%% of %d pkts + %d%% of others]\n", 1483 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1484 } 1485 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1486 1487 static void 1488 fwd_stream_stats_display(streamid_t stream_id) 1489 { 1490 struct fwd_stream *fs; 1491 static const char *fwd_top_stats_border = "-------"; 1492 1493 fs = fwd_streams[stream_id]; 1494 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1495 (fs->fwd_dropped == 0)) 1496 return; 1497 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1498 "TX Port=%2d/Queue=%2d %s\n", 1499 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1500 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1501 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1502 " TX-dropped: %-14"PRIu64, 1503 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1504 1505 /* if checksum mode */ 1506 if (cur_fwd_eng == &csum_fwd_engine) { 1507 printf(" RX- bad IP checksum: %-14"PRIu64 1508 " Rx- bad L4 checksum: %-14"PRIu64 1509 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1510 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1511 fs->rx_bad_outer_l4_csum); 1512 } else { 1513 printf("\n"); 1514 } 1515 1516 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1517 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1518 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1519 #endif 1520 } 1521 1522 void 1523 fwd_stats_display(void) 1524 { 1525 static const char *fwd_stats_border = "----------------------"; 1526 static const char *acc_stats_border = "+++++++++++++++"; 1527 struct { 1528 struct fwd_stream *rx_stream; 1529 struct fwd_stream *tx_stream; 1530 uint64_t tx_dropped; 1531 uint64_t rx_bad_ip_csum; 1532 uint64_t rx_bad_l4_csum; 1533 uint64_t rx_bad_outer_l4_csum; 1534 } ports_stats[RTE_MAX_ETHPORTS]; 1535 uint64_t total_rx_dropped = 0; 1536 uint64_t total_tx_dropped = 0; 1537 uint64_t total_rx_nombuf = 0; 1538 struct rte_eth_stats stats; 1539 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1540 uint64_t fwd_cycles = 0; 1541 #endif 1542 uint64_t total_recv = 0; 1543 uint64_t total_xmit = 0; 1544 struct rte_port *port; 1545 streamid_t sm_id; 1546 portid_t pt_id; 1547 int i; 1548 1549 memset(ports_stats, 0, sizeof(ports_stats)); 1550 1551 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1552 struct fwd_stream *fs = fwd_streams[sm_id]; 1553 1554 if (cur_fwd_config.nb_fwd_streams > 1555 cur_fwd_config.nb_fwd_ports) { 1556 fwd_stream_stats_display(sm_id); 1557 } else { 1558 ports_stats[fs->tx_port].tx_stream = fs; 1559 ports_stats[fs->rx_port].rx_stream = fs; 1560 } 1561 1562 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1563 1564 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1565 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1566 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1567 fs->rx_bad_outer_l4_csum; 1568 1569 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1570 fwd_cycles += fs->core_cycles; 1571 #endif 1572 } 1573 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1574 uint8_t j; 1575 1576 pt_id = fwd_ports_ids[i]; 1577 port = &ports[pt_id]; 1578 1579 rte_eth_stats_get(pt_id, &stats); 1580 stats.ipackets -= port->stats.ipackets; 1581 stats.opackets -= port->stats.opackets; 1582 stats.ibytes -= port->stats.ibytes; 1583 stats.obytes -= port->stats.obytes; 1584 stats.imissed -= port->stats.imissed; 1585 stats.oerrors -= port->stats.oerrors; 1586 stats.rx_nombuf -= port->stats.rx_nombuf; 1587 1588 total_recv += stats.ipackets; 1589 total_xmit += stats.opackets; 1590 total_rx_dropped += stats.imissed; 1591 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1592 total_tx_dropped += stats.oerrors; 1593 total_rx_nombuf += stats.rx_nombuf; 1594 1595 printf("\n %s Forward statistics for port %-2d %s\n", 1596 fwd_stats_border, pt_id, fwd_stats_border); 1597 1598 if (!port->rx_queue_stats_mapping_enabled && 1599 !port->tx_queue_stats_mapping_enabled) { 1600 printf(" RX-packets: %-14"PRIu64 1601 " RX-dropped: %-14"PRIu64 1602 "RX-total: %-"PRIu64"\n", 1603 stats.ipackets, stats.imissed, 1604 stats.ipackets + stats.imissed); 1605 1606 if (cur_fwd_eng == &csum_fwd_engine) 1607 printf(" Bad-ipcsum: %-14"PRIu64 1608 " Bad-l4csum: %-14"PRIu64 1609 "Bad-outer-l4csum: %-14"PRIu64"\n", 1610 ports_stats[pt_id].rx_bad_ip_csum, 1611 ports_stats[pt_id].rx_bad_l4_csum, 1612 ports_stats[pt_id].rx_bad_outer_l4_csum); 1613 if (stats.ierrors + stats.rx_nombuf > 0) { 1614 printf(" RX-error: %-"PRIu64"\n", 1615 stats.ierrors); 1616 printf(" RX-nombufs: %-14"PRIu64"\n", 1617 stats.rx_nombuf); 1618 } 1619 1620 printf(" TX-packets: %-14"PRIu64 1621 " TX-dropped: %-14"PRIu64 1622 "TX-total: %-"PRIu64"\n", 1623 stats.opackets, ports_stats[pt_id].tx_dropped, 1624 stats.opackets + ports_stats[pt_id].tx_dropped); 1625 } else { 1626 printf(" RX-packets: %14"PRIu64 1627 " RX-dropped:%14"PRIu64 1628 " RX-total:%14"PRIu64"\n", 1629 stats.ipackets, stats.imissed, 1630 stats.ipackets + stats.imissed); 1631 1632 if (cur_fwd_eng == &csum_fwd_engine) 1633 printf(" Bad-ipcsum:%14"PRIu64 1634 " Bad-l4csum:%14"PRIu64 1635 " Bad-outer-l4csum: %-14"PRIu64"\n", 1636 ports_stats[pt_id].rx_bad_ip_csum, 1637 ports_stats[pt_id].rx_bad_l4_csum, 1638 ports_stats[pt_id].rx_bad_outer_l4_csum); 1639 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1640 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1641 printf(" RX-nombufs: %14"PRIu64"\n", 1642 stats.rx_nombuf); 1643 } 1644 1645 printf(" TX-packets: %14"PRIu64 1646 " TX-dropped:%14"PRIu64 1647 " TX-total:%14"PRIu64"\n", 1648 stats.opackets, ports_stats[pt_id].tx_dropped, 1649 stats.opackets + ports_stats[pt_id].tx_dropped); 1650 } 1651 1652 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1653 if (ports_stats[pt_id].rx_stream) 1654 pkt_burst_stats_display("RX", 1655 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1656 if (ports_stats[pt_id].tx_stream) 1657 pkt_burst_stats_display("TX", 1658 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1659 #endif 1660 1661 if (port->rx_queue_stats_mapping_enabled) { 1662 printf("\n"); 1663 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1664 printf(" Stats reg %2d RX-packets:%14"PRIu64 1665 " RX-errors:%14"PRIu64 1666 " RX-bytes:%14"PRIu64"\n", 1667 j, stats.q_ipackets[j], 1668 stats.q_errors[j], stats.q_ibytes[j]); 1669 } 1670 printf("\n"); 1671 } 1672 if (port->tx_queue_stats_mapping_enabled) { 1673 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1674 printf(" Stats reg %2d TX-packets:%14"PRIu64 1675 " TX-bytes:%14" 1676 PRIu64"\n", 1677 j, stats.q_opackets[j], 1678 stats.q_obytes[j]); 1679 } 1680 } 1681 1682 printf(" %s--------------------------------%s\n", 1683 fwd_stats_border, fwd_stats_border); 1684 } 1685 1686 printf("\n %s Accumulated forward statistics for all ports" 1687 "%s\n", 1688 acc_stats_border, acc_stats_border); 1689 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1690 "%-"PRIu64"\n" 1691 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1692 "%-"PRIu64"\n", 1693 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1694 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1695 if (total_rx_nombuf > 0) 1696 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1697 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1698 "%s\n", 1699 acc_stats_border, acc_stats_border); 1700 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1701 if (total_recv > 0) 1702 printf("\n CPU cycles/packet=%u (total cycles=" 1703 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1704 (unsigned int)(fwd_cycles / total_recv), 1705 fwd_cycles, total_recv); 1706 #endif 1707 } 1708 1709 void 1710 fwd_stats_reset(void) 1711 { 1712 streamid_t sm_id; 1713 portid_t pt_id; 1714 int i; 1715 1716 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1717 pt_id = fwd_ports_ids[i]; 1718 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1719 } 1720 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1721 struct fwd_stream *fs = fwd_streams[sm_id]; 1722 1723 fs->rx_packets = 0; 1724 fs->tx_packets = 0; 1725 fs->fwd_dropped = 0; 1726 fs->rx_bad_ip_csum = 0; 1727 fs->rx_bad_l4_csum = 0; 1728 fs->rx_bad_outer_l4_csum = 0; 1729 1730 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1731 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1732 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1733 #endif 1734 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1735 fs->core_cycles = 0; 1736 #endif 1737 } 1738 } 1739 1740 static void 1741 flush_fwd_rx_queues(void) 1742 { 1743 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1744 portid_t rxp; 1745 portid_t port_id; 1746 queueid_t rxq; 1747 uint16_t nb_rx; 1748 uint16_t i; 1749 uint8_t j; 1750 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1751 uint64_t timer_period; 1752 1753 /* convert to number of cycles */ 1754 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1755 1756 for (j = 0; j < 2; j++) { 1757 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1758 for (rxq = 0; rxq < nb_rxq; rxq++) { 1759 port_id = fwd_ports_ids[rxp]; 1760 /** 1761 * testpmd can stuck in the below do while loop 1762 * if rte_eth_rx_burst() always returns nonzero 1763 * packets. So timer is added to exit this loop 1764 * after 1sec timer expiry. 1765 */ 1766 prev_tsc = rte_rdtsc(); 1767 do { 1768 nb_rx = rte_eth_rx_burst(port_id, rxq, 1769 pkts_burst, MAX_PKT_BURST); 1770 for (i = 0; i < nb_rx; i++) 1771 rte_pktmbuf_free(pkts_burst[i]); 1772 1773 cur_tsc = rte_rdtsc(); 1774 diff_tsc = cur_tsc - prev_tsc; 1775 timer_tsc += diff_tsc; 1776 } while ((nb_rx > 0) && 1777 (timer_tsc < timer_period)); 1778 timer_tsc = 0; 1779 } 1780 } 1781 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1782 } 1783 } 1784 1785 static void 1786 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1787 { 1788 struct fwd_stream **fsm; 1789 streamid_t nb_fs; 1790 streamid_t sm_id; 1791 #ifdef RTE_LIBRTE_BITRATE 1792 uint64_t tics_per_1sec; 1793 uint64_t tics_datum; 1794 uint64_t tics_current; 1795 uint16_t i, cnt_ports; 1796 1797 cnt_ports = nb_ports; 1798 tics_datum = rte_rdtsc(); 1799 tics_per_1sec = rte_get_timer_hz(); 1800 #endif 1801 fsm = &fwd_streams[fc->stream_idx]; 1802 nb_fs = fc->stream_nb; 1803 do { 1804 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1805 (*pkt_fwd)(fsm[sm_id]); 1806 #ifdef RTE_LIBRTE_BITRATE 1807 if (bitrate_enabled != 0 && 1808 bitrate_lcore_id == rte_lcore_id()) { 1809 tics_current = rte_rdtsc(); 1810 if (tics_current - tics_datum >= tics_per_1sec) { 1811 /* Periodic bitrate calculation */ 1812 for (i = 0; i < cnt_ports; i++) 1813 rte_stats_bitrate_calc(bitrate_data, 1814 ports_ids[i]); 1815 tics_datum = tics_current; 1816 } 1817 } 1818 #endif 1819 #ifdef RTE_LIBRTE_LATENCY_STATS 1820 if (latencystats_enabled != 0 && 1821 latencystats_lcore_id == rte_lcore_id()) 1822 rte_latencystats_update(); 1823 #endif 1824 1825 } while (! fc->stopped); 1826 } 1827 1828 static int 1829 start_pkt_forward_on_core(void *fwd_arg) 1830 { 1831 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1832 cur_fwd_config.fwd_eng->packet_fwd); 1833 return 0; 1834 } 1835 1836 /* 1837 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1838 * Used to start communication flows in network loopback test configurations. 1839 */ 1840 static int 1841 run_one_txonly_burst_on_core(void *fwd_arg) 1842 { 1843 struct fwd_lcore *fwd_lc; 1844 struct fwd_lcore tmp_lcore; 1845 1846 fwd_lc = (struct fwd_lcore *) fwd_arg; 1847 tmp_lcore = *fwd_lc; 1848 tmp_lcore.stopped = 1; 1849 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1850 return 0; 1851 } 1852 1853 /* 1854 * Launch packet forwarding: 1855 * - Setup per-port forwarding context. 1856 * - launch logical cores with their forwarding configuration. 1857 */ 1858 static void 1859 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1860 { 1861 port_fwd_begin_t port_fwd_begin; 1862 unsigned int i; 1863 unsigned int lc_id; 1864 int diag; 1865 1866 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1867 if (port_fwd_begin != NULL) { 1868 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1869 (*port_fwd_begin)(fwd_ports_ids[i]); 1870 } 1871 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1872 lc_id = fwd_lcores_cpuids[i]; 1873 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1874 fwd_lcores[i]->stopped = 0; 1875 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1876 fwd_lcores[i], lc_id); 1877 if (diag != 0) 1878 printf("launch lcore %u failed - diag=%d\n", 1879 lc_id, diag); 1880 } 1881 } 1882 } 1883 1884 /* 1885 * Launch packet forwarding configuration. 1886 */ 1887 void 1888 start_packet_forwarding(int with_tx_first) 1889 { 1890 port_fwd_begin_t port_fwd_begin; 1891 port_fwd_end_t port_fwd_end; 1892 struct rte_port *port; 1893 unsigned int i; 1894 portid_t pt_id; 1895 1896 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1897 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1898 1899 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1900 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1901 1902 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1903 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1904 (!nb_rxq || !nb_txq)) 1905 rte_exit(EXIT_FAILURE, 1906 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1907 cur_fwd_eng->fwd_mode_name); 1908 1909 if (all_ports_started() == 0) { 1910 printf("Not all ports were started\n"); 1911 return; 1912 } 1913 if (test_done == 0) { 1914 printf("Packet forwarding already started\n"); 1915 return; 1916 } 1917 1918 1919 if(dcb_test) { 1920 for (i = 0; i < nb_fwd_ports; i++) { 1921 pt_id = fwd_ports_ids[i]; 1922 port = &ports[pt_id]; 1923 if (!port->dcb_flag) { 1924 printf("In DCB mode, all forwarding ports must " 1925 "be configured in this mode.\n"); 1926 return; 1927 } 1928 } 1929 if (nb_fwd_lcores == 1) { 1930 printf("In DCB mode,the nb forwarding cores " 1931 "should be larger than 1.\n"); 1932 return; 1933 } 1934 } 1935 test_done = 0; 1936 1937 fwd_config_setup(); 1938 1939 if(!no_flush_rx) 1940 flush_fwd_rx_queues(); 1941 1942 pkt_fwd_config_display(&cur_fwd_config); 1943 rxtx_config_display(); 1944 1945 fwd_stats_reset(); 1946 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1947 pt_id = fwd_ports_ids[i]; 1948 port = &ports[pt_id]; 1949 map_port_queue_stats_mapping_registers(pt_id, port); 1950 } 1951 if (with_tx_first) { 1952 port_fwd_begin = tx_only_engine.port_fwd_begin; 1953 if (port_fwd_begin != NULL) { 1954 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1955 (*port_fwd_begin)(fwd_ports_ids[i]); 1956 } 1957 while (with_tx_first--) { 1958 launch_packet_forwarding( 1959 run_one_txonly_burst_on_core); 1960 rte_eal_mp_wait_lcore(); 1961 } 1962 port_fwd_end = tx_only_engine.port_fwd_end; 1963 if (port_fwd_end != NULL) { 1964 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1965 (*port_fwd_end)(fwd_ports_ids[i]); 1966 } 1967 } 1968 launch_packet_forwarding(start_pkt_forward_on_core); 1969 } 1970 1971 void 1972 stop_packet_forwarding(void) 1973 { 1974 port_fwd_end_t port_fwd_end; 1975 lcoreid_t lc_id; 1976 portid_t pt_id; 1977 int i; 1978 1979 if (test_done) { 1980 printf("Packet forwarding not started\n"); 1981 return; 1982 } 1983 printf("Telling cores to stop..."); 1984 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1985 fwd_lcores[lc_id]->stopped = 1; 1986 printf("\nWaiting for lcores to finish...\n"); 1987 rte_eal_mp_wait_lcore(); 1988 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1989 if (port_fwd_end != NULL) { 1990 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1991 pt_id = fwd_ports_ids[i]; 1992 (*port_fwd_end)(pt_id); 1993 } 1994 } 1995 1996 fwd_stats_display(); 1997 1998 printf("\nDone.\n"); 1999 test_done = 1; 2000 } 2001 2002 void 2003 dev_set_link_up(portid_t pid) 2004 { 2005 if (rte_eth_dev_set_link_up(pid) < 0) 2006 printf("\nSet link up fail.\n"); 2007 } 2008 2009 void 2010 dev_set_link_down(portid_t pid) 2011 { 2012 if (rte_eth_dev_set_link_down(pid) < 0) 2013 printf("\nSet link down fail.\n"); 2014 } 2015 2016 static int 2017 all_ports_started(void) 2018 { 2019 portid_t pi; 2020 struct rte_port *port; 2021 2022 RTE_ETH_FOREACH_DEV(pi) { 2023 port = &ports[pi]; 2024 /* Check if there is a port which is not started */ 2025 if ((port->port_status != RTE_PORT_STARTED) && 2026 (port->slave_flag == 0)) 2027 return 0; 2028 } 2029 2030 /* No port is not started */ 2031 return 1; 2032 } 2033 2034 int 2035 port_is_stopped(portid_t port_id) 2036 { 2037 struct rte_port *port = &ports[port_id]; 2038 2039 if ((port->port_status != RTE_PORT_STOPPED) && 2040 (port->slave_flag == 0)) 2041 return 0; 2042 return 1; 2043 } 2044 2045 int 2046 all_ports_stopped(void) 2047 { 2048 portid_t pi; 2049 2050 RTE_ETH_FOREACH_DEV(pi) { 2051 if (!port_is_stopped(pi)) 2052 return 0; 2053 } 2054 2055 return 1; 2056 } 2057 2058 int 2059 port_is_started(portid_t port_id) 2060 { 2061 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2062 return 0; 2063 2064 if (ports[port_id].port_status != RTE_PORT_STARTED) 2065 return 0; 2066 2067 return 1; 2068 } 2069 2070 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2071 static int 2072 setup_hairpin_queues(portid_t pi) 2073 { 2074 queueid_t qi; 2075 struct rte_eth_hairpin_conf hairpin_conf = { 2076 .peer_count = 1, 2077 }; 2078 int i; 2079 int diag; 2080 struct rte_port *port = &ports[pi]; 2081 2082 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2083 hairpin_conf.peers[0].port = pi; 2084 hairpin_conf.peers[0].queue = i + nb_rxq; 2085 diag = rte_eth_tx_hairpin_queue_setup 2086 (pi, qi, nb_txd, &hairpin_conf); 2087 i++; 2088 if (diag == 0) 2089 continue; 2090 2091 /* Fail to setup rx queue, return */ 2092 if (rte_atomic16_cmpset(&(port->port_status), 2093 RTE_PORT_HANDLING, 2094 RTE_PORT_STOPPED) == 0) 2095 printf("Port %d can not be set back " 2096 "to stopped\n", pi); 2097 printf("Fail to configure port %d hairpin " 2098 "queues\n", pi); 2099 /* try to reconfigure queues next time */ 2100 port->need_reconfig_queues = 1; 2101 return -1; 2102 } 2103 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2104 hairpin_conf.peers[0].port = pi; 2105 hairpin_conf.peers[0].queue = i + nb_txq; 2106 diag = rte_eth_rx_hairpin_queue_setup 2107 (pi, qi, nb_rxd, &hairpin_conf); 2108 i++; 2109 if (diag == 0) 2110 continue; 2111 2112 /* Fail to setup rx queue, return */ 2113 if (rte_atomic16_cmpset(&(port->port_status), 2114 RTE_PORT_HANDLING, 2115 RTE_PORT_STOPPED) == 0) 2116 printf("Port %d can not be set back " 2117 "to stopped\n", pi); 2118 printf("Fail to configure port %d hairpin " 2119 "queues\n", pi); 2120 /* try to reconfigure queues next time */ 2121 port->need_reconfig_queues = 1; 2122 return -1; 2123 } 2124 return 0; 2125 } 2126 2127 int 2128 start_port(portid_t pid) 2129 { 2130 int diag, need_check_link_status = -1; 2131 portid_t pi; 2132 queueid_t qi; 2133 struct rte_port *port; 2134 struct rte_ether_addr mac_addr; 2135 struct rte_eth_hairpin_cap cap; 2136 2137 if (port_id_is_invalid(pid, ENABLED_WARN)) 2138 return 0; 2139 2140 if(dcb_config) 2141 dcb_test = 1; 2142 RTE_ETH_FOREACH_DEV(pi) { 2143 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2144 continue; 2145 2146 need_check_link_status = 0; 2147 port = &ports[pi]; 2148 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2149 RTE_PORT_HANDLING) == 0) { 2150 printf("Port %d is now not stopped\n", pi); 2151 continue; 2152 } 2153 2154 if (port->need_reconfig > 0) { 2155 port->need_reconfig = 0; 2156 2157 if (flow_isolate_all) { 2158 int ret = port_flow_isolate(pi, 1); 2159 if (ret) { 2160 printf("Failed to apply isolated" 2161 " mode on port %d\n", pi); 2162 return -1; 2163 } 2164 } 2165 configure_rxtx_dump_callbacks(0); 2166 printf("Configuring Port %d (socket %u)\n", pi, 2167 port->socket_id); 2168 if (nb_hairpinq > 0 && 2169 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2170 printf("Port %d doesn't support hairpin " 2171 "queues\n", pi); 2172 return -1; 2173 } 2174 /* configure port */ 2175 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2176 nb_txq + nb_hairpinq, 2177 &(port->dev_conf)); 2178 if (diag != 0) { 2179 if (rte_atomic16_cmpset(&(port->port_status), 2180 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2181 printf("Port %d can not be set back " 2182 "to stopped\n", pi); 2183 printf("Fail to configure port %d\n", pi); 2184 /* try to reconfigure port next time */ 2185 port->need_reconfig = 1; 2186 return -1; 2187 } 2188 } 2189 if (port->need_reconfig_queues > 0) { 2190 port->need_reconfig_queues = 0; 2191 /* setup tx queues */ 2192 for (qi = 0; qi < nb_txq; qi++) { 2193 if ((numa_support) && 2194 (txring_numa[pi] != NUMA_NO_CONFIG)) 2195 diag = rte_eth_tx_queue_setup(pi, qi, 2196 port->nb_tx_desc[qi], 2197 txring_numa[pi], 2198 &(port->tx_conf[qi])); 2199 else 2200 diag = rte_eth_tx_queue_setup(pi, qi, 2201 port->nb_tx_desc[qi], 2202 port->socket_id, 2203 &(port->tx_conf[qi])); 2204 2205 if (diag == 0) 2206 continue; 2207 2208 /* Fail to setup tx queue, return */ 2209 if (rte_atomic16_cmpset(&(port->port_status), 2210 RTE_PORT_HANDLING, 2211 RTE_PORT_STOPPED) == 0) 2212 printf("Port %d can not be set back " 2213 "to stopped\n", pi); 2214 printf("Fail to configure port %d tx queues\n", 2215 pi); 2216 /* try to reconfigure queues next time */ 2217 port->need_reconfig_queues = 1; 2218 return -1; 2219 } 2220 for (qi = 0; qi < nb_rxq; qi++) { 2221 /* setup rx queues */ 2222 if ((numa_support) && 2223 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2224 struct rte_mempool * mp = 2225 mbuf_pool_find(rxring_numa[pi]); 2226 if (mp == NULL) { 2227 printf("Failed to setup RX queue:" 2228 "No mempool allocation" 2229 " on the socket %d\n", 2230 rxring_numa[pi]); 2231 return -1; 2232 } 2233 2234 diag = rte_eth_rx_queue_setup(pi, qi, 2235 port->nb_rx_desc[qi], 2236 rxring_numa[pi], 2237 &(port->rx_conf[qi]), 2238 mp); 2239 } else { 2240 struct rte_mempool *mp = 2241 mbuf_pool_find(port->socket_id); 2242 if (mp == NULL) { 2243 printf("Failed to setup RX queue:" 2244 "No mempool allocation" 2245 " on the socket %d\n", 2246 port->socket_id); 2247 return -1; 2248 } 2249 diag = rte_eth_rx_queue_setup(pi, qi, 2250 port->nb_rx_desc[qi], 2251 port->socket_id, 2252 &(port->rx_conf[qi]), 2253 mp); 2254 } 2255 if (diag == 0) 2256 continue; 2257 2258 /* Fail to setup rx queue, return */ 2259 if (rte_atomic16_cmpset(&(port->port_status), 2260 RTE_PORT_HANDLING, 2261 RTE_PORT_STOPPED) == 0) 2262 printf("Port %d can not be set back " 2263 "to stopped\n", pi); 2264 printf("Fail to configure port %d rx queues\n", 2265 pi); 2266 /* try to reconfigure queues next time */ 2267 port->need_reconfig_queues = 1; 2268 return -1; 2269 } 2270 /* setup hairpin queues */ 2271 if (setup_hairpin_queues(pi) != 0) 2272 return -1; 2273 } 2274 configure_rxtx_dump_callbacks(verbose_level); 2275 if (clear_ptypes) { 2276 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2277 NULL, 0); 2278 if (diag < 0) 2279 printf( 2280 "Port %d: Failed to disable Ptype parsing\n", 2281 pi); 2282 } 2283 2284 /* start port */ 2285 if (rte_eth_dev_start(pi) < 0) { 2286 printf("Fail to start port %d\n", pi); 2287 2288 /* Fail to setup rx queue, return */ 2289 if (rte_atomic16_cmpset(&(port->port_status), 2290 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2291 printf("Port %d can not be set back to " 2292 "stopped\n", pi); 2293 continue; 2294 } 2295 2296 if (rte_atomic16_cmpset(&(port->port_status), 2297 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2298 printf("Port %d can not be set into started\n", pi); 2299 2300 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2301 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2302 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2303 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2304 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2305 2306 /* at least one port started, need checking link status */ 2307 need_check_link_status = 1; 2308 } 2309 2310 if (need_check_link_status == 1 && !no_link_check) 2311 check_all_ports_link_status(RTE_PORT_ALL); 2312 else if (need_check_link_status == 0) 2313 printf("Please stop the ports first\n"); 2314 2315 printf("Done\n"); 2316 return 0; 2317 } 2318 2319 void 2320 stop_port(portid_t pid) 2321 { 2322 portid_t pi; 2323 struct rte_port *port; 2324 int need_check_link_status = 0; 2325 2326 if (dcb_test) { 2327 dcb_test = 0; 2328 dcb_config = 0; 2329 } 2330 2331 if (port_id_is_invalid(pid, ENABLED_WARN)) 2332 return; 2333 2334 printf("Stopping ports...\n"); 2335 2336 RTE_ETH_FOREACH_DEV(pi) { 2337 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2338 continue; 2339 2340 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2341 printf("Please remove port %d from forwarding configuration.\n", pi); 2342 continue; 2343 } 2344 2345 if (port_is_bonding_slave(pi)) { 2346 printf("Please remove port %d from bonded device.\n", pi); 2347 continue; 2348 } 2349 2350 port = &ports[pi]; 2351 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2352 RTE_PORT_HANDLING) == 0) 2353 continue; 2354 2355 rte_eth_dev_stop(pi); 2356 2357 if (rte_atomic16_cmpset(&(port->port_status), 2358 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2359 printf("Port %d can not be set into stopped\n", pi); 2360 need_check_link_status = 1; 2361 } 2362 if (need_check_link_status && !no_link_check) 2363 check_all_ports_link_status(RTE_PORT_ALL); 2364 2365 printf("Done\n"); 2366 } 2367 2368 static void 2369 remove_invalid_ports_in(portid_t *array, portid_t *total) 2370 { 2371 portid_t i; 2372 portid_t new_total = 0; 2373 2374 for (i = 0; i < *total; i++) 2375 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2376 array[new_total] = array[i]; 2377 new_total++; 2378 } 2379 *total = new_total; 2380 } 2381 2382 static void 2383 remove_invalid_ports(void) 2384 { 2385 remove_invalid_ports_in(ports_ids, &nb_ports); 2386 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2387 nb_cfg_ports = nb_fwd_ports; 2388 } 2389 2390 void 2391 close_port(portid_t pid) 2392 { 2393 portid_t pi; 2394 struct rte_port *port; 2395 2396 if (port_id_is_invalid(pid, ENABLED_WARN)) 2397 return; 2398 2399 printf("Closing ports...\n"); 2400 2401 RTE_ETH_FOREACH_DEV(pi) { 2402 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2403 continue; 2404 2405 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2406 printf("Please remove port %d from forwarding configuration.\n", pi); 2407 continue; 2408 } 2409 2410 if (port_is_bonding_slave(pi)) { 2411 printf("Please remove port %d from bonded device.\n", pi); 2412 continue; 2413 } 2414 2415 port = &ports[pi]; 2416 if (rte_atomic16_cmpset(&(port->port_status), 2417 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2418 printf("Port %d is already closed\n", pi); 2419 continue; 2420 } 2421 2422 if (rte_atomic16_cmpset(&(port->port_status), 2423 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2424 printf("Port %d is now not stopped\n", pi); 2425 continue; 2426 } 2427 2428 if (port->flow_list) 2429 port_flow_flush(pi); 2430 rte_eth_dev_close(pi); 2431 2432 remove_invalid_ports(); 2433 2434 if (rte_atomic16_cmpset(&(port->port_status), 2435 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2436 printf("Port %d cannot be set to closed\n", pi); 2437 } 2438 2439 printf("Done\n"); 2440 } 2441 2442 void 2443 reset_port(portid_t pid) 2444 { 2445 int diag; 2446 portid_t pi; 2447 struct rte_port *port; 2448 2449 if (port_id_is_invalid(pid, ENABLED_WARN)) 2450 return; 2451 2452 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2453 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2454 printf("Can not reset port(s), please stop port(s) first.\n"); 2455 return; 2456 } 2457 2458 printf("Resetting ports...\n"); 2459 2460 RTE_ETH_FOREACH_DEV(pi) { 2461 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2462 continue; 2463 2464 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2465 printf("Please remove port %d from forwarding " 2466 "configuration.\n", pi); 2467 continue; 2468 } 2469 2470 if (port_is_bonding_slave(pi)) { 2471 printf("Please remove port %d from bonded device.\n", 2472 pi); 2473 continue; 2474 } 2475 2476 diag = rte_eth_dev_reset(pi); 2477 if (diag == 0) { 2478 port = &ports[pi]; 2479 port->need_reconfig = 1; 2480 port->need_reconfig_queues = 1; 2481 } else { 2482 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2483 } 2484 } 2485 2486 printf("Done\n"); 2487 } 2488 2489 void 2490 attach_port(char *identifier) 2491 { 2492 portid_t pi; 2493 struct rte_dev_iterator iterator; 2494 2495 printf("Attaching a new port...\n"); 2496 2497 if (identifier == NULL) { 2498 printf("Invalid parameters are specified\n"); 2499 return; 2500 } 2501 2502 if (rte_dev_probe(identifier) < 0) { 2503 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2504 return; 2505 } 2506 2507 /* first attach mode: event */ 2508 if (setup_on_probe_event) { 2509 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2510 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2511 if (ports[pi].port_status == RTE_PORT_HANDLING && 2512 ports[pi].need_setup != 0) 2513 setup_attached_port(pi); 2514 return; 2515 } 2516 2517 /* second attach mode: iterator */ 2518 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2519 /* setup ports matching the devargs used for probing */ 2520 if (port_is_forwarding(pi)) 2521 continue; /* port was already attached before */ 2522 setup_attached_port(pi); 2523 } 2524 } 2525 2526 static void 2527 setup_attached_port(portid_t pi) 2528 { 2529 unsigned int socket_id; 2530 int ret; 2531 2532 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2533 /* if socket_id is invalid, set to the first available socket. */ 2534 if (check_socket_id(socket_id) < 0) 2535 socket_id = socket_ids[0]; 2536 reconfig(pi, socket_id); 2537 ret = rte_eth_promiscuous_enable(pi); 2538 if (ret != 0) 2539 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2540 pi, rte_strerror(-ret)); 2541 2542 ports_ids[nb_ports++] = pi; 2543 fwd_ports_ids[nb_fwd_ports++] = pi; 2544 nb_cfg_ports = nb_fwd_ports; 2545 ports[pi].need_setup = 0; 2546 ports[pi].port_status = RTE_PORT_STOPPED; 2547 2548 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2549 printf("Done\n"); 2550 } 2551 2552 static void 2553 detach_device(struct rte_device *dev) 2554 { 2555 portid_t sibling; 2556 2557 if (dev == NULL) { 2558 printf("Device already removed\n"); 2559 return; 2560 } 2561 2562 printf("Removing a device...\n"); 2563 2564 if (rte_dev_remove(dev) < 0) { 2565 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2566 return; 2567 } 2568 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2569 /* reset mapping between old ports and removed device */ 2570 rte_eth_devices[sibling].device = NULL; 2571 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2572 /* sibling ports are forced to be closed */ 2573 ports[sibling].port_status = RTE_PORT_CLOSED; 2574 printf("Port %u is closed\n", sibling); 2575 } 2576 } 2577 2578 remove_invalid_ports(); 2579 2580 printf("Device is detached\n"); 2581 printf("Now total ports is %d\n", nb_ports); 2582 printf("Done\n"); 2583 return; 2584 } 2585 2586 void 2587 detach_port_device(portid_t port_id) 2588 { 2589 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2590 return; 2591 2592 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2593 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2594 printf("Port not stopped\n"); 2595 return; 2596 } 2597 printf("Port was not closed\n"); 2598 if (ports[port_id].flow_list) 2599 port_flow_flush(port_id); 2600 } 2601 2602 detach_device(rte_eth_devices[port_id].device); 2603 } 2604 2605 void 2606 detach_devargs(char *identifier) 2607 { 2608 struct rte_dev_iterator iterator; 2609 struct rte_devargs da; 2610 portid_t port_id; 2611 2612 printf("Removing a device...\n"); 2613 2614 memset(&da, 0, sizeof(da)); 2615 if (rte_devargs_parsef(&da, "%s", identifier)) { 2616 printf("cannot parse identifier\n"); 2617 if (da.args) 2618 free(da.args); 2619 return; 2620 } 2621 2622 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2623 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2624 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2625 printf("Port %u not stopped\n", port_id); 2626 rte_eth_iterator_cleanup(&iterator); 2627 return; 2628 } 2629 2630 /* sibling ports are forced to be closed */ 2631 if (ports[port_id].flow_list) 2632 port_flow_flush(port_id); 2633 ports[port_id].port_status = RTE_PORT_CLOSED; 2634 printf("Port %u is now closed\n", port_id); 2635 } 2636 } 2637 2638 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2639 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2640 da.name, da.bus->name); 2641 return; 2642 } 2643 2644 remove_invalid_ports(); 2645 2646 printf("Device %s is detached\n", identifier); 2647 printf("Now total ports is %d\n", nb_ports); 2648 printf("Done\n"); 2649 } 2650 2651 void 2652 pmd_test_exit(void) 2653 { 2654 portid_t pt_id; 2655 int ret; 2656 int i; 2657 2658 if (test_done == 0) 2659 stop_packet_forwarding(); 2660 2661 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2662 if (mempools[i]) { 2663 if (mp_alloc_type == MP_ALLOC_ANON) 2664 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2665 NULL); 2666 } 2667 } 2668 if (ports != NULL) { 2669 no_link_check = 1; 2670 RTE_ETH_FOREACH_DEV(pt_id) { 2671 printf("\nStopping port %d...\n", pt_id); 2672 fflush(stdout); 2673 stop_port(pt_id); 2674 } 2675 RTE_ETH_FOREACH_DEV(pt_id) { 2676 printf("\nShutting down port %d...\n", pt_id); 2677 fflush(stdout); 2678 close_port(pt_id); 2679 } 2680 } 2681 2682 if (hot_plug) { 2683 ret = rte_dev_event_monitor_stop(); 2684 if (ret) { 2685 RTE_LOG(ERR, EAL, 2686 "fail to stop device event monitor."); 2687 return; 2688 } 2689 2690 ret = rte_dev_event_callback_unregister(NULL, 2691 dev_event_callback, NULL); 2692 if (ret < 0) { 2693 RTE_LOG(ERR, EAL, 2694 "fail to unregister device event callback.\n"); 2695 return; 2696 } 2697 2698 ret = rte_dev_hotplug_handle_disable(); 2699 if (ret) { 2700 RTE_LOG(ERR, EAL, 2701 "fail to disable hotplug handling.\n"); 2702 return; 2703 } 2704 } 2705 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2706 if (mempools[i]) 2707 rte_mempool_free(mempools[i]); 2708 } 2709 2710 printf("\nBye...\n"); 2711 } 2712 2713 typedef void (*cmd_func_t)(void); 2714 struct pmd_test_command { 2715 const char *cmd_name; 2716 cmd_func_t cmd_func; 2717 }; 2718 2719 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2720 2721 /* Check the link status of all ports in up to 9s, and print them finally */ 2722 static void 2723 check_all_ports_link_status(uint32_t port_mask) 2724 { 2725 #define CHECK_INTERVAL 100 /* 100ms */ 2726 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2727 portid_t portid; 2728 uint8_t count, all_ports_up, print_flag = 0; 2729 struct rte_eth_link link; 2730 int ret; 2731 2732 printf("Checking link statuses...\n"); 2733 fflush(stdout); 2734 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2735 all_ports_up = 1; 2736 RTE_ETH_FOREACH_DEV(portid) { 2737 if ((port_mask & (1 << portid)) == 0) 2738 continue; 2739 memset(&link, 0, sizeof(link)); 2740 ret = rte_eth_link_get_nowait(portid, &link); 2741 if (ret < 0) { 2742 all_ports_up = 0; 2743 if (print_flag == 1) 2744 printf("Port %u link get failed: %s\n", 2745 portid, rte_strerror(-ret)); 2746 continue; 2747 } 2748 /* print link status if flag set */ 2749 if (print_flag == 1) { 2750 if (link.link_status) 2751 printf( 2752 "Port%d Link Up. speed %u Mbps- %s\n", 2753 portid, link.link_speed, 2754 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2755 ("full-duplex") : ("half-duplex\n")); 2756 else 2757 printf("Port %d Link Down\n", portid); 2758 continue; 2759 } 2760 /* clear all_ports_up flag if any link down */ 2761 if (link.link_status == ETH_LINK_DOWN) { 2762 all_ports_up = 0; 2763 break; 2764 } 2765 } 2766 /* after finally printing all link status, get out */ 2767 if (print_flag == 1) 2768 break; 2769 2770 if (all_ports_up == 0) { 2771 fflush(stdout); 2772 rte_delay_ms(CHECK_INTERVAL); 2773 } 2774 2775 /* set the print_flag if all ports up or timeout */ 2776 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2777 print_flag = 1; 2778 } 2779 2780 if (lsc_interrupt) 2781 break; 2782 } 2783 } 2784 2785 /* 2786 * This callback is for remove a port for a device. It has limitation because 2787 * it is not for multiple port removal for a device. 2788 * TODO: the device detach invoke will plan to be removed from user side to 2789 * eal. And convert all PMDs to free port resources on ether device closing. 2790 */ 2791 static void 2792 rmv_port_callback(void *arg) 2793 { 2794 int need_to_start = 0; 2795 int org_no_link_check = no_link_check; 2796 portid_t port_id = (intptr_t)arg; 2797 struct rte_device *dev; 2798 2799 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2800 2801 if (!test_done && port_is_forwarding(port_id)) { 2802 need_to_start = 1; 2803 stop_packet_forwarding(); 2804 } 2805 no_link_check = 1; 2806 stop_port(port_id); 2807 no_link_check = org_no_link_check; 2808 2809 /* Save rte_device pointer before closing ethdev port */ 2810 dev = rte_eth_devices[port_id].device; 2811 close_port(port_id); 2812 detach_device(dev); /* might be already removed or have more ports */ 2813 2814 if (need_to_start) 2815 start_packet_forwarding(0); 2816 } 2817 2818 /* This function is used by the interrupt thread */ 2819 static int 2820 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2821 void *ret_param) 2822 { 2823 RTE_SET_USED(param); 2824 RTE_SET_USED(ret_param); 2825 2826 if (type >= RTE_ETH_EVENT_MAX) { 2827 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2828 port_id, __func__, type); 2829 fflush(stderr); 2830 } else if (event_print_mask & (UINT32_C(1) << type)) { 2831 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2832 eth_event_desc[type]); 2833 fflush(stdout); 2834 } 2835 2836 switch (type) { 2837 case RTE_ETH_EVENT_NEW: 2838 ports[port_id].need_setup = 1; 2839 ports[port_id].port_status = RTE_PORT_HANDLING; 2840 break; 2841 case RTE_ETH_EVENT_INTR_RMV: 2842 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2843 break; 2844 if (rte_eal_alarm_set(100000, 2845 rmv_port_callback, (void *)(intptr_t)port_id)) 2846 fprintf(stderr, "Could not set up deferred device removal\n"); 2847 break; 2848 default: 2849 break; 2850 } 2851 return 0; 2852 } 2853 2854 static int 2855 register_eth_event_callback(void) 2856 { 2857 int ret; 2858 enum rte_eth_event_type event; 2859 2860 for (event = RTE_ETH_EVENT_UNKNOWN; 2861 event < RTE_ETH_EVENT_MAX; event++) { 2862 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2863 event, 2864 eth_event_callback, 2865 NULL); 2866 if (ret != 0) { 2867 TESTPMD_LOG(ERR, "Failed to register callback for " 2868 "%s event\n", eth_event_desc[event]); 2869 return -1; 2870 } 2871 } 2872 2873 return 0; 2874 } 2875 2876 /* This function is used by the interrupt thread */ 2877 static void 2878 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2879 __rte_unused void *arg) 2880 { 2881 uint16_t port_id; 2882 int ret; 2883 2884 if (type >= RTE_DEV_EVENT_MAX) { 2885 fprintf(stderr, "%s called upon invalid event %d\n", 2886 __func__, type); 2887 fflush(stderr); 2888 } 2889 2890 switch (type) { 2891 case RTE_DEV_EVENT_REMOVE: 2892 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2893 device_name); 2894 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2895 if (ret) { 2896 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2897 device_name); 2898 return; 2899 } 2900 /* 2901 * Because the user's callback is invoked in eal interrupt 2902 * callback, the interrupt callback need to be finished before 2903 * it can be unregistered when detaching device. So finish 2904 * callback soon and use a deferred removal to detach device 2905 * is need. It is a workaround, once the device detaching be 2906 * moved into the eal in the future, the deferred removal could 2907 * be deleted. 2908 */ 2909 if (rte_eal_alarm_set(100000, 2910 rmv_port_callback, (void *)(intptr_t)port_id)) 2911 RTE_LOG(ERR, EAL, 2912 "Could not set up deferred device removal\n"); 2913 break; 2914 case RTE_DEV_EVENT_ADD: 2915 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2916 device_name); 2917 /* TODO: After finish kernel driver binding, 2918 * begin to attach port. 2919 */ 2920 break; 2921 default: 2922 break; 2923 } 2924 } 2925 2926 static int 2927 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2928 { 2929 uint16_t i; 2930 int diag; 2931 uint8_t mapping_found = 0; 2932 2933 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2934 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2935 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2936 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2937 tx_queue_stats_mappings[i].queue_id, 2938 tx_queue_stats_mappings[i].stats_counter_id); 2939 if (diag != 0) 2940 return diag; 2941 mapping_found = 1; 2942 } 2943 } 2944 if (mapping_found) 2945 port->tx_queue_stats_mapping_enabled = 1; 2946 return 0; 2947 } 2948 2949 static int 2950 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2951 { 2952 uint16_t i; 2953 int diag; 2954 uint8_t mapping_found = 0; 2955 2956 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2957 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2958 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2959 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2960 rx_queue_stats_mappings[i].queue_id, 2961 rx_queue_stats_mappings[i].stats_counter_id); 2962 if (diag != 0) 2963 return diag; 2964 mapping_found = 1; 2965 } 2966 } 2967 if (mapping_found) 2968 port->rx_queue_stats_mapping_enabled = 1; 2969 return 0; 2970 } 2971 2972 static void 2973 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2974 { 2975 int diag = 0; 2976 2977 diag = set_tx_queue_stats_mapping_registers(pi, port); 2978 if (diag != 0) { 2979 if (diag == -ENOTSUP) { 2980 port->tx_queue_stats_mapping_enabled = 0; 2981 printf("TX queue stats mapping not supported port id=%d\n", pi); 2982 } 2983 else 2984 rte_exit(EXIT_FAILURE, 2985 "set_tx_queue_stats_mapping_registers " 2986 "failed for port id=%d diag=%d\n", 2987 pi, diag); 2988 } 2989 2990 diag = set_rx_queue_stats_mapping_registers(pi, port); 2991 if (diag != 0) { 2992 if (diag == -ENOTSUP) { 2993 port->rx_queue_stats_mapping_enabled = 0; 2994 printf("RX queue stats mapping not supported port id=%d\n", pi); 2995 } 2996 else 2997 rte_exit(EXIT_FAILURE, 2998 "set_rx_queue_stats_mapping_registers " 2999 "failed for port id=%d diag=%d\n", 3000 pi, diag); 3001 } 3002 } 3003 3004 static void 3005 rxtx_port_config(struct rte_port *port) 3006 { 3007 uint16_t qid; 3008 uint64_t offloads; 3009 3010 for (qid = 0; qid < nb_rxq; qid++) { 3011 offloads = port->rx_conf[qid].offloads; 3012 port->rx_conf[qid] = port->dev_info.default_rxconf; 3013 if (offloads != 0) 3014 port->rx_conf[qid].offloads = offloads; 3015 3016 /* Check if any Rx parameters have been passed */ 3017 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3018 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3019 3020 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3021 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3022 3023 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3024 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3025 3026 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3027 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3028 3029 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3030 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3031 3032 port->nb_rx_desc[qid] = nb_rxd; 3033 } 3034 3035 for (qid = 0; qid < nb_txq; qid++) { 3036 offloads = port->tx_conf[qid].offloads; 3037 port->tx_conf[qid] = port->dev_info.default_txconf; 3038 if (offloads != 0) 3039 port->tx_conf[qid].offloads = offloads; 3040 3041 /* Check if any Tx parameters have been passed */ 3042 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3043 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3044 3045 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3046 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3047 3048 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3049 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3050 3051 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3052 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3053 3054 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3055 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3056 3057 port->nb_tx_desc[qid] = nb_txd; 3058 } 3059 } 3060 3061 void 3062 init_port_config(void) 3063 { 3064 portid_t pid; 3065 struct rte_port *port; 3066 int ret; 3067 3068 RTE_ETH_FOREACH_DEV(pid) { 3069 port = &ports[pid]; 3070 port->dev_conf.fdir_conf = fdir_conf; 3071 3072 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3073 if (ret != 0) 3074 return; 3075 3076 if (nb_rxq > 1) { 3077 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3078 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3079 rss_hf & port->dev_info.flow_type_rss_offloads; 3080 } else { 3081 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3082 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3083 } 3084 3085 if (port->dcb_flag == 0) { 3086 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3087 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 3088 else 3089 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3090 } 3091 3092 rxtx_port_config(port); 3093 3094 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3095 if (ret != 0) 3096 return; 3097 3098 map_port_queue_stats_mapping_registers(pid, port); 3099 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3100 rte_pmd_ixgbe_bypass_init(pid); 3101 #endif 3102 3103 if (lsc_interrupt && 3104 (rte_eth_devices[pid].data->dev_flags & 3105 RTE_ETH_DEV_INTR_LSC)) 3106 port->dev_conf.intr_conf.lsc = 1; 3107 if (rmv_interrupt && 3108 (rte_eth_devices[pid].data->dev_flags & 3109 RTE_ETH_DEV_INTR_RMV)) 3110 port->dev_conf.intr_conf.rmv = 1; 3111 } 3112 } 3113 3114 void set_port_slave_flag(portid_t slave_pid) 3115 { 3116 struct rte_port *port; 3117 3118 port = &ports[slave_pid]; 3119 port->slave_flag = 1; 3120 } 3121 3122 void clear_port_slave_flag(portid_t slave_pid) 3123 { 3124 struct rte_port *port; 3125 3126 port = &ports[slave_pid]; 3127 port->slave_flag = 0; 3128 } 3129 3130 uint8_t port_is_bonding_slave(portid_t slave_pid) 3131 { 3132 struct rte_port *port; 3133 3134 port = &ports[slave_pid]; 3135 if ((rte_eth_devices[slave_pid].data->dev_flags & 3136 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3137 return 1; 3138 return 0; 3139 } 3140 3141 const uint16_t vlan_tags[] = { 3142 0, 1, 2, 3, 4, 5, 6, 7, 3143 8, 9, 10, 11, 12, 13, 14, 15, 3144 16, 17, 18, 19, 20, 21, 22, 23, 3145 24, 25, 26, 27, 28, 29, 30, 31 3146 }; 3147 3148 static int 3149 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3150 enum dcb_mode_enable dcb_mode, 3151 enum rte_eth_nb_tcs num_tcs, 3152 uint8_t pfc_en) 3153 { 3154 uint8_t i; 3155 int32_t rc; 3156 struct rte_eth_rss_conf rss_conf; 3157 3158 /* 3159 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3160 * given above, and the number of traffic classes available for use. 3161 */ 3162 if (dcb_mode == DCB_VT_ENABLED) { 3163 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3164 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3165 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3166 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3167 3168 /* VMDQ+DCB RX and TX configurations */ 3169 vmdq_rx_conf->enable_default_pool = 0; 3170 vmdq_rx_conf->default_pool = 0; 3171 vmdq_rx_conf->nb_queue_pools = 3172 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3173 vmdq_tx_conf->nb_queue_pools = 3174 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3175 3176 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3177 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3178 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3179 vmdq_rx_conf->pool_map[i].pools = 3180 1 << (i % vmdq_rx_conf->nb_queue_pools); 3181 } 3182 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3183 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3184 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3185 } 3186 3187 /* set DCB mode of RX and TX of multiple queues */ 3188 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 3189 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3190 } else { 3191 struct rte_eth_dcb_rx_conf *rx_conf = 3192 ð_conf->rx_adv_conf.dcb_rx_conf; 3193 struct rte_eth_dcb_tx_conf *tx_conf = 3194 ð_conf->tx_adv_conf.dcb_tx_conf; 3195 3196 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3197 if (rc != 0) 3198 return rc; 3199 3200 rx_conf->nb_tcs = num_tcs; 3201 tx_conf->nb_tcs = num_tcs; 3202 3203 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3204 rx_conf->dcb_tc[i] = i % num_tcs; 3205 tx_conf->dcb_tc[i] = i % num_tcs; 3206 } 3207 3208 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3209 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3210 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3211 } 3212 3213 if (pfc_en) 3214 eth_conf->dcb_capability_en = 3215 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3216 else 3217 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3218 3219 return 0; 3220 } 3221 3222 int 3223 init_port_dcb_config(portid_t pid, 3224 enum dcb_mode_enable dcb_mode, 3225 enum rte_eth_nb_tcs num_tcs, 3226 uint8_t pfc_en) 3227 { 3228 struct rte_eth_conf port_conf; 3229 struct rte_port *rte_port; 3230 int retval; 3231 uint16_t i; 3232 3233 rte_port = &ports[pid]; 3234 3235 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3236 /* Enter DCB configuration status */ 3237 dcb_config = 1; 3238 3239 port_conf.rxmode = rte_port->dev_conf.rxmode; 3240 port_conf.txmode = rte_port->dev_conf.txmode; 3241 3242 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3243 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3244 if (retval < 0) 3245 return retval; 3246 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3247 3248 /* re-configure the device . */ 3249 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3250 if (retval < 0) 3251 return retval; 3252 3253 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3254 if (retval != 0) 3255 return retval; 3256 3257 /* If dev_info.vmdq_pool_base is greater than 0, 3258 * the queue id of vmdq pools is started after pf queues. 3259 */ 3260 if (dcb_mode == DCB_VT_ENABLED && 3261 rte_port->dev_info.vmdq_pool_base > 0) { 3262 printf("VMDQ_DCB multi-queue mode is nonsensical" 3263 " for port %d.", pid); 3264 return -1; 3265 } 3266 3267 /* Assume the ports in testpmd have the same dcb capability 3268 * and has the same number of rxq and txq in dcb mode 3269 */ 3270 if (dcb_mode == DCB_VT_ENABLED) { 3271 if (rte_port->dev_info.max_vfs > 0) { 3272 nb_rxq = rte_port->dev_info.nb_rx_queues; 3273 nb_txq = rte_port->dev_info.nb_tx_queues; 3274 } else { 3275 nb_rxq = rte_port->dev_info.max_rx_queues; 3276 nb_txq = rte_port->dev_info.max_tx_queues; 3277 } 3278 } else { 3279 /*if vt is disabled, use all pf queues */ 3280 if (rte_port->dev_info.vmdq_pool_base == 0) { 3281 nb_rxq = rte_port->dev_info.max_rx_queues; 3282 nb_txq = rte_port->dev_info.max_tx_queues; 3283 } else { 3284 nb_rxq = (queueid_t)num_tcs; 3285 nb_txq = (queueid_t)num_tcs; 3286 3287 } 3288 } 3289 rx_free_thresh = 64; 3290 3291 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3292 3293 rxtx_port_config(rte_port); 3294 /* VLAN filter */ 3295 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3296 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3297 rx_vft_set(pid, vlan_tags[i], 1); 3298 3299 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3300 if (retval != 0) 3301 return retval; 3302 3303 map_port_queue_stats_mapping_registers(pid, rte_port); 3304 3305 rte_port->dcb_flag = 1; 3306 3307 return 0; 3308 } 3309 3310 static void 3311 init_port(void) 3312 { 3313 /* Configuration of Ethernet ports. */ 3314 ports = rte_zmalloc("testpmd: ports", 3315 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3316 RTE_CACHE_LINE_SIZE); 3317 if (ports == NULL) { 3318 rte_exit(EXIT_FAILURE, 3319 "rte_zmalloc(%d struct rte_port) failed\n", 3320 RTE_MAX_ETHPORTS); 3321 } 3322 3323 /* Initialize ports NUMA structures */ 3324 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3325 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3326 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3327 } 3328 3329 static void 3330 force_quit(void) 3331 { 3332 pmd_test_exit(); 3333 prompt_exit(); 3334 } 3335 3336 static void 3337 print_stats(void) 3338 { 3339 uint8_t i; 3340 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3341 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3342 3343 /* Clear screen and move to top left */ 3344 printf("%s%s", clr, top_left); 3345 3346 printf("\nPort statistics ===================================="); 3347 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3348 nic_stats_display(fwd_ports_ids[i]); 3349 3350 fflush(stdout); 3351 } 3352 3353 static void 3354 signal_handler(int signum) 3355 { 3356 if (signum == SIGINT || signum == SIGTERM) { 3357 printf("\nSignal %d received, preparing to exit...\n", 3358 signum); 3359 #ifdef RTE_LIBRTE_PDUMP 3360 /* uninitialize packet capture framework */ 3361 rte_pdump_uninit(); 3362 #endif 3363 #ifdef RTE_LIBRTE_LATENCY_STATS 3364 if (latencystats_enabled != 0) 3365 rte_latencystats_uninit(); 3366 #endif 3367 force_quit(); 3368 /* Set flag to indicate the force termination. */ 3369 f_quit = 1; 3370 /* exit with the expected status */ 3371 signal(signum, SIG_DFL); 3372 kill(getpid(), signum); 3373 } 3374 } 3375 3376 int 3377 main(int argc, char** argv) 3378 { 3379 int diag; 3380 portid_t port_id; 3381 uint16_t count; 3382 int ret; 3383 3384 signal(SIGINT, signal_handler); 3385 signal(SIGTERM, signal_handler); 3386 3387 testpmd_logtype = rte_log_register("testpmd"); 3388 if (testpmd_logtype < 0) 3389 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3390 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3391 3392 diag = rte_eal_init(argc, argv); 3393 if (diag < 0) 3394 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3395 rte_strerror(rte_errno)); 3396 3397 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3398 rte_exit(EXIT_FAILURE, 3399 "Secondary process type not supported.\n"); 3400 3401 ret = register_eth_event_callback(); 3402 if (ret != 0) 3403 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3404 3405 #ifdef RTE_LIBRTE_PDUMP 3406 /* initialize packet capture framework */ 3407 rte_pdump_init(); 3408 #endif 3409 3410 count = 0; 3411 RTE_ETH_FOREACH_DEV(port_id) { 3412 ports_ids[count] = port_id; 3413 count++; 3414 } 3415 nb_ports = (portid_t) count; 3416 if (nb_ports == 0) 3417 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3418 3419 /* allocate port structures, and init them */ 3420 init_port(); 3421 3422 set_def_fwd_config(); 3423 if (nb_lcores == 0) 3424 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3425 "Check the core mask argument\n"); 3426 3427 /* Bitrate/latency stats disabled by default */ 3428 #ifdef RTE_LIBRTE_BITRATE 3429 bitrate_enabled = 0; 3430 #endif 3431 #ifdef RTE_LIBRTE_LATENCY_STATS 3432 latencystats_enabled = 0; 3433 #endif 3434 3435 /* on FreeBSD, mlockall() is disabled by default */ 3436 #ifdef RTE_EXEC_ENV_FREEBSD 3437 do_mlockall = 0; 3438 #else 3439 do_mlockall = 1; 3440 #endif 3441 3442 argc -= diag; 3443 argv += diag; 3444 if (argc > 1) 3445 launch_args_parse(argc, argv); 3446 3447 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3448 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3449 strerror(errno)); 3450 } 3451 3452 if (tx_first && interactive) 3453 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3454 "interactive mode.\n"); 3455 3456 if (tx_first && lsc_interrupt) { 3457 printf("Warning: lsc_interrupt needs to be off when " 3458 " using tx_first. Disabling.\n"); 3459 lsc_interrupt = 0; 3460 } 3461 3462 if (!nb_rxq && !nb_txq) 3463 printf("Warning: Either rx or tx queues should be non-zero\n"); 3464 3465 if (nb_rxq > 1 && nb_rxq > nb_txq) 3466 printf("Warning: nb_rxq=%d enables RSS configuration, " 3467 "but nb_txq=%d will prevent to fully test it.\n", 3468 nb_rxq, nb_txq); 3469 3470 init_config(); 3471 3472 if (hot_plug) { 3473 ret = rte_dev_hotplug_handle_enable(); 3474 if (ret) { 3475 RTE_LOG(ERR, EAL, 3476 "fail to enable hotplug handling."); 3477 return -1; 3478 } 3479 3480 ret = rte_dev_event_monitor_start(); 3481 if (ret) { 3482 RTE_LOG(ERR, EAL, 3483 "fail to start device event monitoring."); 3484 return -1; 3485 } 3486 3487 ret = rte_dev_event_callback_register(NULL, 3488 dev_event_callback, NULL); 3489 if (ret) { 3490 RTE_LOG(ERR, EAL, 3491 "fail to register device event callback\n"); 3492 return -1; 3493 } 3494 } 3495 3496 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3497 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3498 3499 /* set all ports to promiscuous mode by default */ 3500 RTE_ETH_FOREACH_DEV(port_id) { 3501 ret = rte_eth_promiscuous_enable(port_id); 3502 if (ret != 0) 3503 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3504 port_id, rte_strerror(-ret)); 3505 } 3506 3507 /* Init metrics library */ 3508 rte_metrics_init(rte_socket_id()); 3509 3510 #ifdef RTE_LIBRTE_LATENCY_STATS 3511 if (latencystats_enabled != 0) { 3512 int ret = rte_latencystats_init(1, NULL); 3513 if (ret) 3514 printf("Warning: latencystats init()" 3515 " returned error %d\n", ret); 3516 printf("Latencystats running on lcore %d\n", 3517 latencystats_lcore_id); 3518 } 3519 #endif 3520 3521 /* Setup bitrate stats */ 3522 #ifdef RTE_LIBRTE_BITRATE 3523 if (bitrate_enabled != 0) { 3524 bitrate_data = rte_stats_bitrate_create(); 3525 if (bitrate_data == NULL) 3526 rte_exit(EXIT_FAILURE, 3527 "Could not allocate bitrate data.\n"); 3528 rte_stats_bitrate_reg(bitrate_data); 3529 } 3530 #endif 3531 3532 #ifdef RTE_LIBRTE_CMDLINE 3533 if (strlen(cmdline_filename) != 0) 3534 cmdline_read_from_file(cmdline_filename); 3535 3536 if (interactive == 1) { 3537 if (auto_start) { 3538 printf("Start automatic packet forwarding\n"); 3539 start_packet_forwarding(0); 3540 } 3541 prompt(); 3542 pmd_test_exit(); 3543 } else 3544 #endif 3545 { 3546 char c; 3547 int rc; 3548 3549 f_quit = 0; 3550 3551 printf("No commandline core given, start packet forwarding\n"); 3552 start_packet_forwarding(tx_first); 3553 if (stats_period != 0) { 3554 uint64_t prev_time = 0, cur_time, diff_time = 0; 3555 uint64_t timer_period; 3556 3557 /* Convert to number of cycles */ 3558 timer_period = stats_period * rte_get_timer_hz(); 3559 3560 while (f_quit == 0) { 3561 cur_time = rte_get_timer_cycles(); 3562 diff_time += cur_time - prev_time; 3563 3564 if (diff_time >= timer_period) { 3565 print_stats(); 3566 /* Reset the timer */ 3567 diff_time = 0; 3568 } 3569 /* Sleep to avoid unnecessary checks */ 3570 prev_time = cur_time; 3571 sleep(1); 3572 } 3573 } 3574 3575 printf("Press enter to exit\n"); 3576 rc = read(0, &c, 1); 3577 pmd_test_exit(); 3578 if (rc < 0) 3579 return 1; 3580 } 3581 3582 ret = rte_eal_cleanup(); 3583 if (ret != 0) 3584 rte_exit(EXIT_FAILURE, 3585 "EAL cleanup failed: %s\n", strerror(-ret)); 3586 3587 return EXIT_SUCCESS; 3588 } 3589