1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
3 */
4
5 #include <stdarg.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <signal.h>
9 #include <string.h>
10 #include <time.h>
11 #include <fcntl.h>
12 #include <sys/mman.h>
13 #include <sys/types.h>
14 #include <errno.h>
15 #include <stdbool.h>
16
17 #include <sys/queue.h>
18 #include <sys/stat.h>
19
20 #include <stdint.h>
21 #include <unistd.h>
22 #include <inttypes.h>
23
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_byteorder.h>
27 #include <rte_log.h>
28 #include <rte_debug.h>
29 #include <rte_cycles.h>
30 #include <rte_memory.h>
31 #include <rte_memcpy.h>
32 #include <rte_launch.h>
33 #include <rte_eal.h>
34 #include <rte_alarm.h>
35 #include <rte_per_lcore.h>
36 #include <rte_lcore.h>
37 #include <rte_atomic.h>
38 #include <rte_branch_prediction.h>
39 #include <rte_mempool.h>
40 #include <rte_malloc.h>
41 #include <rte_mbuf.h>
42 #include <rte_mbuf_pool_ops.h>
43 #include <rte_interrupts.h>
44 #include <rte_pci.h>
45 #include <rte_ether.h>
46 #include <rte_ethdev.h>
47 #include <rte_dev.h>
48 #include <rte_string_fns.h>
49 #ifdef RTE_NET_IXGBE
50 #include <rte_pmd_ixgbe.h>
51 #endif
52 #ifdef RTE_LIB_PDUMP
53 #include <rte_pdump.h>
54 #endif
55 #include <rte_flow.h>
56 #include <rte_metrics.h>
57 #ifdef RTE_LIB_BITRATESTATS
58 #include <rte_bitrate.h>
59 #endif
60 #ifdef RTE_LIB_LATENCYSTATS
61 #include <rte_latencystats.h>
62 #endif
63
64 #include "testpmd.h"
65
66 #ifndef MAP_HUGETLB
67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */
68 #define HUGE_FLAG (0x40000)
69 #else
70 #define HUGE_FLAG MAP_HUGETLB
71 #endif
72
73 #ifndef MAP_HUGE_SHIFT
74 /* older kernels (or FreeBSD) will not have this define */
75 #define HUGE_SHIFT (26)
76 #else
77 #define HUGE_SHIFT MAP_HUGE_SHIFT
78 #endif
79
80 #define EXTMEM_HEAP_NAME "extmem"
81 #define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
82
83 uint16_t verbose_level = 0; /**< Silent by default. */
84 int testpmd_logtype; /**< Log type for testpmd logs */
85
86 /* use main core for command line ? */
87 uint8_t interactive = 0;
88 uint8_t auto_start = 0;
89 uint8_t tx_first;
90 char cmdline_filename[PATH_MAX] = {0};
91
92 /*
93 * NUMA support configuration.
94 * When set, the NUMA support attempts to dispatch the allocation of the
95 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the
96 * probed ports among the CPU sockets 0 and 1.
97 * Otherwise, all memory is allocated from CPU socket 0.
98 */
99 uint8_t numa_support = 1; /**< numa enabled by default */
100
101 /*
102 * In UMA mode,all memory is allocated from socket 0 if --socket-num is
103 * not configured.
104 */
105 uint8_t socket_num = UMA_NO_CONFIG;
106
107 /*
108 * Select mempool allocation type:
109 * - native: use regular DPDK memory
110 * - anon: use regular DPDK memory to create mempool, but populate using
111 * anonymous memory (may not be IOVA-contiguous)
112 * - xmem: use externally allocated hugepage memory
113 */
114 uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
115
116 /*
117 * Store specified sockets on which memory pool to be used by ports
118 * is allocated.
119 */
120 uint8_t port_numa[RTE_MAX_ETHPORTS];
121
122 /*
123 * Store specified sockets on which RX ring to be used by ports
124 * is allocated.
125 */
126 uint8_t rxring_numa[RTE_MAX_ETHPORTS];
127
128 /*
129 * Store specified sockets on which TX ring to be used by ports
130 * is allocated.
131 */
132 uint8_t txring_numa[RTE_MAX_ETHPORTS];
133
134 /*
135 * Record the Ethernet address of peer target ports to which packets are
136 * forwarded.
137 * Must be instantiated with the ethernet addresses of peer traffic generator
138 * ports.
139 */
140 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
141 portid_t nb_peer_eth_addrs = 0;
142
143 /*
144 * Probed Target Environment.
145 */
146 struct rte_port *ports; /**< For all probed ethernet ports. */
147 portid_t nb_ports; /**< Number of probed ethernet ports. */
148 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */
149 lcoreid_t nb_lcores; /**< Number of probed logical cores. */
150
151 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */
152
153 /*
154 * Test Forwarding Configuration.
155 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores
156 * nb_fwd_ports <= nb_cfg_ports <= nb_ports
157 */
158 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */
159 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */
160 portid_t nb_cfg_ports; /**< Number of configured ports. */
161 portid_t nb_fwd_ports; /**< Number of forwarding ports. */
162
163 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */
164 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */
165
166 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */
167 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */
168
169 /*
170 * Forwarding engines.
171 */
172 struct fwd_engine * fwd_engines[] = {
173 &io_fwd_engine,
174 &mac_fwd_engine,
175 &mac_swap_engine,
176 &flow_gen_engine,
177 &rx_only_engine,
178 &tx_only_engine,
179 &csum_fwd_engine,
180 &icmp_echo_engine,
181 &noisy_vnf_engine,
182 &five_tuple_swap_fwd_engine,
183 #ifdef RTE_LIBRTE_IEEE1588
184 &ieee1588_fwd_engine,
185 #endif
186 NULL,
187 };
188
189 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
190 uint16_t mempool_flags;
191
192 struct fwd_config cur_fwd_config;
193 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
194 uint32_t retry_enabled;
195 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
196 uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
197
198 uint32_t mbuf_data_size_n = 1; /* Number of specified mbuf sizes. */
199 uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
200 DEFAULT_MBUF_DATA_SIZE
201 }; /**< Mbuf data space size. */
202 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if
203 * specified on command-line. */
204 uint16_t stats_period; /**< Period to show statistics (disabled by default) */
205
206 /*
207 * In container, it cannot terminate the process which running with 'stats-period'
208 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM.
209 */
210 uint8_t f_quit;
211
212 /*
213 * Configuration of packet segments used to scatter received packets
214 * if some of split features is configured.
215 */
216 uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
217 uint8_t rx_pkt_nb_segs; /**< Number of segments to split */
218 uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
219 uint8_t rx_pkt_nb_offs; /**< Number of specified offsets */
220
221 /*
222 * Configuration of packet segments used by the "txonly" processing engine.
223 */
224 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */
225 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
226 TXONLY_DEF_PACKET_LEN,
227 };
228 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */
229
230 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
231 /**< Split policy for packets to TX. */
232
233 uint8_t txonly_multi_flow;
234 /**< Whether multiple flows are generated in TXONLY mode. */
235
236 uint32_t tx_pkt_times_inter;
237 /**< Timings for send scheduling in TXONLY mode, time between bursts. */
238
239 uint32_t tx_pkt_times_intra;
240 /**< Timings for send scheduling in TXONLY mode, time between packets. */
241
242 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */
243 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */
244
245 /* current configuration is in DCB or not,0 means it is not in DCB mode */
246 uint8_t dcb_config = 0;
247
248 /* Whether the dcb is in testing status */
249 uint8_t dcb_test = 0;
250
251 /*
252 * Configurable number of RX/TX queues.
253 */
254 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */
255 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */
256 queueid_t nb_txq = 1; /**< Number of TX queues per port. */
257
258 /*
259 * Configurable number of RX/TX ring descriptors.
260 * Defaults are supplied by drivers via ethdev.
261 */
262 #define RTE_TEST_RX_DESC_DEFAULT 0
263 #define RTE_TEST_TX_DESC_DEFAULT 0
264 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */
265 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */
266
267 #define RTE_PMD_PARAM_UNSET -1
268 /*
269 * Configurable values of RX and TX ring threshold registers.
270 */
271
272 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
273 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
274 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
275
276 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
277 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
278 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
279
280 /*
281 * Configurable value of RX free threshold.
282 */
283 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
284
285 /*
286 * Configurable value of RX drop enable.
287 */
288 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
289
290 /*
291 * Configurable value of TX free threshold.
292 */
293 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
294
295 /*
296 * Configurable value of TX RS bit threshold.
297 */
298 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
299
300 /*
301 * Configurable value of buffered packets before sending.
302 */
303 uint16_t noisy_tx_sw_bufsz;
304
305 /*
306 * Configurable value of packet buffer timeout.
307 */
308 uint16_t noisy_tx_sw_buf_flush_time;
309
310 /*
311 * Configurable value for size of VNF internal memory area
312 * used for simulating noisy neighbour behaviour
313 */
314 uint64_t noisy_lkup_mem_sz;
315
316 /*
317 * Configurable value of number of random writes done in
318 * VNF simulation memory area.
319 */
320 uint64_t noisy_lkup_num_writes;
321
322 /*
323 * Configurable value of number of random reads done in
324 * VNF simulation memory area.
325 */
326 uint64_t noisy_lkup_num_reads;
327
328 /*
329 * Configurable value of number of random reads/writes done in
330 * VNF simulation memory area.
331 */
332 uint64_t noisy_lkup_num_reads_writes;
333
334 /*
335 * Receive Side Scaling (RSS) configuration.
336 */
337 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */
338
339 /*
340 * Port topology configuration
341 */
342 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */
343
344 /*
345 * Avoids to flush all the RX streams before starts forwarding.
346 */
347 uint8_t no_flush_rx = 0; /* flush by default */
348
349 /*
350 * Flow API isolated mode.
351 */
352 uint8_t flow_isolate_all;
353
354 /*
355 * Avoids to check link status when starting/stopping a port.
356 */
357 uint8_t no_link_check = 0; /* check by default */
358
359 /*
360 * Don't automatically start all ports in interactive mode.
361 */
362 uint8_t no_device_start = 0;
363
364 /*
365 * Enable link status change notification
366 */
367 uint8_t lsc_interrupt = 1; /* enabled by default */
368
369 /*
370 * Enable device removal notification.
371 */
372 uint8_t rmv_interrupt = 1; /* enabled by default */
373
374 uint8_t hot_plug = 0; /**< hotplug disabled by default. */
375
376 /* After attach, port setup is called on event or by iterator */
377 bool setup_on_probe_event = true;
378
379 /* Clear ptypes on port initialization. */
380 uint8_t clear_ptypes = true;
381
382 /* Hairpin ports configuration mode. */
383 uint16_t hairpin_mode;
384
385 /* Pretty printing of ethdev events */
386 static const char * const eth_event_desc[] = {
387 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
388 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
389 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
390 [RTE_ETH_EVENT_INTR_RESET] = "reset",
391 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
392 [RTE_ETH_EVENT_IPSEC] = "IPsec",
393 [RTE_ETH_EVENT_MACSEC] = "MACsec",
394 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
395 [RTE_ETH_EVENT_NEW] = "device probed",
396 [RTE_ETH_EVENT_DESTROY] = "device released",
397 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
398 [RTE_ETH_EVENT_MAX] = NULL,
399 };
400
401 /*
402 * Display or mask ether events
403 * Default to all events except VF_MBOX
404 */
405 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
406 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
407 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
408 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
409 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
410 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
411 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
412 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
413 /*
414 * Decide if all memory are locked for performance.
415 */
416 int do_mlockall = 0;
417
418 /*
419 * NIC bypass mode configuration options.
420 */
421
422 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
423 /* The NIC bypass watchdog timeout. */
424 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
425 #endif
426
427
428 #ifdef RTE_LIB_LATENCYSTATS
429
430 /*
431 * Set when latency stats is enabled in the commandline
432 */
433 uint8_t latencystats_enabled;
434
435 /*
436 * Lcore ID to serive latency statistics.
437 */
438 lcoreid_t latencystats_lcore_id = -1;
439
440 #endif
441
442 /*
443 * Ethernet device configuration.
444 */
445 struct rte_eth_rxmode rx_mode = {
446 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
447 /**< Default maximum frame length. */
448 };
449
450 struct rte_eth_txmode tx_mode = {
451 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
452 };
453
454 struct rte_fdir_conf fdir_conf = {
455 .mode = RTE_FDIR_MODE_NONE,
456 .pballoc = RTE_FDIR_PBALLOC_64K,
457 .status = RTE_FDIR_REPORT_STATUS,
458 .mask = {
459 .vlan_tci_mask = 0xFFEF,
460 .ipv4_mask = {
461 .src_ip = 0xFFFFFFFF,
462 .dst_ip = 0xFFFFFFFF,
463 },
464 .ipv6_mask = {
465 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
466 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
467 },
468 .src_port_mask = 0xFFFF,
469 .dst_port_mask = 0xFFFF,
470 .mac_addr_byte_mask = 0xFF,
471 .tunnel_type_mask = 1,
472 .tunnel_id_mask = 0xFFFFFFFF,
473 },
474 .drop_queue = 127,
475 };
476
477 volatile int test_done = 1; /* stop packet forwarding when set to 1. */
478
479 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS];
480 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS];
481
482 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array;
483 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array;
484
485 uint16_t nb_tx_queue_stats_mappings = 0;
486 uint16_t nb_rx_queue_stats_mappings = 0;
487
488 /*
489 * Display zero values by default for xstats
490 */
491 uint8_t xstats_hide_zero;
492
493 /*
494 * Measure of CPU cycles disabled by default
495 */
496 uint8_t record_core_cycles;
497
498 /*
499 * Display of RX and TX bursts disabled by default
500 */
501 uint8_t record_burst_stats;
502
503 unsigned int num_sockets = 0;
504 unsigned int socket_ids[RTE_MAX_NUMA_NODES];
505
506 #ifdef RTE_LIB_BITRATESTATS
507 /* Bitrate statistics */
508 struct rte_stats_bitrates *bitrate_data;
509 lcoreid_t bitrate_lcore_id;
510 uint8_t bitrate_enabled;
511 #endif
512
513 struct gro_status gro_ports[RTE_MAX_ETHPORTS];
514 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
515
516 /*
517 * hexadecimal bitmask of RX mq mode can be enabled.
518 */
519 enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
520
521 /* Forward function declarations */
522 static void setup_attached_port(portid_t pi);
523 static void map_port_queue_stats_mapping_registers(portid_t pi,
524 struct rte_port *port);
525 static void check_all_ports_link_status(uint32_t port_mask);
526 static int eth_event_callback(portid_t port_id,
527 enum rte_eth_event_type type,
528 void *param, void *ret_param);
529 static void dev_event_callback(const char *device_name,
530 enum rte_dev_event_type type,
531 void *param);
532
533 /*
534 * Check if all the ports are started.
535 * If yes, return positive value. If not, return zero.
536 */
537 static int all_ports_started(void);
538
539 struct gso_status gso_ports[RTE_MAX_ETHPORTS];
540 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
541
542 /* Holds the registered mbuf dynamic flags names. */
543 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
544
545 /*
546 * Helper function to check if socket is already discovered.
547 * If yes, return positive value. If not, return zero.
548 */
549 int
new_socket_id(unsigned int socket_id)550 new_socket_id(unsigned int socket_id)
551 {
552 unsigned int i;
553
554 for (i = 0; i < num_sockets; i++) {
555 if (socket_ids[i] == socket_id)
556 return 0;
557 }
558 return 1;
559 }
560
561 /*
562 * Setup default configuration.
563 */
564 static void
set_default_fwd_lcores_config(void)565 set_default_fwd_lcores_config(void)
566 {
567 unsigned int i;
568 unsigned int nb_lc;
569 unsigned int sock_num;
570
571 nb_lc = 0;
572 for (i = 0; i < RTE_MAX_LCORE; i++) {
573 if (!rte_lcore_is_enabled(i))
574 continue;
575 sock_num = rte_lcore_to_socket_id(i);
576 if (new_socket_id(sock_num)) {
577 if (num_sockets >= RTE_MAX_NUMA_NODES) {
578 rte_exit(EXIT_FAILURE,
579 "Total sockets greater than %u\n",
580 RTE_MAX_NUMA_NODES);
581 }
582 socket_ids[num_sockets++] = sock_num;
583 }
584 if (i == rte_get_main_lcore())
585 continue;
586 fwd_lcores_cpuids[nb_lc++] = i;
587 }
588 nb_lcores = (lcoreid_t) nb_lc;
589 nb_cfg_lcores = nb_lcores;
590 nb_fwd_lcores = 1;
591 }
592
593 static void
set_def_peer_eth_addrs(void)594 set_def_peer_eth_addrs(void)
595 {
596 portid_t i;
597
598 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
599 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
600 peer_eth_addrs[i].addr_bytes[5] = i;
601 }
602 }
603
604 static void
set_default_fwd_ports_config(void)605 set_default_fwd_ports_config(void)
606 {
607 portid_t pt_id;
608 int i = 0;
609
610 RTE_ETH_FOREACH_DEV(pt_id) {
611 fwd_ports_ids[i++] = pt_id;
612
613 /* Update sockets info according to the attached device */
614 int socket_id = rte_eth_dev_socket_id(pt_id);
615 if (socket_id >= 0 && new_socket_id(socket_id)) {
616 if (num_sockets >= RTE_MAX_NUMA_NODES) {
617 rte_exit(EXIT_FAILURE,
618 "Total sockets greater than %u\n",
619 RTE_MAX_NUMA_NODES);
620 }
621 socket_ids[num_sockets++] = socket_id;
622 }
623 }
624
625 nb_cfg_ports = nb_ports;
626 nb_fwd_ports = nb_ports;
627 }
628
629 void
set_def_fwd_config(void)630 set_def_fwd_config(void)
631 {
632 set_default_fwd_lcores_config();
633 set_def_peer_eth_addrs();
634 set_default_fwd_ports_config();
635 }
636
637 /* extremely pessimistic estimation of memory required to create a mempool */
638 static int
calc_mem_size(uint32_t nb_mbufs,uint32_t mbuf_sz,size_t pgsz,size_t * out)639 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
640 {
641 unsigned int n_pages, mbuf_per_pg, leftover;
642 uint64_t total_mem, mbuf_mem, obj_sz;
643
644 /* there is no good way to predict how much space the mempool will
645 * occupy because it will allocate chunks on the fly, and some of those
646 * will come from default DPDK memory while some will come from our
647 * external memory, so just assume 128MB will be enough for everyone.
648 */
649 uint64_t hdr_mem = 128 << 20;
650
651 /* account for possible non-contiguousness */
652 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
653 if (obj_sz > pgsz) {
654 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
655 return -1;
656 }
657
658 mbuf_per_pg = pgsz / obj_sz;
659 leftover = (nb_mbufs % mbuf_per_pg) > 0;
660 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
661
662 mbuf_mem = n_pages * pgsz;
663
664 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
665
666 if (total_mem > SIZE_MAX) {
667 TESTPMD_LOG(ERR, "Memory size too big\n");
668 return -1;
669 }
670 *out = (size_t)total_mem;
671
672 return 0;
673 }
674
675 static int
pagesz_flags(uint64_t page_sz)676 pagesz_flags(uint64_t page_sz)
677 {
678 /* as per mmap() manpage, all page sizes are log2 of page size
679 * shifted by MAP_HUGE_SHIFT
680 */
681 int log2 = rte_log2_u64(page_sz);
682
683 return (log2 << HUGE_SHIFT);
684 }
685
686 static void *
alloc_mem(size_t memsz,size_t pgsz,bool huge)687 alloc_mem(size_t memsz, size_t pgsz, bool huge)
688 {
689 void *addr;
690 int flags;
691
692 /* allocate anonymous hugepages */
693 flags = MAP_ANONYMOUS | MAP_PRIVATE;
694 if (huge)
695 flags |= HUGE_FLAG | pagesz_flags(pgsz);
696
697 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
698 if (addr == MAP_FAILED)
699 return NULL;
700
701 return addr;
702 }
703
704 struct extmem_param {
705 void *addr;
706 size_t len;
707 size_t pgsz;
708 rte_iova_t *iova_table;
709 unsigned int iova_table_len;
710 };
711
712 static int
create_extmem(uint32_t nb_mbufs,uint32_t mbuf_sz,struct extmem_param * param,bool huge)713 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
714 bool huge)
715 {
716 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */
717 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */
718 unsigned int cur_page, n_pages, pgsz_idx;
719 size_t mem_sz, cur_pgsz;
720 rte_iova_t *iovas = NULL;
721 void *addr;
722 int ret;
723
724 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
725 /* skip anything that is too big */
726 if (pgsizes[pgsz_idx] > SIZE_MAX)
727 continue;
728
729 cur_pgsz = pgsizes[pgsz_idx];
730
731 /* if we were told not to allocate hugepages, override */
732 if (!huge)
733 cur_pgsz = sysconf(_SC_PAGESIZE);
734
735 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
736 if (ret < 0) {
737 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
738 return -1;
739 }
740
741 /* allocate our memory */
742 addr = alloc_mem(mem_sz, cur_pgsz, huge);
743
744 /* if we couldn't allocate memory with a specified page size,
745 * that doesn't mean we can't do it with other page sizes, so
746 * try another one.
747 */
748 if (addr == NULL)
749 continue;
750
751 /* store IOVA addresses for every page in this memory area */
752 n_pages = mem_sz / cur_pgsz;
753
754 iovas = malloc(sizeof(*iovas) * n_pages);
755
756 if (iovas == NULL) {
757 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
758 goto fail;
759 }
760 /* lock memory if it's not huge pages */
761 if (!huge)
762 mlock(addr, mem_sz);
763
764 /* populate IOVA addresses */
765 for (cur_page = 0; cur_page < n_pages; cur_page++) {
766 rte_iova_t iova;
767 size_t offset;
768 void *cur;
769
770 offset = cur_pgsz * cur_page;
771 cur = RTE_PTR_ADD(addr, offset);
772
773 /* touch the page before getting its IOVA */
774 *(volatile char *)cur = 0;
775
776 iova = rte_mem_virt2iova(cur);
777
778 iovas[cur_page] = iova;
779 }
780
781 break;
782 }
783 /* if we couldn't allocate anything */
784 if (iovas == NULL)
785 return -1;
786
787 param->addr = addr;
788 param->len = mem_sz;
789 param->pgsz = cur_pgsz;
790 param->iova_table = iovas;
791 param->iova_table_len = n_pages;
792
793 return 0;
794 fail:
795 if (iovas)
796 free(iovas);
797 if (addr)
798 munmap(addr, mem_sz);
799
800 return -1;
801 }
802
803 static int
setup_extmem(uint32_t nb_mbufs,uint32_t mbuf_sz,bool huge)804 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
805 {
806 struct extmem_param param;
807 int socket_id, ret;
808
809 memset(¶m, 0, sizeof(param));
810
811 /* check if our heap exists */
812 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
813 if (socket_id < 0) {
814 /* create our heap */
815 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
816 if (ret < 0) {
817 TESTPMD_LOG(ERR, "Cannot create heap\n");
818 return -1;
819 }
820 }
821
822 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
823 if (ret < 0) {
824 TESTPMD_LOG(ERR, "Cannot create memory area\n");
825 return -1;
826 }
827
828 /* we now have a valid memory area, so add it to heap */
829 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
830 param.addr, param.len, param.iova_table,
831 param.iova_table_len, param.pgsz);
832
833 /* when using VFIO, memory is automatically mapped for DMA by EAL */
834
835 /* not needed any more */
836 free(param.iova_table);
837
838 if (ret < 0) {
839 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
840 munmap(param.addr, param.len);
841 return -1;
842 }
843
844 /* success */
845
846 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
847 param.len >> 20);
848
849 return 0;
850 }
851 static void
dma_unmap_cb(struct rte_mempool * mp __rte_unused,void * opaque __rte_unused,struct rte_mempool_memhdr * memhdr,unsigned mem_idx __rte_unused)852 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
853 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
854 {
855 uint16_t pid = 0;
856 int ret;
857
858 RTE_ETH_FOREACH_DEV(pid) {
859 struct rte_eth_dev *dev =
860 &rte_eth_devices[pid];
861
862 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
863 memhdr->len);
864 if (ret) {
865 TESTPMD_LOG(DEBUG,
866 "unable to DMA unmap addr 0x%p "
867 "for device %s\n",
868 memhdr->addr, dev->data->name);
869 }
870 }
871 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
872 if (ret) {
873 TESTPMD_LOG(DEBUG,
874 "unable to un-register addr 0x%p\n", memhdr->addr);
875 }
876 }
877
878 static void
dma_map_cb(struct rte_mempool * mp __rte_unused,void * opaque __rte_unused,struct rte_mempool_memhdr * memhdr,unsigned mem_idx __rte_unused)879 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
880 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
881 {
882 uint16_t pid = 0;
883 size_t page_size = sysconf(_SC_PAGESIZE);
884 int ret;
885
886 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
887 page_size);
888 if (ret) {
889 TESTPMD_LOG(DEBUG,
890 "unable to register addr 0x%p\n", memhdr->addr);
891 return;
892 }
893 RTE_ETH_FOREACH_DEV(pid) {
894 struct rte_eth_dev *dev =
895 &rte_eth_devices[pid];
896
897 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
898 memhdr->len);
899 if (ret) {
900 TESTPMD_LOG(DEBUG,
901 "unable to DMA map addr 0x%p "
902 "for device %s\n",
903 memhdr->addr, dev->data->name);
904 }
905 }
906 }
907
908 static unsigned int
setup_extbuf(uint32_t nb_mbufs,uint16_t mbuf_sz,unsigned int socket_id,char * pool_name,struct rte_pktmbuf_extmem ** ext_mem)909 setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
910 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
911 {
912 struct rte_pktmbuf_extmem *xmem;
913 unsigned int ext_num, zone_num, elt_num;
914 uint16_t elt_size;
915
916 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
917 elt_num = EXTBUF_ZONE_SIZE / elt_size;
918 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
919
920 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
921 if (xmem == NULL) {
922 TESTPMD_LOG(ERR, "Cannot allocate memory for "
923 "external buffer descriptors\n");
924 *ext_mem = NULL;
925 return 0;
926 }
927 for (ext_num = 0; ext_num < zone_num; ext_num++) {
928 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
929 const struct rte_memzone *mz;
930 char mz_name[RTE_MEMZONE_NAMESIZE];
931 int ret;
932
933 ret = snprintf(mz_name, sizeof(mz_name),
934 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
935 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
936 errno = ENAMETOOLONG;
937 ext_num = 0;
938 break;
939 }
940 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
941 socket_id,
942 RTE_MEMZONE_IOVA_CONTIG |
943 RTE_MEMZONE_1GB |
944 RTE_MEMZONE_SIZE_HINT_ONLY,
945 EXTBUF_ZONE_SIZE);
946 if (mz == NULL) {
947 /*
948 * The caller exits on external buffer creation
949 * error, so there is no need to free memzones.
950 */
951 errno = ENOMEM;
952 ext_num = 0;
953 break;
954 }
955 xseg->buf_ptr = mz->addr;
956 xseg->buf_iova = mz->iova;
957 xseg->buf_len = EXTBUF_ZONE_SIZE;
958 xseg->elt_size = elt_size;
959 }
960 if (ext_num == 0 && xmem != NULL) {
961 free(xmem);
962 xmem = NULL;
963 }
964 *ext_mem = xmem;
965 return ext_num;
966 }
967
968 /*
969 * Configuration initialisation done once at init time.
970 */
971 static struct rte_mempool *
mbuf_pool_create(uint16_t mbuf_seg_size,unsigned nb_mbuf,unsigned int socket_id,uint16_t size_idx)972 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
973 unsigned int socket_id, uint16_t size_idx)
974 {
975 char pool_name[RTE_MEMPOOL_NAMESIZE];
976 struct rte_mempool *rte_mp = NULL;
977 uint32_t mb_size;
978
979 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
980 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
981
982 TESTPMD_LOG(INFO,
983 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
984 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
985
986 switch (mp_alloc_type) {
987 case MP_ALLOC_NATIVE:
988 {
989 /* wrapper to rte_mempool_create() */
990 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
991 rte_mbuf_best_mempool_ops());
992 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
993 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
994 break;
995 }
996 case MP_ALLOC_ANON:
997 {
998 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
999 mb_size, (unsigned int) mb_mempool_cache,
1000 sizeof(struct rte_pktmbuf_pool_private),
1001 socket_id, mempool_flags);
1002 if (rte_mp == NULL)
1003 goto err;
1004
1005 if (rte_mempool_populate_anon(rte_mp) == 0) {
1006 rte_mempool_free(rte_mp);
1007 rte_mp = NULL;
1008 goto err;
1009 }
1010 rte_pktmbuf_pool_init(rte_mp, NULL);
1011 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1012 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1013 break;
1014 }
1015 case MP_ALLOC_XMEM:
1016 case MP_ALLOC_XMEM_HUGE:
1017 {
1018 int heap_socket;
1019 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1020
1021 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1022 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1023
1024 heap_socket =
1025 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1026 if (heap_socket < 0)
1027 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1028
1029 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1030 rte_mbuf_best_mempool_ops());
1031 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1032 mb_mempool_cache, 0, mbuf_seg_size,
1033 heap_socket);
1034 break;
1035 }
1036 case MP_ALLOC_XBUF:
1037 {
1038 struct rte_pktmbuf_extmem *ext_mem;
1039 unsigned int ext_num;
1040
1041 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1042 socket_id, pool_name, &ext_mem);
1043 if (ext_num == 0)
1044 rte_exit(EXIT_FAILURE,
1045 "Can't create pinned data buffers\n");
1046
1047 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1048 rte_mbuf_best_mempool_ops());
1049 rte_mp = rte_pktmbuf_pool_create_extbuf
1050 (pool_name, nb_mbuf, mb_mempool_cache,
1051 0, mbuf_seg_size, socket_id,
1052 ext_mem, ext_num);
1053 free(ext_mem);
1054 break;
1055 }
1056 default:
1057 {
1058 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1059 }
1060 }
1061
1062 err:
1063 if (rte_mp == NULL) {
1064 rte_exit(EXIT_FAILURE,
1065 "Creation of mbuf pool for socket %u failed: %s\n",
1066 socket_id, rte_strerror(rte_errno));
1067 } else if (verbose_level > 0) {
1068 rte_mempool_dump(stdout, rte_mp);
1069 }
1070 return rte_mp;
1071 }
1072
1073 /*
1074 * Check given socket id is valid or not with NUMA mode,
1075 * if valid, return 0, else return -1
1076 */
1077 static int
check_socket_id(const unsigned int socket_id)1078 check_socket_id(const unsigned int socket_id)
1079 {
1080 static int warning_once = 0;
1081
1082 if (new_socket_id(socket_id)) {
1083 if (!warning_once && numa_support)
1084 printf("Warning: NUMA should be configured manually by"
1085 " using --port-numa-config and"
1086 " --ring-numa-config parameters along with"
1087 " --numa.\n");
1088 warning_once = 1;
1089 return -1;
1090 }
1091 return 0;
1092 }
1093
1094 /*
1095 * Get the allowed maximum number of RX queues.
1096 * *pid return the port id which has minimal value of
1097 * max_rx_queues in all ports.
1098 */
1099 queueid_t
get_allowed_max_nb_rxq(portid_t * pid)1100 get_allowed_max_nb_rxq(portid_t *pid)
1101 {
1102 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1103 bool max_rxq_valid = false;
1104 portid_t pi;
1105 struct rte_eth_dev_info dev_info;
1106
1107 RTE_ETH_FOREACH_DEV(pi) {
1108 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1109 continue;
1110
1111 max_rxq_valid = true;
1112 if (dev_info.max_rx_queues < allowed_max_rxq) {
1113 allowed_max_rxq = dev_info.max_rx_queues;
1114 *pid = pi;
1115 }
1116 }
1117 return max_rxq_valid ? allowed_max_rxq : 0;
1118 }
1119
1120 /*
1121 * Check input rxq is valid or not.
1122 * If input rxq is not greater than any of maximum number
1123 * of RX queues of all ports, it is valid.
1124 * if valid, return 0, else return -1
1125 */
1126 int
check_nb_rxq(queueid_t rxq)1127 check_nb_rxq(queueid_t rxq)
1128 {
1129 queueid_t allowed_max_rxq;
1130 portid_t pid = 0;
1131
1132 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1133 if (rxq > allowed_max_rxq) {
1134 printf("Fail: input rxq (%u) can't be greater "
1135 "than max_rx_queues (%u) of port %u\n",
1136 rxq,
1137 allowed_max_rxq,
1138 pid);
1139 return -1;
1140 }
1141 return 0;
1142 }
1143
1144 /*
1145 * Get the allowed maximum number of TX queues.
1146 * *pid return the port id which has minimal value of
1147 * max_tx_queues in all ports.
1148 */
1149 queueid_t
get_allowed_max_nb_txq(portid_t * pid)1150 get_allowed_max_nb_txq(portid_t *pid)
1151 {
1152 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1153 bool max_txq_valid = false;
1154 portid_t pi;
1155 struct rte_eth_dev_info dev_info;
1156
1157 RTE_ETH_FOREACH_DEV(pi) {
1158 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1159 continue;
1160
1161 max_txq_valid = true;
1162 if (dev_info.max_tx_queues < allowed_max_txq) {
1163 allowed_max_txq = dev_info.max_tx_queues;
1164 *pid = pi;
1165 }
1166 }
1167 return max_txq_valid ? allowed_max_txq : 0;
1168 }
1169
1170 /*
1171 * Check input txq is valid or not.
1172 * If input txq is not greater than any of maximum number
1173 * of TX queues of all ports, it is valid.
1174 * if valid, return 0, else return -1
1175 */
1176 int
check_nb_txq(queueid_t txq)1177 check_nb_txq(queueid_t txq)
1178 {
1179 queueid_t allowed_max_txq;
1180 portid_t pid = 0;
1181
1182 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1183 if (txq > allowed_max_txq) {
1184 printf("Fail: input txq (%u) can't be greater "
1185 "than max_tx_queues (%u) of port %u\n",
1186 txq,
1187 allowed_max_txq,
1188 pid);
1189 return -1;
1190 }
1191 return 0;
1192 }
1193
1194 /*
1195 * Get the allowed maximum number of RXDs of every rx queue.
1196 * *pid return the port id which has minimal value of
1197 * max_rxd in all queues of all ports.
1198 */
1199 static uint16_t
get_allowed_max_nb_rxd(portid_t * pid)1200 get_allowed_max_nb_rxd(portid_t *pid)
1201 {
1202 uint16_t allowed_max_rxd = UINT16_MAX;
1203 portid_t pi;
1204 struct rte_eth_dev_info dev_info;
1205
1206 RTE_ETH_FOREACH_DEV(pi) {
1207 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1208 continue;
1209
1210 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1211 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1212 *pid = pi;
1213 }
1214 }
1215 return allowed_max_rxd;
1216 }
1217
1218 /*
1219 * Get the allowed minimal number of RXDs of every rx queue.
1220 * *pid return the port id which has minimal value of
1221 * min_rxd in all queues of all ports.
1222 */
1223 static uint16_t
get_allowed_min_nb_rxd(portid_t * pid)1224 get_allowed_min_nb_rxd(portid_t *pid)
1225 {
1226 uint16_t allowed_min_rxd = 0;
1227 portid_t pi;
1228 struct rte_eth_dev_info dev_info;
1229
1230 RTE_ETH_FOREACH_DEV(pi) {
1231 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1232 continue;
1233
1234 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1235 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1236 *pid = pi;
1237 }
1238 }
1239
1240 return allowed_min_rxd;
1241 }
1242
1243 /*
1244 * Check input rxd is valid or not.
1245 * If input rxd is not greater than any of maximum number
1246 * of RXDs of every Rx queues and is not less than any of
1247 * minimal number of RXDs of every Rx queues, it is valid.
1248 * if valid, return 0, else return -1
1249 */
1250 int
check_nb_rxd(queueid_t rxd)1251 check_nb_rxd(queueid_t rxd)
1252 {
1253 uint16_t allowed_max_rxd;
1254 uint16_t allowed_min_rxd;
1255 portid_t pid = 0;
1256
1257 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1258 if (rxd > allowed_max_rxd) {
1259 printf("Fail: input rxd (%u) can't be greater "
1260 "than max_rxds (%u) of port %u\n",
1261 rxd,
1262 allowed_max_rxd,
1263 pid);
1264 return -1;
1265 }
1266
1267 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1268 if (rxd < allowed_min_rxd) {
1269 printf("Fail: input rxd (%u) can't be less "
1270 "than min_rxds (%u) of port %u\n",
1271 rxd,
1272 allowed_min_rxd,
1273 pid);
1274 return -1;
1275 }
1276
1277 return 0;
1278 }
1279
1280 /*
1281 * Get the allowed maximum number of TXDs of every rx queues.
1282 * *pid return the port id which has minimal value of
1283 * max_txd in every tx queue.
1284 */
1285 static uint16_t
get_allowed_max_nb_txd(portid_t * pid)1286 get_allowed_max_nb_txd(portid_t *pid)
1287 {
1288 uint16_t allowed_max_txd = UINT16_MAX;
1289 portid_t pi;
1290 struct rte_eth_dev_info dev_info;
1291
1292 RTE_ETH_FOREACH_DEV(pi) {
1293 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1294 continue;
1295
1296 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1297 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1298 *pid = pi;
1299 }
1300 }
1301 return allowed_max_txd;
1302 }
1303
1304 /*
1305 * Get the allowed maximum number of TXDs of every tx queues.
1306 * *pid return the port id which has minimal value of
1307 * min_txd in every tx queue.
1308 */
1309 static uint16_t
get_allowed_min_nb_txd(portid_t * pid)1310 get_allowed_min_nb_txd(portid_t *pid)
1311 {
1312 uint16_t allowed_min_txd = 0;
1313 portid_t pi;
1314 struct rte_eth_dev_info dev_info;
1315
1316 RTE_ETH_FOREACH_DEV(pi) {
1317 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1318 continue;
1319
1320 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1321 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1322 *pid = pi;
1323 }
1324 }
1325
1326 return allowed_min_txd;
1327 }
1328
1329 /*
1330 * Check input txd is valid or not.
1331 * If input txd is not greater than any of maximum number
1332 * of TXDs of every Rx queues, it is valid.
1333 * if valid, return 0, else return -1
1334 */
1335 int
check_nb_txd(queueid_t txd)1336 check_nb_txd(queueid_t txd)
1337 {
1338 uint16_t allowed_max_txd;
1339 uint16_t allowed_min_txd;
1340 portid_t pid = 0;
1341
1342 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1343 if (txd > allowed_max_txd) {
1344 printf("Fail: input txd (%u) can't be greater "
1345 "than max_txds (%u) of port %u\n",
1346 txd,
1347 allowed_max_txd,
1348 pid);
1349 return -1;
1350 }
1351
1352 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1353 if (txd < allowed_min_txd) {
1354 printf("Fail: input txd (%u) can't be less "
1355 "than min_txds (%u) of port %u\n",
1356 txd,
1357 allowed_min_txd,
1358 pid);
1359 return -1;
1360 }
1361 return 0;
1362 }
1363
1364
1365 /*
1366 * Get the allowed maximum number of hairpin queues.
1367 * *pid return the port id which has minimal value of
1368 * max_hairpin_queues in all ports.
1369 */
1370 queueid_t
get_allowed_max_nb_hairpinq(portid_t * pid)1371 get_allowed_max_nb_hairpinq(portid_t *pid)
1372 {
1373 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1374 portid_t pi;
1375 struct rte_eth_hairpin_cap cap;
1376
1377 RTE_ETH_FOREACH_DEV(pi) {
1378 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1379 *pid = pi;
1380 return 0;
1381 }
1382 if (cap.max_nb_queues < allowed_max_hairpinq) {
1383 allowed_max_hairpinq = cap.max_nb_queues;
1384 *pid = pi;
1385 }
1386 }
1387 return allowed_max_hairpinq;
1388 }
1389
1390 /*
1391 * Check input hairpin is valid or not.
1392 * If input hairpin is not greater than any of maximum number
1393 * of hairpin queues of all ports, it is valid.
1394 * if valid, return 0, else return -1
1395 */
1396 int
check_nb_hairpinq(queueid_t hairpinq)1397 check_nb_hairpinq(queueid_t hairpinq)
1398 {
1399 queueid_t allowed_max_hairpinq;
1400 portid_t pid = 0;
1401
1402 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1403 if (hairpinq > allowed_max_hairpinq) {
1404 printf("Fail: input hairpin (%u) can't be greater "
1405 "than max_hairpin_queues (%u) of port %u\n",
1406 hairpinq, allowed_max_hairpinq, pid);
1407 return -1;
1408 }
1409 return 0;
1410 }
1411
1412 static void
init_config(void)1413 init_config(void)
1414 {
1415 portid_t pid;
1416 struct rte_port *port;
1417 struct rte_mempool *mbp;
1418 unsigned int nb_mbuf_per_pool;
1419 lcoreid_t lc_id;
1420 uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1421 struct rte_gro_param gro_param;
1422 uint32_t gso_types;
1423 uint16_t data_size;
1424 bool warning = 0;
1425 int k;
1426 int ret;
1427
1428 memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1429
1430 /* Configuration of logical cores. */
1431 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1432 sizeof(struct fwd_lcore *) * nb_lcores,
1433 RTE_CACHE_LINE_SIZE);
1434 if (fwd_lcores == NULL) {
1435 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1436 "failed\n", nb_lcores);
1437 }
1438 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1439 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1440 sizeof(struct fwd_lcore),
1441 RTE_CACHE_LINE_SIZE);
1442 if (fwd_lcores[lc_id] == NULL) {
1443 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1444 "failed\n");
1445 }
1446 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1447 }
1448
1449 RTE_ETH_FOREACH_DEV(pid) {
1450 port = &ports[pid];
1451 /* Apply default TxRx configuration for all ports */
1452 port->dev_conf.txmode = tx_mode;
1453 port->dev_conf.rxmode = rx_mode;
1454
1455 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1456 if (ret != 0)
1457 rte_exit(EXIT_FAILURE,
1458 "rte_eth_dev_info_get() failed\n");
1459
1460 if (!(port->dev_info.tx_offload_capa &
1461 DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1462 port->dev_conf.txmode.offloads &=
1463 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1464 if (numa_support) {
1465 if (port_numa[pid] != NUMA_NO_CONFIG)
1466 port_per_socket[port_numa[pid]]++;
1467 else {
1468 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1469
1470 /*
1471 * if socket_id is invalid,
1472 * set to the first available socket.
1473 */
1474 if (check_socket_id(socket_id) < 0)
1475 socket_id = socket_ids[0];
1476 port_per_socket[socket_id]++;
1477 }
1478 }
1479
1480 /* Apply Rx offloads configuration */
1481 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1482 port->rx_conf[k].offloads =
1483 port->dev_conf.rxmode.offloads;
1484 /* Apply Tx offloads configuration */
1485 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1486 port->tx_conf[k].offloads =
1487 port->dev_conf.txmode.offloads;
1488
1489 /* set flag to initialize port/queue */
1490 port->need_reconfig = 1;
1491 port->need_reconfig_queues = 1;
1492 port->tx_metadata = 0;
1493
1494 /* Check for maximum number of segments per MTU. Accordingly
1495 * update the mbuf data size.
1496 */
1497 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1498 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1499 data_size = rx_mode.max_rx_pkt_len /
1500 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1501
1502 if ((data_size + RTE_PKTMBUF_HEADROOM) >
1503 mbuf_data_size[0]) {
1504 mbuf_data_size[0] = data_size +
1505 RTE_PKTMBUF_HEADROOM;
1506 warning = 1;
1507 }
1508 }
1509 }
1510
1511 if (warning)
1512 TESTPMD_LOG(WARNING,
1513 "Configured mbuf size of the first segment %hu\n",
1514 mbuf_data_size[0]);
1515 /*
1516 * Create pools of mbuf.
1517 * If NUMA support is disabled, create a single pool of mbuf in
1518 * socket 0 memory by default.
1519 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1.
1520 *
1521 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and
1522 * nb_txd can be configured at run time.
1523 */
1524 if (param_total_num_mbufs)
1525 nb_mbuf_per_pool = param_total_num_mbufs;
1526 else {
1527 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1528 (nb_lcores * mb_mempool_cache) +
1529 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1530 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1531 }
1532
1533 if (numa_support) {
1534 uint8_t i, j;
1535
1536 for (i = 0; i < num_sockets; i++)
1537 for (j = 0; j < mbuf_data_size_n; j++)
1538 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1539 mbuf_pool_create(mbuf_data_size[j],
1540 nb_mbuf_per_pool,
1541 socket_ids[i], j);
1542 } else {
1543 uint8_t i;
1544
1545 for (i = 0; i < mbuf_data_size_n; i++)
1546 mempools[i] = mbuf_pool_create
1547 (mbuf_data_size[i],
1548 nb_mbuf_per_pool,
1549 socket_num == UMA_NO_CONFIG ?
1550 0 : socket_num, i);
1551 }
1552
1553 init_port_config();
1554
1555 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1556 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1557 /*
1558 * Records which Mbuf pool to use by each logical core, if needed.
1559 */
1560 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1561 mbp = mbuf_pool_find(
1562 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1563
1564 if (mbp == NULL)
1565 mbp = mbuf_pool_find(0, 0);
1566 fwd_lcores[lc_id]->mbp = mbp;
1567 /* initialize GSO context */
1568 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1569 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1570 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1571 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1572 RTE_ETHER_CRC_LEN;
1573 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1574 }
1575
1576 /* Configuration of packet forwarding streams. */
1577 if (init_fwd_streams() < 0)
1578 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1579
1580 fwd_config_setup();
1581
1582 /* create a gro context for each lcore */
1583 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1584 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1585 gro_param.max_item_per_flow = MAX_PKT_BURST;
1586 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1587 gro_param.socket_id = rte_lcore_to_socket_id(
1588 fwd_lcores_cpuids[lc_id]);
1589 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1590 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1591 rte_exit(EXIT_FAILURE,
1592 "rte_gro_ctx_create() failed\n");
1593 }
1594 }
1595 }
1596
1597
1598 void
reconfig(portid_t new_port_id,unsigned socket_id)1599 reconfig(portid_t new_port_id, unsigned socket_id)
1600 {
1601 struct rte_port *port;
1602 int ret;
1603
1604 /* Reconfiguration of Ethernet ports. */
1605 port = &ports[new_port_id];
1606
1607 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1608 if (ret != 0)
1609 return;
1610
1611 /* set flag to initialize port/queue */
1612 port->need_reconfig = 1;
1613 port->need_reconfig_queues = 1;
1614 port->socket_id = socket_id;
1615
1616 init_port_config();
1617 }
1618
1619
1620 int
init_fwd_streams(void)1621 init_fwd_streams(void)
1622 {
1623 portid_t pid;
1624 struct rte_port *port;
1625 streamid_t sm_id, nb_fwd_streams_new;
1626 queueid_t q;
1627
1628 /* set socket id according to numa or not */
1629 RTE_ETH_FOREACH_DEV(pid) {
1630 port = &ports[pid];
1631 if (nb_rxq > port->dev_info.max_rx_queues) {
1632 printf("Fail: nb_rxq(%d) is greater than "
1633 "max_rx_queues(%d)\n", nb_rxq,
1634 port->dev_info.max_rx_queues);
1635 return -1;
1636 }
1637 if (nb_txq > port->dev_info.max_tx_queues) {
1638 printf("Fail: nb_txq(%d) is greater than "
1639 "max_tx_queues(%d)\n", nb_txq,
1640 port->dev_info.max_tx_queues);
1641 return -1;
1642 }
1643 if (numa_support) {
1644 if (port_numa[pid] != NUMA_NO_CONFIG)
1645 port->socket_id = port_numa[pid];
1646 else {
1647 port->socket_id = rte_eth_dev_socket_id(pid);
1648
1649 /*
1650 * if socket_id is invalid,
1651 * set to the first available socket.
1652 */
1653 if (check_socket_id(port->socket_id) < 0)
1654 port->socket_id = socket_ids[0];
1655 }
1656 }
1657 else {
1658 if (socket_num == UMA_NO_CONFIG)
1659 port->socket_id = 0;
1660 else
1661 port->socket_id = socket_num;
1662 }
1663 }
1664
1665 q = RTE_MAX(nb_rxq, nb_txq);
1666 if (q == 0) {
1667 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1668 return -1;
1669 }
1670 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1671 if (nb_fwd_streams_new == nb_fwd_streams)
1672 return 0;
1673 /* clear the old */
1674 if (fwd_streams != NULL) {
1675 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1676 if (fwd_streams[sm_id] == NULL)
1677 continue;
1678 rte_free(fwd_streams[sm_id]);
1679 fwd_streams[sm_id] = NULL;
1680 }
1681 rte_free(fwd_streams);
1682 fwd_streams = NULL;
1683 }
1684
1685 /* init new */
1686 nb_fwd_streams = nb_fwd_streams_new;
1687 if (nb_fwd_streams) {
1688 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1689 sizeof(struct fwd_stream *) * nb_fwd_streams,
1690 RTE_CACHE_LINE_SIZE);
1691 if (fwd_streams == NULL)
1692 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1693 " (struct fwd_stream *)) failed\n",
1694 nb_fwd_streams);
1695
1696 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1697 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1698 " struct fwd_stream", sizeof(struct fwd_stream),
1699 RTE_CACHE_LINE_SIZE);
1700 if (fwd_streams[sm_id] == NULL)
1701 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1702 "(struct fwd_stream) failed\n");
1703 }
1704 }
1705
1706 return 0;
1707 }
1708
1709 static void
pkt_burst_stats_display(const char * rx_tx,struct pkt_burst_stats * pbs)1710 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1711 {
1712 uint64_t total_burst, sburst;
1713 uint64_t nb_burst;
1714 uint64_t burst_stats[4];
1715 uint16_t pktnb_stats[4];
1716 uint16_t nb_pkt;
1717 int burst_percent[4], sburstp;
1718 int i;
1719
1720 /*
1721 * First compute the total number of packet bursts and the
1722 * two highest numbers of bursts of the same number of packets.
1723 */
1724 memset(&burst_stats, 0x0, sizeof(burst_stats));
1725 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1726
1727 /* Show stats for 0 burst size always */
1728 total_burst = pbs->pkt_burst_spread[0];
1729 burst_stats[0] = pbs->pkt_burst_spread[0];
1730 pktnb_stats[0] = 0;
1731
1732 /* Find the next 2 burst sizes with highest occurrences. */
1733 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1734 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1735
1736 if (nb_burst == 0)
1737 continue;
1738
1739 total_burst += nb_burst;
1740
1741 if (nb_burst > burst_stats[1]) {
1742 burst_stats[2] = burst_stats[1];
1743 pktnb_stats[2] = pktnb_stats[1];
1744 burst_stats[1] = nb_burst;
1745 pktnb_stats[1] = nb_pkt;
1746 } else if (nb_burst > burst_stats[2]) {
1747 burst_stats[2] = nb_burst;
1748 pktnb_stats[2] = nb_pkt;
1749 }
1750 }
1751 if (total_burst == 0)
1752 return;
1753
1754 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1755 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1756 if (i == 3) {
1757 printf("%d%% of other]\n", 100 - sburstp);
1758 return;
1759 }
1760
1761 sburst += burst_stats[i];
1762 if (sburst == total_burst) {
1763 printf("%d%% of %d pkts]\n",
1764 100 - sburstp, (int) pktnb_stats[i]);
1765 return;
1766 }
1767
1768 burst_percent[i] =
1769 (double)burst_stats[i] / total_burst * 100;
1770 printf("%d%% of %d pkts + ",
1771 burst_percent[i], (int) pktnb_stats[i]);
1772 sburstp += burst_percent[i];
1773 }
1774 }
1775
1776 static void
fwd_stream_stats_display(streamid_t stream_id)1777 fwd_stream_stats_display(streamid_t stream_id)
1778 {
1779 struct fwd_stream *fs;
1780 static const char *fwd_top_stats_border = "-------";
1781
1782 fs = fwd_streams[stream_id];
1783 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1784 (fs->fwd_dropped == 0))
1785 return;
1786 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1787 "TX Port=%2d/Queue=%2d %s\n",
1788 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1789 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1790 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1791 " TX-dropped: %-14"PRIu64,
1792 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1793
1794 /* if checksum mode */
1795 if (cur_fwd_eng == &csum_fwd_engine) {
1796 printf(" RX- bad IP checksum: %-14"PRIu64
1797 " Rx- bad L4 checksum: %-14"PRIu64
1798 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1799 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1800 fs->rx_bad_outer_l4_csum);
1801 } else {
1802 printf("\n");
1803 }
1804
1805 if (record_burst_stats) {
1806 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1807 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1808 }
1809 }
1810
1811 void
fwd_stats_display(void)1812 fwd_stats_display(void)
1813 {
1814 static const char *fwd_stats_border = "----------------------";
1815 static const char *acc_stats_border = "+++++++++++++++";
1816 struct {
1817 struct fwd_stream *rx_stream;
1818 struct fwd_stream *tx_stream;
1819 uint64_t tx_dropped;
1820 uint64_t rx_bad_ip_csum;
1821 uint64_t rx_bad_l4_csum;
1822 uint64_t rx_bad_outer_l4_csum;
1823 } ports_stats[RTE_MAX_ETHPORTS];
1824 uint64_t total_rx_dropped = 0;
1825 uint64_t total_tx_dropped = 0;
1826 uint64_t total_rx_nombuf = 0;
1827 struct rte_eth_stats stats;
1828 uint64_t fwd_cycles = 0;
1829 uint64_t total_recv = 0;
1830 uint64_t total_xmit = 0;
1831 struct rte_port *port;
1832 streamid_t sm_id;
1833 portid_t pt_id;
1834 int i;
1835
1836 memset(ports_stats, 0, sizeof(ports_stats));
1837
1838 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1839 struct fwd_stream *fs = fwd_streams[sm_id];
1840
1841 if (cur_fwd_config.nb_fwd_streams >
1842 cur_fwd_config.nb_fwd_ports) {
1843 fwd_stream_stats_display(sm_id);
1844 } else {
1845 ports_stats[fs->tx_port].tx_stream = fs;
1846 ports_stats[fs->rx_port].rx_stream = fs;
1847 }
1848
1849 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1850
1851 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1852 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1853 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1854 fs->rx_bad_outer_l4_csum;
1855
1856 if (record_core_cycles)
1857 fwd_cycles += fs->core_cycles;
1858 }
1859 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1860 uint8_t j;
1861
1862 pt_id = fwd_ports_ids[i];
1863 port = &ports[pt_id];
1864
1865 rte_eth_stats_get(pt_id, &stats);
1866 stats.ipackets -= port->stats.ipackets;
1867 stats.opackets -= port->stats.opackets;
1868 stats.ibytes -= port->stats.ibytes;
1869 stats.obytes -= port->stats.obytes;
1870 stats.imissed -= port->stats.imissed;
1871 stats.oerrors -= port->stats.oerrors;
1872 stats.rx_nombuf -= port->stats.rx_nombuf;
1873
1874 total_recv += stats.ipackets;
1875 total_xmit += stats.opackets;
1876 total_rx_dropped += stats.imissed;
1877 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1878 total_tx_dropped += stats.oerrors;
1879 total_rx_nombuf += stats.rx_nombuf;
1880
1881 printf("\n %s Forward statistics for port %-2d %s\n",
1882 fwd_stats_border, pt_id, fwd_stats_border);
1883
1884 if (!port->rx_queue_stats_mapping_enabled &&
1885 !port->tx_queue_stats_mapping_enabled) {
1886 printf(" RX-packets: %-14"PRIu64
1887 " RX-dropped: %-14"PRIu64
1888 "RX-total: %-"PRIu64"\n",
1889 stats.ipackets, stats.imissed,
1890 stats.ipackets + stats.imissed);
1891
1892 if (cur_fwd_eng == &csum_fwd_engine)
1893 printf(" Bad-ipcsum: %-14"PRIu64
1894 " Bad-l4csum: %-14"PRIu64
1895 "Bad-outer-l4csum: %-14"PRIu64"\n",
1896 ports_stats[pt_id].rx_bad_ip_csum,
1897 ports_stats[pt_id].rx_bad_l4_csum,
1898 ports_stats[pt_id].rx_bad_outer_l4_csum);
1899 if (stats.ierrors + stats.rx_nombuf > 0) {
1900 printf(" RX-error: %-"PRIu64"\n",
1901 stats.ierrors);
1902 printf(" RX-nombufs: %-14"PRIu64"\n",
1903 stats.rx_nombuf);
1904 }
1905
1906 printf(" TX-packets: %-14"PRIu64
1907 " TX-dropped: %-14"PRIu64
1908 "TX-total: %-"PRIu64"\n",
1909 stats.opackets, ports_stats[pt_id].tx_dropped,
1910 stats.opackets + ports_stats[pt_id].tx_dropped);
1911 } else {
1912 printf(" RX-packets: %14"PRIu64
1913 " RX-dropped:%14"PRIu64
1914 " RX-total:%14"PRIu64"\n",
1915 stats.ipackets, stats.imissed,
1916 stats.ipackets + stats.imissed);
1917
1918 if (cur_fwd_eng == &csum_fwd_engine)
1919 printf(" Bad-ipcsum:%14"PRIu64
1920 " Bad-l4csum:%14"PRIu64
1921 " Bad-outer-l4csum: %-14"PRIu64"\n",
1922 ports_stats[pt_id].rx_bad_ip_csum,
1923 ports_stats[pt_id].rx_bad_l4_csum,
1924 ports_stats[pt_id].rx_bad_outer_l4_csum);
1925 if ((stats.ierrors + stats.rx_nombuf) > 0) {
1926 printf(" RX-error:%"PRIu64"\n", stats.ierrors);
1927 printf(" RX-nombufs: %14"PRIu64"\n",
1928 stats.rx_nombuf);
1929 }
1930
1931 printf(" TX-packets: %14"PRIu64
1932 " TX-dropped:%14"PRIu64
1933 " TX-total:%14"PRIu64"\n",
1934 stats.opackets, ports_stats[pt_id].tx_dropped,
1935 stats.opackets + ports_stats[pt_id].tx_dropped);
1936 }
1937
1938 if (record_burst_stats) {
1939 if (ports_stats[pt_id].rx_stream)
1940 pkt_burst_stats_display("RX",
1941 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1942 if (ports_stats[pt_id].tx_stream)
1943 pkt_burst_stats_display("TX",
1944 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1945 }
1946
1947 if (port->rx_queue_stats_mapping_enabled) {
1948 printf("\n");
1949 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1950 printf(" Stats reg %2d RX-packets:%14"PRIu64
1951 " RX-errors:%14"PRIu64
1952 " RX-bytes:%14"PRIu64"\n",
1953 j, stats.q_ipackets[j],
1954 stats.q_errors[j], stats.q_ibytes[j]);
1955 }
1956 printf("\n");
1957 }
1958 if (port->tx_queue_stats_mapping_enabled) {
1959 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
1960 printf(" Stats reg %2d TX-packets:%14"PRIu64
1961 " TX-bytes:%14"
1962 PRIu64"\n",
1963 j, stats.q_opackets[j],
1964 stats.q_obytes[j]);
1965 }
1966 }
1967
1968 printf(" %s--------------------------------%s\n",
1969 fwd_stats_border, fwd_stats_border);
1970 }
1971
1972 printf("\n %s Accumulated forward statistics for all ports"
1973 "%s\n",
1974 acc_stats_border, acc_stats_border);
1975 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1976 "%-"PRIu64"\n"
1977 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1978 "%-"PRIu64"\n",
1979 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1980 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1981 if (total_rx_nombuf > 0)
1982 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1983 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1984 "%s\n",
1985 acc_stats_border, acc_stats_border);
1986 if (record_core_cycles) {
1987 #define CYC_PER_MHZ 1E6
1988 if (total_recv > 0 || total_xmit > 0) {
1989 uint64_t total_pkts = 0;
1990 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1991 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1992 total_pkts = total_xmit;
1993 else
1994 total_pkts = total_recv;
1995
1996 printf("\n CPU cycles/packet=%.2F (total cycles="
1997 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1998 " MHz Clock\n",
1999 (double) fwd_cycles / total_pkts,
2000 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2001 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2002 }
2003 }
2004 }
2005
2006 void
fwd_stats_reset(void)2007 fwd_stats_reset(void)
2008 {
2009 streamid_t sm_id;
2010 portid_t pt_id;
2011 int i;
2012
2013 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2014 pt_id = fwd_ports_ids[i];
2015 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2016 }
2017 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2018 struct fwd_stream *fs = fwd_streams[sm_id];
2019
2020 fs->rx_packets = 0;
2021 fs->tx_packets = 0;
2022 fs->fwd_dropped = 0;
2023 fs->rx_bad_ip_csum = 0;
2024 fs->rx_bad_l4_csum = 0;
2025 fs->rx_bad_outer_l4_csum = 0;
2026
2027 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2028 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2029 fs->core_cycles = 0;
2030 }
2031 }
2032
2033 static void
flush_fwd_rx_queues(void)2034 flush_fwd_rx_queues(void)
2035 {
2036 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2037 portid_t rxp;
2038 portid_t port_id;
2039 queueid_t rxq;
2040 uint16_t nb_rx;
2041 uint16_t i;
2042 uint8_t j;
2043 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2044 uint64_t timer_period;
2045
2046 /* convert to number of cycles */
2047 timer_period = rte_get_timer_hz(); /* 1 second timeout */
2048
2049 for (j = 0; j < 2; j++) {
2050 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2051 for (rxq = 0; rxq < nb_rxq; rxq++) {
2052 port_id = fwd_ports_ids[rxp];
2053 /**
2054 * testpmd can stuck in the below do while loop
2055 * if rte_eth_rx_burst() always returns nonzero
2056 * packets. So timer is added to exit this loop
2057 * after 1sec timer expiry.
2058 */
2059 prev_tsc = rte_rdtsc();
2060 do {
2061 nb_rx = rte_eth_rx_burst(port_id, rxq,
2062 pkts_burst, MAX_PKT_BURST);
2063 for (i = 0; i < nb_rx; i++)
2064 rte_pktmbuf_free(pkts_burst[i]);
2065
2066 cur_tsc = rte_rdtsc();
2067 diff_tsc = cur_tsc - prev_tsc;
2068 timer_tsc += diff_tsc;
2069 } while ((nb_rx > 0) &&
2070 (timer_tsc < timer_period));
2071 timer_tsc = 0;
2072 }
2073 }
2074 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */
2075 }
2076 }
2077
2078 static void
run_pkt_fwd_on_lcore(struct fwd_lcore * fc,packet_fwd_t pkt_fwd)2079 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2080 {
2081 struct fwd_stream **fsm;
2082 streamid_t nb_fs;
2083 streamid_t sm_id;
2084 #ifdef RTE_LIB_BITRATESTATS
2085 uint64_t tics_per_1sec;
2086 uint64_t tics_datum;
2087 uint64_t tics_current;
2088 uint16_t i, cnt_ports;
2089
2090 cnt_ports = nb_ports;
2091 tics_datum = rte_rdtsc();
2092 tics_per_1sec = rte_get_timer_hz();
2093 #endif
2094 fsm = &fwd_streams[fc->stream_idx];
2095 nb_fs = fc->stream_nb;
2096 do {
2097 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2098 (*pkt_fwd)(fsm[sm_id]);
2099 #ifdef RTE_LIB_BITRATESTATS
2100 if (bitrate_enabled != 0 &&
2101 bitrate_lcore_id == rte_lcore_id()) {
2102 tics_current = rte_rdtsc();
2103 if (tics_current - tics_datum >= tics_per_1sec) {
2104 /* Periodic bitrate calculation */
2105 for (i = 0; i < cnt_ports; i++)
2106 rte_stats_bitrate_calc(bitrate_data,
2107 ports_ids[i]);
2108 tics_datum = tics_current;
2109 }
2110 }
2111 #endif
2112 #ifdef RTE_LIB_LATENCYSTATS
2113 if (latencystats_enabled != 0 &&
2114 latencystats_lcore_id == rte_lcore_id())
2115 rte_latencystats_update();
2116 #endif
2117
2118 } while (! fc->stopped);
2119 }
2120
2121 static int
start_pkt_forward_on_core(void * fwd_arg)2122 start_pkt_forward_on_core(void *fwd_arg)
2123 {
2124 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2125 cur_fwd_config.fwd_eng->packet_fwd);
2126 return 0;
2127 }
2128
2129 /*
2130 * Run the TXONLY packet forwarding engine to send a single burst of packets.
2131 * Used to start communication flows in network loopback test configurations.
2132 */
2133 static int
run_one_txonly_burst_on_core(void * fwd_arg)2134 run_one_txonly_burst_on_core(void *fwd_arg)
2135 {
2136 struct fwd_lcore *fwd_lc;
2137 struct fwd_lcore tmp_lcore;
2138
2139 fwd_lc = (struct fwd_lcore *) fwd_arg;
2140 tmp_lcore = *fwd_lc;
2141 tmp_lcore.stopped = 1;
2142 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2143 return 0;
2144 }
2145
2146 /*
2147 * Launch packet forwarding:
2148 * - Setup per-port forwarding context.
2149 * - launch logical cores with their forwarding configuration.
2150 */
2151 static void
launch_packet_forwarding(lcore_function_t * pkt_fwd_on_lcore)2152 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2153 {
2154 port_fwd_begin_t port_fwd_begin;
2155 unsigned int i;
2156 unsigned int lc_id;
2157 int diag;
2158
2159 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2160 if (port_fwd_begin != NULL) {
2161 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2162 (*port_fwd_begin)(fwd_ports_ids[i]);
2163 }
2164 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2165 lc_id = fwd_lcores_cpuids[i];
2166 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2167 fwd_lcores[i]->stopped = 0;
2168 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2169 fwd_lcores[i], lc_id);
2170 if (diag != 0)
2171 printf("launch lcore %u failed - diag=%d\n",
2172 lc_id, diag);
2173 }
2174 }
2175 }
2176
2177 /*
2178 * Launch packet forwarding configuration.
2179 */
2180 void
start_packet_forwarding(int with_tx_first)2181 start_packet_forwarding(int with_tx_first)
2182 {
2183 port_fwd_begin_t port_fwd_begin;
2184 port_fwd_end_t port_fwd_end;
2185 struct rte_port *port;
2186 unsigned int i;
2187 portid_t pt_id;
2188
2189 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2190 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2191
2192 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2193 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2194
2195 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2196 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2197 (!nb_rxq || !nb_txq))
2198 rte_exit(EXIT_FAILURE,
2199 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2200 cur_fwd_eng->fwd_mode_name);
2201
2202 if (all_ports_started() == 0) {
2203 printf("Not all ports were started\n");
2204 return;
2205 }
2206 if (test_done == 0) {
2207 printf("Packet forwarding already started\n");
2208 return;
2209 }
2210
2211
2212 if(dcb_test) {
2213 for (i = 0; i < nb_fwd_ports; i++) {
2214 pt_id = fwd_ports_ids[i];
2215 port = &ports[pt_id];
2216 if (!port->dcb_flag) {
2217 printf("In DCB mode, all forwarding ports must "
2218 "be configured in this mode.\n");
2219 return;
2220 }
2221 }
2222 if (nb_fwd_lcores == 1) {
2223 printf("In DCB mode,the nb forwarding cores "
2224 "should be larger than 1.\n");
2225 return;
2226 }
2227 }
2228 test_done = 0;
2229
2230 fwd_config_setup();
2231
2232 if(!no_flush_rx)
2233 flush_fwd_rx_queues();
2234
2235 pkt_fwd_config_display(&cur_fwd_config);
2236 rxtx_config_display();
2237
2238 fwd_stats_reset();
2239 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2240 pt_id = fwd_ports_ids[i];
2241 port = &ports[pt_id];
2242 map_port_queue_stats_mapping_registers(pt_id, port);
2243 }
2244 if (with_tx_first) {
2245 port_fwd_begin = tx_only_engine.port_fwd_begin;
2246 if (port_fwd_begin != NULL) {
2247 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2248 (*port_fwd_begin)(fwd_ports_ids[i]);
2249 }
2250 while (with_tx_first--) {
2251 launch_packet_forwarding(
2252 run_one_txonly_burst_on_core);
2253 rte_eal_mp_wait_lcore();
2254 }
2255 port_fwd_end = tx_only_engine.port_fwd_end;
2256 if (port_fwd_end != NULL) {
2257 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2258 (*port_fwd_end)(fwd_ports_ids[i]);
2259 }
2260 }
2261 launch_packet_forwarding(start_pkt_forward_on_core);
2262 }
2263
2264 void
stop_packet_forwarding(void)2265 stop_packet_forwarding(void)
2266 {
2267 port_fwd_end_t port_fwd_end;
2268 lcoreid_t lc_id;
2269 portid_t pt_id;
2270 int i;
2271
2272 if (test_done) {
2273 printf("Packet forwarding not started\n");
2274 return;
2275 }
2276 printf("Telling cores to stop...");
2277 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2278 fwd_lcores[lc_id]->stopped = 1;
2279 printf("\nWaiting for lcores to finish...\n");
2280 rte_eal_mp_wait_lcore();
2281 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2282 if (port_fwd_end != NULL) {
2283 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2284 pt_id = fwd_ports_ids[i];
2285 (*port_fwd_end)(pt_id);
2286 }
2287 }
2288
2289 fwd_stats_display();
2290
2291 printf("\nDone.\n");
2292 test_done = 1;
2293 }
2294
2295 void
dev_set_link_up(portid_t pid)2296 dev_set_link_up(portid_t pid)
2297 {
2298 if (rte_eth_dev_set_link_up(pid) < 0)
2299 printf("\nSet link up fail.\n");
2300 }
2301
2302 void
dev_set_link_down(portid_t pid)2303 dev_set_link_down(portid_t pid)
2304 {
2305 if (rte_eth_dev_set_link_down(pid) < 0)
2306 printf("\nSet link down fail.\n");
2307 }
2308
2309 static int
all_ports_started(void)2310 all_ports_started(void)
2311 {
2312 portid_t pi;
2313 struct rte_port *port;
2314
2315 RTE_ETH_FOREACH_DEV(pi) {
2316 port = &ports[pi];
2317 /* Check if there is a port which is not started */
2318 if ((port->port_status != RTE_PORT_STARTED) &&
2319 (port->slave_flag == 0))
2320 return 0;
2321 }
2322
2323 /* No port is not started */
2324 return 1;
2325 }
2326
2327 int
port_is_stopped(portid_t port_id)2328 port_is_stopped(portid_t port_id)
2329 {
2330 struct rte_port *port = &ports[port_id];
2331
2332 if ((port->port_status != RTE_PORT_STOPPED) &&
2333 (port->slave_flag == 0))
2334 return 0;
2335 return 1;
2336 }
2337
2338 int
all_ports_stopped(void)2339 all_ports_stopped(void)
2340 {
2341 portid_t pi;
2342
2343 RTE_ETH_FOREACH_DEV(pi) {
2344 if (!port_is_stopped(pi))
2345 return 0;
2346 }
2347
2348 return 1;
2349 }
2350
2351 int
port_is_started(portid_t port_id)2352 port_is_started(portid_t port_id)
2353 {
2354 if (port_id_is_invalid(port_id, ENABLED_WARN))
2355 return 0;
2356
2357 if (ports[port_id].port_status != RTE_PORT_STARTED)
2358 return 0;
2359
2360 return 1;
2361 }
2362
2363 /* Configure the Rx and Tx hairpin queues for the selected port. */
2364 static int
setup_hairpin_queues(portid_t pi,portid_t p_pi,uint16_t cnt_pi)2365 setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2366 {
2367 queueid_t qi;
2368 struct rte_eth_hairpin_conf hairpin_conf = {
2369 .peer_count = 1,
2370 };
2371 int i;
2372 int diag;
2373 struct rte_port *port = &ports[pi];
2374 uint16_t peer_rx_port = pi;
2375 uint16_t peer_tx_port = pi;
2376 uint32_t manual = 1;
2377 uint32_t tx_exp = hairpin_mode & 0x10;
2378
2379 if (!(hairpin_mode & 0xf)) {
2380 peer_rx_port = pi;
2381 peer_tx_port = pi;
2382 manual = 0;
2383 } else if (hairpin_mode & 0x1) {
2384 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2385 RTE_ETH_DEV_NO_OWNER);
2386 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2387 peer_tx_port = rte_eth_find_next_owned_by(0,
2388 RTE_ETH_DEV_NO_OWNER);
2389 if (p_pi != RTE_MAX_ETHPORTS) {
2390 peer_rx_port = p_pi;
2391 } else {
2392 uint16_t next_pi;
2393
2394 /* Last port will be the peer RX port of the first. */
2395 RTE_ETH_FOREACH_DEV(next_pi)
2396 peer_rx_port = next_pi;
2397 }
2398 manual = 1;
2399 } else if (hairpin_mode & 0x2) {
2400 if (cnt_pi & 0x1) {
2401 peer_rx_port = p_pi;
2402 } else {
2403 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2404 RTE_ETH_DEV_NO_OWNER);
2405 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2406 peer_rx_port = pi;
2407 }
2408 peer_tx_port = peer_rx_port;
2409 manual = 1;
2410 }
2411
2412 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2413 hairpin_conf.peers[0].port = peer_rx_port;
2414 hairpin_conf.peers[0].queue = i + nb_rxq;
2415 hairpin_conf.manual_bind = !!manual;
2416 hairpin_conf.tx_explicit = !!tx_exp;
2417 diag = rte_eth_tx_hairpin_queue_setup
2418 (pi, qi, nb_txd, &hairpin_conf);
2419 i++;
2420 if (diag == 0)
2421 continue;
2422
2423 /* Fail to setup rx queue, return */
2424 if (rte_atomic16_cmpset(&(port->port_status),
2425 RTE_PORT_HANDLING,
2426 RTE_PORT_STOPPED) == 0)
2427 printf("Port %d can not be set back "
2428 "to stopped\n", pi);
2429 printf("Fail to configure port %d hairpin "
2430 "queues\n", pi);
2431 /* try to reconfigure queues next time */
2432 port->need_reconfig_queues = 1;
2433 return -1;
2434 }
2435 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2436 hairpin_conf.peers[0].port = peer_tx_port;
2437 hairpin_conf.peers[0].queue = i + nb_txq;
2438 hairpin_conf.manual_bind = !!manual;
2439 hairpin_conf.tx_explicit = !!tx_exp;
2440 diag = rte_eth_rx_hairpin_queue_setup
2441 (pi, qi, nb_rxd, &hairpin_conf);
2442 i++;
2443 if (diag == 0)
2444 continue;
2445
2446 /* Fail to setup rx queue, return */
2447 if (rte_atomic16_cmpset(&(port->port_status),
2448 RTE_PORT_HANDLING,
2449 RTE_PORT_STOPPED) == 0)
2450 printf("Port %d can not be set back "
2451 "to stopped\n", pi);
2452 printf("Fail to configure port %d hairpin "
2453 "queues\n", pi);
2454 /* try to reconfigure queues next time */
2455 port->need_reconfig_queues = 1;
2456 return -1;
2457 }
2458 return 0;
2459 }
2460
2461 /* Configure the Rx with optional split. */
2462 int
rx_queue_setup(uint16_t port_id,uint16_t rx_queue_id,uint16_t nb_rx_desc,unsigned int socket_id,struct rte_eth_rxconf * rx_conf,struct rte_mempool * mp)2463 rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2464 uint16_t nb_rx_desc, unsigned int socket_id,
2465 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2466 {
2467 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2468 unsigned int i, mp_n;
2469 int ret;
2470
2471 if (rx_pkt_nb_segs <= 1 ||
2472 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2473 rx_conf->rx_seg = NULL;
2474 rx_conf->rx_nseg = 0;
2475 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2476 nb_rx_desc, socket_id,
2477 rx_conf, mp);
2478 return ret;
2479 }
2480 for (i = 0; i < rx_pkt_nb_segs; i++) {
2481 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2482 struct rte_mempool *mpx;
2483 /*
2484 * Use last valid pool for the segments with number
2485 * exceeding the pool index.
2486 */
2487 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2488 mpx = mbuf_pool_find(socket_id, mp_n);
2489 /* Handle zero as mbuf data buffer size. */
2490 rx_seg->length = rx_pkt_seg_lengths[i] ?
2491 rx_pkt_seg_lengths[i] :
2492 mbuf_data_size[mp_n];
2493 rx_seg->offset = i < rx_pkt_nb_offs ?
2494 rx_pkt_seg_offsets[i] : 0;
2495 rx_seg->mp = mpx ? mpx : mp;
2496 }
2497 rx_conf->rx_nseg = rx_pkt_nb_segs;
2498 rx_conf->rx_seg = rx_useg;
2499 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2500 socket_id, rx_conf, NULL);
2501 rx_conf->rx_seg = NULL;
2502 rx_conf->rx_nseg = 0;
2503 return ret;
2504 }
2505
2506 int
start_port(portid_t pid)2507 start_port(portid_t pid)
2508 {
2509 int diag, need_check_link_status = -1;
2510 portid_t pi;
2511 portid_t p_pi = RTE_MAX_ETHPORTS;
2512 portid_t pl[RTE_MAX_ETHPORTS];
2513 portid_t peer_pl[RTE_MAX_ETHPORTS];
2514 uint16_t cnt_pi = 0;
2515 uint16_t cfg_pi = 0;
2516 int peer_pi;
2517 queueid_t qi;
2518 struct rte_port *port;
2519 struct rte_ether_addr mac_addr;
2520 struct rte_eth_hairpin_cap cap;
2521
2522 if (port_id_is_invalid(pid, ENABLED_WARN))
2523 return 0;
2524
2525 if(dcb_config)
2526 dcb_test = 1;
2527 RTE_ETH_FOREACH_DEV(pi) {
2528 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2529 continue;
2530
2531 need_check_link_status = 0;
2532 port = &ports[pi];
2533 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2534 RTE_PORT_HANDLING) == 0) {
2535 printf("Port %d is now not stopped\n", pi);
2536 continue;
2537 }
2538
2539 if (port->need_reconfig > 0) {
2540 port->need_reconfig = 0;
2541
2542 if (flow_isolate_all) {
2543 int ret = port_flow_isolate(pi, 1);
2544 if (ret) {
2545 printf("Failed to apply isolated"
2546 " mode on port %d\n", pi);
2547 return -1;
2548 }
2549 }
2550 configure_rxtx_dump_callbacks(0);
2551 printf("Configuring Port %d (socket %u)\n", pi,
2552 port->socket_id);
2553 if (nb_hairpinq > 0 &&
2554 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2555 printf("Port %d doesn't support hairpin "
2556 "queues\n", pi);
2557 return -1;
2558 }
2559 /* configure port */
2560 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2561 nb_txq + nb_hairpinq,
2562 &(port->dev_conf));
2563 if (diag != 0) {
2564 if (rte_atomic16_cmpset(&(port->port_status),
2565 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2566 printf("Port %d can not be set back "
2567 "to stopped\n", pi);
2568 printf("Fail to configure port %d\n", pi);
2569 /* try to reconfigure port next time */
2570 port->need_reconfig = 1;
2571 return -1;
2572 }
2573 }
2574 if (port->need_reconfig_queues > 0) {
2575 port->need_reconfig_queues = 0;
2576 /* setup tx queues */
2577 for (qi = 0; qi < nb_txq; qi++) {
2578 if ((numa_support) &&
2579 (txring_numa[pi] != NUMA_NO_CONFIG))
2580 diag = rte_eth_tx_queue_setup(pi, qi,
2581 port->nb_tx_desc[qi],
2582 txring_numa[pi],
2583 &(port->tx_conf[qi]));
2584 else
2585 diag = rte_eth_tx_queue_setup(pi, qi,
2586 port->nb_tx_desc[qi],
2587 port->socket_id,
2588 &(port->tx_conf[qi]));
2589
2590 if (diag == 0)
2591 continue;
2592
2593 /* Fail to setup tx queue, return */
2594 if (rte_atomic16_cmpset(&(port->port_status),
2595 RTE_PORT_HANDLING,
2596 RTE_PORT_STOPPED) == 0)
2597 printf("Port %d can not be set back "
2598 "to stopped\n", pi);
2599 printf("Fail to configure port %d tx queues\n",
2600 pi);
2601 /* try to reconfigure queues next time */
2602 port->need_reconfig_queues = 1;
2603 return -1;
2604 }
2605 for (qi = 0; qi < nb_rxq; qi++) {
2606 /* setup rx queues */
2607 if ((numa_support) &&
2608 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2609 struct rte_mempool * mp =
2610 mbuf_pool_find
2611 (rxring_numa[pi], 0);
2612 if (mp == NULL) {
2613 printf("Failed to setup RX queue:"
2614 "No mempool allocation"
2615 " on the socket %d\n",
2616 rxring_numa[pi]);
2617 return -1;
2618 }
2619
2620 diag = rx_queue_setup(pi, qi,
2621 port->nb_rx_desc[qi],
2622 rxring_numa[pi],
2623 &(port->rx_conf[qi]),
2624 mp);
2625 } else {
2626 struct rte_mempool *mp =
2627 mbuf_pool_find
2628 (port->socket_id, 0);
2629 if (mp == NULL) {
2630 printf("Failed to setup RX queue:"
2631 "No mempool allocation"
2632 " on the socket %d\n",
2633 port->socket_id);
2634 return -1;
2635 }
2636 diag = rx_queue_setup(pi, qi,
2637 port->nb_rx_desc[qi],
2638 port->socket_id,
2639 &(port->rx_conf[qi]),
2640 mp);
2641 }
2642 if (diag == 0)
2643 continue;
2644
2645 /* Fail to setup rx queue, return */
2646 if (rte_atomic16_cmpset(&(port->port_status),
2647 RTE_PORT_HANDLING,
2648 RTE_PORT_STOPPED) == 0)
2649 printf("Port %d can not be set back "
2650 "to stopped\n", pi);
2651 printf("Fail to configure port %d rx queues\n",
2652 pi);
2653 /* try to reconfigure queues next time */
2654 port->need_reconfig_queues = 1;
2655 return -1;
2656 }
2657 /* setup hairpin queues */
2658 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2659 return -1;
2660 }
2661 configure_rxtx_dump_callbacks(verbose_level);
2662 if (clear_ptypes) {
2663 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2664 NULL, 0);
2665 if (diag < 0)
2666 printf(
2667 "Port %d: Failed to disable Ptype parsing\n",
2668 pi);
2669 }
2670
2671 p_pi = pi;
2672 cnt_pi++;
2673
2674 /* start port */
2675 if (rte_eth_dev_start(pi) < 0) {
2676 printf("Fail to start port %d\n", pi);
2677
2678 /* Fail to setup rx queue, return */
2679 if (rte_atomic16_cmpset(&(port->port_status),
2680 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2681 printf("Port %d can not be set back to "
2682 "stopped\n", pi);
2683 continue;
2684 }
2685
2686 if (rte_atomic16_cmpset(&(port->port_status),
2687 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2688 printf("Port %d can not be set into started\n", pi);
2689
2690 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2691 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2692 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2693 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2694 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2695
2696 /* at least one port started, need checking link status */
2697 need_check_link_status = 1;
2698
2699 pl[cfg_pi++] = pi;
2700 }
2701
2702 if (need_check_link_status == 1 && !no_link_check)
2703 check_all_ports_link_status(RTE_PORT_ALL);
2704 else if (need_check_link_status == 0)
2705 printf("Please stop the ports first\n");
2706
2707 if (hairpin_mode & 0xf) {
2708 uint16_t i;
2709 int j;
2710
2711 /* bind all started hairpin ports */
2712 for (i = 0; i < cfg_pi; i++) {
2713 pi = pl[i];
2714 /* bind current Tx to all peer Rx */
2715 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2716 RTE_MAX_ETHPORTS, 1);
2717 if (peer_pi < 0)
2718 return peer_pi;
2719 for (j = 0; j < peer_pi; j++) {
2720 if (!port_is_started(peer_pl[j]))
2721 continue;
2722 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2723 if (diag < 0) {
2724 printf("Error during binding hairpin"
2725 " Tx port %u to %u: %s\n",
2726 pi, peer_pl[j],
2727 rte_strerror(-diag));
2728 return -1;
2729 }
2730 }
2731 /* bind all peer Tx to current Rx */
2732 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2733 RTE_MAX_ETHPORTS, 0);
2734 if (peer_pi < 0)
2735 return peer_pi;
2736 for (j = 0; j < peer_pi; j++) {
2737 if (!port_is_started(peer_pl[j]))
2738 continue;
2739 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2740 if (diag < 0) {
2741 printf("Error during binding hairpin"
2742 " Tx port %u to %u: %s\n",
2743 peer_pl[j], pi,
2744 rte_strerror(-diag));
2745 return -1;
2746 }
2747 }
2748 }
2749 }
2750
2751 printf("Done\n");
2752 return 0;
2753 }
2754
2755 void
stop_port(portid_t pid)2756 stop_port(portid_t pid)
2757 {
2758 portid_t pi;
2759 struct rte_port *port;
2760 int need_check_link_status = 0;
2761 portid_t peer_pl[RTE_MAX_ETHPORTS];
2762 int peer_pi;
2763
2764 if (dcb_test) {
2765 dcb_test = 0;
2766 dcb_config = 0;
2767 }
2768
2769 if (port_id_is_invalid(pid, ENABLED_WARN))
2770 return;
2771
2772 printf("Stopping ports...\n");
2773
2774 RTE_ETH_FOREACH_DEV(pi) {
2775 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2776 continue;
2777
2778 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2779 printf("Please remove port %d from forwarding configuration.\n", pi);
2780 continue;
2781 }
2782
2783 if (port_is_bonding_slave(pi)) {
2784 printf("Please remove port %d from bonded device.\n", pi);
2785 continue;
2786 }
2787
2788 port = &ports[pi];
2789 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2790 RTE_PORT_HANDLING) == 0)
2791 continue;
2792
2793 if (hairpin_mode & 0xf) {
2794 int j;
2795
2796 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2797 /* unbind all peer Tx from current Rx */
2798 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2799 RTE_MAX_ETHPORTS, 0);
2800 if (peer_pi < 0)
2801 continue;
2802 for (j = 0; j < peer_pi; j++) {
2803 if (!port_is_started(peer_pl[j]))
2804 continue;
2805 rte_eth_hairpin_unbind(peer_pl[j], pi);
2806 }
2807 }
2808
2809 if (rte_eth_dev_stop(pi) != 0)
2810 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2811 pi);
2812
2813 if (rte_atomic16_cmpset(&(port->port_status),
2814 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2815 printf("Port %d can not be set into stopped\n", pi);
2816 need_check_link_status = 1;
2817 }
2818 if (need_check_link_status && !no_link_check)
2819 check_all_ports_link_status(RTE_PORT_ALL);
2820
2821 printf("Done\n");
2822 }
2823
2824 static void
remove_invalid_ports_in(portid_t * array,portid_t * total)2825 remove_invalid_ports_in(portid_t *array, portid_t *total)
2826 {
2827 portid_t i;
2828 portid_t new_total = 0;
2829
2830 for (i = 0; i < *total; i++)
2831 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2832 array[new_total] = array[i];
2833 new_total++;
2834 }
2835 *total = new_total;
2836 }
2837
2838 static void
remove_invalid_ports(void)2839 remove_invalid_ports(void)
2840 {
2841 remove_invalid_ports_in(ports_ids, &nb_ports);
2842 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2843 nb_cfg_ports = nb_fwd_ports;
2844 }
2845
2846 void
close_port(portid_t pid)2847 close_port(portid_t pid)
2848 {
2849 portid_t pi;
2850 struct rte_port *port;
2851
2852 if (port_id_is_invalid(pid, ENABLED_WARN))
2853 return;
2854
2855 printf("Closing ports...\n");
2856
2857 RTE_ETH_FOREACH_DEV(pi) {
2858 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2859 continue;
2860
2861 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2862 printf("Please remove port %d from forwarding configuration.\n", pi);
2863 continue;
2864 }
2865
2866 if (port_is_bonding_slave(pi)) {
2867 printf("Please remove port %d from bonded device.\n", pi);
2868 continue;
2869 }
2870
2871 port = &ports[pi];
2872 if (rte_atomic16_cmpset(&(port->port_status),
2873 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2874 printf("Port %d is already closed\n", pi);
2875 continue;
2876 }
2877
2878 port_flow_flush(pi);
2879 rte_eth_dev_close(pi);
2880 }
2881
2882 remove_invalid_ports();
2883 printf("Done\n");
2884 }
2885
2886 void
reset_port(portid_t pid)2887 reset_port(portid_t pid)
2888 {
2889 int diag;
2890 portid_t pi;
2891 struct rte_port *port;
2892
2893 if (port_id_is_invalid(pid, ENABLED_WARN))
2894 return;
2895
2896 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2897 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2898 printf("Can not reset port(s), please stop port(s) first.\n");
2899 return;
2900 }
2901
2902 printf("Resetting ports...\n");
2903
2904 RTE_ETH_FOREACH_DEV(pi) {
2905 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2906 continue;
2907
2908 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2909 printf("Please remove port %d from forwarding "
2910 "configuration.\n", pi);
2911 continue;
2912 }
2913
2914 if (port_is_bonding_slave(pi)) {
2915 printf("Please remove port %d from bonded device.\n",
2916 pi);
2917 continue;
2918 }
2919
2920 diag = rte_eth_dev_reset(pi);
2921 if (diag == 0) {
2922 port = &ports[pi];
2923 port->need_reconfig = 1;
2924 port->need_reconfig_queues = 1;
2925 } else {
2926 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2927 }
2928 }
2929
2930 printf("Done\n");
2931 }
2932
2933 void
attach_port(char * identifier)2934 attach_port(char *identifier)
2935 {
2936 portid_t pi;
2937 struct rte_dev_iterator iterator;
2938
2939 printf("Attaching a new port...\n");
2940
2941 if (identifier == NULL) {
2942 printf("Invalid parameters are specified\n");
2943 return;
2944 }
2945
2946 if (rte_dev_probe(identifier) < 0) {
2947 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2948 return;
2949 }
2950
2951 /* first attach mode: event */
2952 if (setup_on_probe_event) {
2953 /* new ports are detected on RTE_ETH_EVENT_NEW event */
2954 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2955 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2956 ports[pi].need_setup != 0)
2957 setup_attached_port(pi);
2958 return;
2959 }
2960
2961 /* second attach mode: iterator */
2962 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2963 /* setup ports matching the devargs used for probing */
2964 if (port_is_forwarding(pi))
2965 continue; /* port was already attached before */
2966 setup_attached_port(pi);
2967 }
2968 }
2969
2970 static void
setup_attached_port(portid_t pi)2971 setup_attached_port(portid_t pi)
2972 {
2973 unsigned int socket_id;
2974 int ret;
2975
2976 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2977 /* if socket_id is invalid, set to the first available socket. */
2978 if (check_socket_id(socket_id) < 0)
2979 socket_id = socket_ids[0];
2980 reconfig(pi, socket_id);
2981 ret = rte_eth_promiscuous_enable(pi);
2982 if (ret != 0)
2983 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2984 pi, rte_strerror(-ret));
2985
2986 ports_ids[nb_ports++] = pi;
2987 fwd_ports_ids[nb_fwd_ports++] = pi;
2988 nb_cfg_ports = nb_fwd_ports;
2989 ports[pi].need_setup = 0;
2990 ports[pi].port_status = RTE_PORT_STOPPED;
2991
2992 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2993 printf("Done\n");
2994 }
2995
2996 static void
detach_device(struct rte_device * dev)2997 detach_device(struct rte_device *dev)
2998 {
2999 portid_t sibling;
3000
3001 if (dev == NULL) {
3002 printf("Device already removed\n");
3003 return;
3004 }
3005
3006 printf("Removing a device...\n");
3007
3008 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3009 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3010 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3011 printf("Port %u not stopped\n", sibling);
3012 return;
3013 }
3014 port_flow_flush(sibling);
3015 }
3016 }
3017
3018 if (rte_dev_remove(dev) < 0) {
3019 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3020 return;
3021 }
3022 remove_invalid_ports();
3023
3024 printf("Device is detached\n");
3025 printf("Now total ports is %d\n", nb_ports);
3026 printf("Done\n");
3027 return;
3028 }
3029
3030 void
detach_port_device(portid_t port_id)3031 detach_port_device(portid_t port_id)
3032 {
3033 if (port_id_is_invalid(port_id, ENABLED_WARN))
3034 return;
3035
3036 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3037 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3038 printf("Port not stopped\n");
3039 return;
3040 }
3041 printf("Port was not closed\n");
3042 }
3043
3044 detach_device(rte_eth_devices[port_id].device);
3045 }
3046
3047 void
detach_devargs(char * identifier)3048 detach_devargs(char *identifier)
3049 {
3050 struct rte_dev_iterator iterator;
3051 struct rte_devargs da;
3052 portid_t port_id;
3053
3054 printf("Removing a device...\n");
3055
3056 memset(&da, 0, sizeof(da));
3057 if (rte_devargs_parsef(&da, "%s", identifier)) {
3058 printf("cannot parse identifier\n");
3059 if (da.args)
3060 free(da.args);
3061 return;
3062 }
3063
3064 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3065 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3066 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3067 printf("Port %u not stopped\n", port_id);
3068 rte_eth_iterator_cleanup(&iterator);
3069 return;
3070 }
3071 port_flow_flush(port_id);
3072 }
3073 }
3074
3075 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3076 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3077 da.name, da.bus->name);
3078 return;
3079 }
3080
3081 remove_invalid_ports();
3082
3083 printf("Device %s is detached\n", identifier);
3084 printf("Now total ports is %d\n", nb_ports);
3085 printf("Done\n");
3086 }
3087
3088 void
pmd_test_exit(void)3089 pmd_test_exit(void)
3090 {
3091 portid_t pt_id;
3092 unsigned int i;
3093 int ret;
3094
3095 if (test_done == 0)
3096 stop_packet_forwarding();
3097
3098 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3099 if (mempools[i]) {
3100 if (mp_alloc_type == MP_ALLOC_ANON)
3101 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3102 NULL);
3103 }
3104 }
3105 if (ports != NULL) {
3106 no_link_check = 1;
3107 RTE_ETH_FOREACH_DEV(pt_id) {
3108 printf("\nStopping port %d...\n", pt_id);
3109 fflush(stdout);
3110 stop_port(pt_id);
3111 }
3112 RTE_ETH_FOREACH_DEV(pt_id) {
3113 printf("\nShutting down port %d...\n", pt_id);
3114 fflush(stdout);
3115 close_port(pt_id);
3116 }
3117 }
3118
3119 if (hot_plug) {
3120 ret = rte_dev_event_monitor_stop();
3121 if (ret) {
3122 RTE_LOG(ERR, EAL,
3123 "fail to stop device event monitor.");
3124 return;
3125 }
3126
3127 ret = rte_dev_event_callback_unregister(NULL,
3128 dev_event_callback, NULL);
3129 if (ret < 0) {
3130 RTE_LOG(ERR, EAL,
3131 "fail to unregister device event callback.\n");
3132 return;
3133 }
3134
3135 ret = rte_dev_hotplug_handle_disable();
3136 if (ret) {
3137 RTE_LOG(ERR, EAL,
3138 "fail to disable hotplug handling.\n");
3139 return;
3140 }
3141 }
3142 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3143 if (mempools[i])
3144 rte_mempool_free(mempools[i]);
3145 }
3146
3147 printf("\nBye...\n");
3148 }
3149
3150 typedef void (*cmd_func_t)(void);
3151 struct pmd_test_command {
3152 const char *cmd_name;
3153 cmd_func_t cmd_func;
3154 };
3155
3156 /* Check the link status of all ports in up to 9s, and print them finally */
3157 static void
check_all_ports_link_status(uint32_t port_mask)3158 check_all_ports_link_status(uint32_t port_mask)
3159 {
3160 #define CHECK_INTERVAL 100 /* 100ms */
3161 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
3162 portid_t portid;
3163 uint8_t count, all_ports_up, print_flag = 0;
3164 struct rte_eth_link link;
3165 int ret;
3166 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3167
3168 printf("Checking link statuses...\n");
3169 fflush(stdout);
3170 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3171 all_ports_up = 1;
3172 RTE_ETH_FOREACH_DEV(portid) {
3173 if ((port_mask & (1 << portid)) == 0)
3174 continue;
3175 memset(&link, 0, sizeof(link));
3176 ret = rte_eth_link_get_nowait(portid, &link);
3177 if (ret < 0) {
3178 all_ports_up = 0;
3179 if (print_flag == 1)
3180 printf("Port %u link get failed: %s\n",
3181 portid, rte_strerror(-ret));
3182 continue;
3183 }
3184 /* print link status if flag set */
3185 if (print_flag == 1) {
3186 rte_eth_link_to_str(link_status,
3187 sizeof(link_status), &link);
3188 printf("Port %d %s\n", portid, link_status);
3189 continue;
3190 }
3191 /* clear all_ports_up flag if any link down */
3192 if (link.link_status == ETH_LINK_DOWN) {
3193 all_ports_up = 0;
3194 break;
3195 }
3196 }
3197 /* after finally printing all link status, get out */
3198 if (print_flag == 1)
3199 break;
3200
3201 if (all_ports_up == 0) {
3202 fflush(stdout);
3203 rte_delay_ms(CHECK_INTERVAL);
3204 }
3205
3206 /* set the print_flag if all ports up or timeout */
3207 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3208 print_flag = 1;
3209 }
3210
3211 if (lsc_interrupt)
3212 break;
3213 }
3214 }
3215
3216 static void
rmv_port_callback(void * arg)3217 rmv_port_callback(void *arg)
3218 {
3219 int need_to_start = 0;
3220 int org_no_link_check = no_link_check;
3221 portid_t port_id = (intptr_t)arg;
3222 struct rte_device *dev;
3223
3224 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3225
3226 if (!test_done && port_is_forwarding(port_id)) {
3227 need_to_start = 1;
3228 stop_packet_forwarding();
3229 }
3230 no_link_check = 1;
3231 stop_port(port_id);
3232 no_link_check = org_no_link_check;
3233
3234 /* Save rte_device pointer before closing ethdev port */
3235 dev = rte_eth_devices[port_id].device;
3236 close_port(port_id);
3237 detach_device(dev); /* might be already removed or have more ports */
3238
3239 if (need_to_start)
3240 start_packet_forwarding(0);
3241 }
3242
3243 /* This function is used by the interrupt thread */
3244 static int
eth_event_callback(portid_t port_id,enum rte_eth_event_type type,void * param,void * ret_param)3245 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3246 void *ret_param)
3247 {
3248 RTE_SET_USED(param);
3249 RTE_SET_USED(ret_param);
3250
3251 if (type >= RTE_ETH_EVENT_MAX) {
3252 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3253 port_id, __func__, type);
3254 fflush(stderr);
3255 } else if (event_print_mask & (UINT32_C(1) << type)) {
3256 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3257 eth_event_desc[type]);
3258 fflush(stdout);
3259 }
3260
3261 switch (type) {
3262 case RTE_ETH_EVENT_NEW:
3263 ports[port_id].need_setup = 1;
3264 ports[port_id].port_status = RTE_PORT_HANDLING;
3265 break;
3266 case RTE_ETH_EVENT_INTR_RMV:
3267 if (port_id_is_invalid(port_id, DISABLED_WARN))
3268 break;
3269 if (rte_eal_alarm_set(100000,
3270 rmv_port_callback, (void *)(intptr_t)port_id))
3271 fprintf(stderr, "Could not set up deferred device removal\n");
3272 break;
3273 case RTE_ETH_EVENT_DESTROY:
3274 ports[port_id].port_status = RTE_PORT_CLOSED;
3275 printf("Port %u is closed\n", port_id);
3276 break;
3277 default:
3278 break;
3279 }
3280 return 0;
3281 }
3282
3283 static int
register_eth_event_callback(void)3284 register_eth_event_callback(void)
3285 {
3286 int ret;
3287 enum rte_eth_event_type event;
3288
3289 for (event = RTE_ETH_EVENT_UNKNOWN;
3290 event < RTE_ETH_EVENT_MAX; event++) {
3291 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3292 event,
3293 eth_event_callback,
3294 NULL);
3295 if (ret != 0) {
3296 TESTPMD_LOG(ERR, "Failed to register callback for "
3297 "%s event\n", eth_event_desc[event]);
3298 return -1;
3299 }
3300 }
3301
3302 return 0;
3303 }
3304
3305 /* This function is used by the interrupt thread */
3306 static void
dev_event_callback(const char * device_name,enum rte_dev_event_type type,__rte_unused void * arg)3307 dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3308 __rte_unused void *arg)
3309 {
3310 uint16_t port_id;
3311 int ret;
3312
3313 if (type >= RTE_DEV_EVENT_MAX) {
3314 fprintf(stderr, "%s called upon invalid event %d\n",
3315 __func__, type);
3316 fflush(stderr);
3317 }
3318
3319 switch (type) {
3320 case RTE_DEV_EVENT_REMOVE:
3321 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3322 device_name);
3323 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3324 if (ret) {
3325 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3326 device_name);
3327 return;
3328 }
3329 /*
3330 * Because the user's callback is invoked in eal interrupt
3331 * callback, the interrupt callback need to be finished before
3332 * it can be unregistered when detaching device. So finish
3333 * callback soon and use a deferred removal to detach device
3334 * is need. It is a workaround, once the device detaching be
3335 * moved into the eal in the future, the deferred removal could
3336 * be deleted.
3337 */
3338 if (rte_eal_alarm_set(100000,
3339 rmv_port_callback, (void *)(intptr_t)port_id))
3340 RTE_LOG(ERR, EAL,
3341 "Could not set up deferred device removal\n");
3342 break;
3343 case RTE_DEV_EVENT_ADD:
3344 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3345 device_name);
3346 /* TODO: After finish kernel driver binding,
3347 * begin to attach port.
3348 */
3349 break;
3350 default:
3351 break;
3352 }
3353 }
3354
3355 static int
set_tx_queue_stats_mapping_registers(portid_t port_id,struct rte_port * port)3356 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3357 {
3358 uint16_t i;
3359 int diag;
3360 uint8_t mapping_found = 0;
3361
3362 for (i = 0; i < nb_tx_queue_stats_mappings; i++) {
3363 if ((tx_queue_stats_mappings[i].port_id == port_id) &&
3364 (tx_queue_stats_mappings[i].queue_id < nb_txq )) {
3365 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id,
3366 tx_queue_stats_mappings[i].queue_id,
3367 tx_queue_stats_mappings[i].stats_counter_id);
3368 if (diag != 0)
3369 return diag;
3370 mapping_found = 1;
3371 }
3372 }
3373 if (mapping_found)
3374 port->tx_queue_stats_mapping_enabled = 1;
3375 return 0;
3376 }
3377
3378 static int
set_rx_queue_stats_mapping_registers(portid_t port_id,struct rte_port * port)3379 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port)
3380 {
3381 uint16_t i;
3382 int diag;
3383 uint8_t mapping_found = 0;
3384
3385 for (i = 0; i < nb_rx_queue_stats_mappings; i++) {
3386 if ((rx_queue_stats_mappings[i].port_id == port_id) &&
3387 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) {
3388 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id,
3389 rx_queue_stats_mappings[i].queue_id,
3390 rx_queue_stats_mappings[i].stats_counter_id);
3391 if (diag != 0)
3392 return diag;
3393 mapping_found = 1;
3394 }
3395 }
3396 if (mapping_found)
3397 port->rx_queue_stats_mapping_enabled = 1;
3398 return 0;
3399 }
3400
3401 static void
map_port_queue_stats_mapping_registers(portid_t pi,struct rte_port * port)3402 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port)
3403 {
3404 int diag = 0;
3405
3406 diag = set_tx_queue_stats_mapping_registers(pi, port);
3407 if (diag != 0) {
3408 if (diag == -ENOTSUP) {
3409 port->tx_queue_stats_mapping_enabled = 0;
3410 printf("TX queue stats mapping not supported port id=%d\n", pi);
3411 }
3412 else
3413 rte_exit(EXIT_FAILURE,
3414 "set_tx_queue_stats_mapping_registers "
3415 "failed for port id=%d diag=%d\n",
3416 pi, diag);
3417 }
3418
3419 diag = set_rx_queue_stats_mapping_registers(pi, port);
3420 if (diag != 0) {
3421 if (diag == -ENOTSUP) {
3422 port->rx_queue_stats_mapping_enabled = 0;
3423 printf("RX queue stats mapping not supported port id=%d\n", pi);
3424 }
3425 else
3426 rte_exit(EXIT_FAILURE,
3427 "set_rx_queue_stats_mapping_registers "
3428 "failed for port id=%d diag=%d\n",
3429 pi, diag);
3430 }
3431 }
3432
3433 static void
rxtx_port_config(struct rte_port * port)3434 rxtx_port_config(struct rte_port *port)
3435 {
3436 uint16_t qid;
3437 uint64_t offloads;
3438
3439 for (qid = 0; qid < nb_rxq; qid++) {
3440 offloads = port->rx_conf[qid].offloads;
3441 port->rx_conf[qid] = port->dev_info.default_rxconf;
3442 if (offloads != 0)
3443 port->rx_conf[qid].offloads = offloads;
3444
3445 /* Check if any Rx parameters have been passed */
3446 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3447 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3448
3449 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3450 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3451
3452 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3453 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3454
3455 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3456 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3457
3458 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3459 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3460
3461 port->nb_rx_desc[qid] = nb_rxd;
3462 }
3463
3464 for (qid = 0; qid < nb_txq; qid++) {
3465 offloads = port->tx_conf[qid].offloads;
3466 port->tx_conf[qid] = port->dev_info.default_txconf;
3467 if (offloads != 0)
3468 port->tx_conf[qid].offloads = offloads;
3469
3470 /* Check if any Tx parameters have been passed */
3471 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3472 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3473
3474 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3475 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3476
3477 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3478 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3479
3480 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3481 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3482
3483 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3484 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3485
3486 port->nb_tx_desc[qid] = nb_txd;
3487 }
3488 }
3489
3490 void
init_port_config(void)3491 init_port_config(void)
3492 {
3493 portid_t pid;
3494 struct rte_port *port;
3495 int ret;
3496
3497 RTE_ETH_FOREACH_DEV(pid) {
3498 port = &ports[pid];
3499 port->dev_conf.fdir_conf = fdir_conf;
3500
3501 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3502 if (ret != 0)
3503 return;
3504
3505 if (nb_rxq > 1) {
3506 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3507 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3508 rss_hf & port->dev_info.flow_type_rss_offloads;
3509 } else {
3510 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3511 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3512 }
3513
3514 if (port->dcb_flag == 0) {
3515 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3516 port->dev_conf.rxmode.mq_mode =
3517 (enum rte_eth_rx_mq_mode)
3518 (rx_mq_mode & ETH_MQ_RX_RSS);
3519 else
3520 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3521 }
3522
3523 rxtx_port_config(port);
3524
3525 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3526 if (ret != 0)
3527 return;
3528
3529 map_port_queue_stats_mapping_registers(pid, port);
3530 #if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3531 rte_pmd_ixgbe_bypass_init(pid);
3532 #endif
3533
3534 if (lsc_interrupt &&
3535 (rte_eth_devices[pid].data->dev_flags &
3536 RTE_ETH_DEV_INTR_LSC))
3537 port->dev_conf.intr_conf.lsc = 1;
3538 if (rmv_interrupt &&
3539 (rte_eth_devices[pid].data->dev_flags &
3540 RTE_ETH_DEV_INTR_RMV))
3541 port->dev_conf.intr_conf.rmv = 1;
3542 }
3543 }
3544
set_port_slave_flag(portid_t slave_pid)3545 void set_port_slave_flag(portid_t slave_pid)
3546 {
3547 struct rte_port *port;
3548
3549 port = &ports[slave_pid];
3550 port->slave_flag = 1;
3551 }
3552
clear_port_slave_flag(portid_t slave_pid)3553 void clear_port_slave_flag(portid_t slave_pid)
3554 {
3555 struct rte_port *port;
3556
3557 port = &ports[slave_pid];
3558 port->slave_flag = 0;
3559 }
3560
port_is_bonding_slave(portid_t slave_pid)3561 uint8_t port_is_bonding_slave(portid_t slave_pid)
3562 {
3563 struct rte_port *port;
3564
3565 port = &ports[slave_pid];
3566 if ((rte_eth_devices[slave_pid].data->dev_flags &
3567 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3568 return 1;
3569 return 0;
3570 }
3571
3572 const uint16_t vlan_tags[] = {
3573 0, 1, 2, 3, 4, 5, 6, 7,
3574 8, 9, 10, 11, 12, 13, 14, 15,
3575 16, 17, 18, 19, 20, 21, 22, 23,
3576 24, 25, 26, 27, 28, 29, 30, 31
3577 };
3578
3579 static int
get_eth_dcb_conf(portid_t pid,struct rte_eth_conf * eth_conf,enum dcb_mode_enable dcb_mode,enum rte_eth_nb_tcs num_tcs,uint8_t pfc_en)3580 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3581 enum dcb_mode_enable dcb_mode,
3582 enum rte_eth_nb_tcs num_tcs,
3583 uint8_t pfc_en)
3584 {
3585 uint8_t i;
3586 int32_t rc;
3587 struct rte_eth_rss_conf rss_conf;
3588
3589 /*
3590 * Builds up the correct configuration for dcb+vt based on the vlan tags array
3591 * given above, and the number of traffic classes available for use.
3592 */
3593 if (dcb_mode == DCB_VT_ENABLED) {
3594 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3595 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3596 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3597 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3598
3599 /* VMDQ+DCB RX and TX configurations */
3600 vmdq_rx_conf->enable_default_pool = 0;
3601 vmdq_rx_conf->default_pool = 0;
3602 vmdq_rx_conf->nb_queue_pools =
3603 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3604 vmdq_tx_conf->nb_queue_pools =
3605 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3606
3607 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3608 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3609 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3610 vmdq_rx_conf->pool_map[i].pools =
3611 1 << (i % vmdq_rx_conf->nb_queue_pools);
3612 }
3613 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3614 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3615 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3616 }
3617
3618 /* set DCB mode of RX and TX of multiple queues */
3619 eth_conf->rxmode.mq_mode =
3620 (enum rte_eth_rx_mq_mode)
3621 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3622 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3623 } else {
3624 struct rte_eth_dcb_rx_conf *rx_conf =
3625 ð_conf->rx_adv_conf.dcb_rx_conf;
3626 struct rte_eth_dcb_tx_conf *tx_conf =
3627 ð_conf->tx_adv_conf.dcb_tx_conf;
3628
3629 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3630
3631 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3632 if (rc != 0)
3633 return rc;
3634
3635 rx_conf->nb_tcs = num_tcs;
3636 tx_conf->nb_tcs = num_tcs;
3637
3638 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3639 rx_conf->dcb_tc[i] = i % num_tcs;
3640 tx_conf->dcb_tc[i] = i % num_tcs;
3641 }
3642
3643 eth_conf->rxmode.mq_mode =
3644 (enum rte_eth_rx_mq_mode)
3645 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3646 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3647 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3648 }
3649
3650 if (pfc_en)
3651 eth_conf->dcb_capability_en =
3652 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3653 else
3654 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3655
3656 return 0;
3657 }
3658
3659 int
init_port_dcb_config(portid_t pid,enum dcb_mode_enable dcb_mode,enum rte_eth_nb_tcs num_tcs,uint8_t pfc_en)3660 init_port_dcb_config(portid_t pid,
3661 enum dcb_mode_enable dcb_mode,
3662 enum rte_eth_nb_tcs num_tcs,
3663 uint8_t pfc_en)
3664 {
3665 struct rte_eth_conf port_conf;
3666 struct rte_port *rte_port;
3667 int retval;
3668 uint16_t i;
3669
3670 rte_port = &ports[pid];
3671
3672 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3673 /* Enter DCB configuration status */
3674 dcb_config = 1;
3675
3676 port_conf.rxmode = rte_port->dev_conf.rxmode;
3677 port_conf.txmode = rte_port->dev_conf.txmode;
3678
3679 /*set configuration of DCB in vt mode and DCB in non-vt mode*/
3680 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3681 if (retval < 0)
3682 return retval;
3683 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3684
3685 /* re-configure the device . */
3686 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3687 if (retval < 0)
3688 return retval;
3689
3690 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3691 if (retval != 0)
3692 return retval;
3693
3694 /* If dev_info.vmdq_pool_base is greater than 0,
3695 * the queue id of vmdq pools is started after pf queues.
3696 */
3697 if (dcb_mode == DCB_VT_ENABLED &&
3698 rte_port->dev_info.vmdq_pool_base > 0) {
3699 printf("VMDQ_DCB multi-queue mode is nonsensical"
3700 " for port %d.", pid);
3701 return -1;
3702 }
3703
3704 /* Assume the ports in testpmd have the same dcb capability
3705 * and has the same number of rxq and txq in dcb mode
3706 */
3707 if (dcb_mode == DCB_VT_ENABLED) {
3708 if (rte_port->dev_info.max_vfs > 0) {
3709 nb_rxq = rte_port->dev_info.nb_rx_queues;
3710 nb_txq = rte_port->dev_info.nb_tx_queues;
3711 } else {
3712 nb_rxq = rte_port->dev_info.max_rx_queues;
3713 nb_txq = rte_port->dev_info.max_tx_queues;
3714 }
3715 } else {
3716 /*if vt is disabled, use all pf queues */
3717 if (rte_port->dev_info.vmdq_pool_base == 0) {
3718 nb_rxq = rte_port->dev_info.max_rx_queues;
3719 nb_txq = rte_port->dev_info.max_tx_queues;
3720 } else {
3721 nb_rxq = (queueid_t)num_tcs;
3722 nb_txq = (queueid_t)num_tcs;
3723
3724 }
3725 }
3726 rx_free_thresh = 64;
3727
3728 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3729
3730 rxtx_port_config(rte_port);
3731 /* VLAN filter */
3732 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3733 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3734 rx_vft_set(pid, vlan_tags[i], 1);
3735
3736 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3737 if (retval != 0)
3738 return retval;
3739
3740 map_port_queue_stats_mapping_registers(pid, rte_port);
3741
3742 rte_port->dcb_flag = 1;
3743
3744 return 0;
3745 }
3746
3747 static void
init_port(void)3748 init_port(void)
3749 {
3750 int i;
3751
3752 /* Configuration of Ethernet ports. */
3753 ports = rte_zmalloc("testpmd: ports",
3754 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3755 RTE_CACHE_LINE_SIZE);
3756 if (ports == NULL) {
3757 rte_exit(EXIT_FAILURE,
3758 "rte_zmalloc(%d struct rte_port) failed\n",
3759 RTE_MAX_ETHPORTS);
3760 }
3761 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3762 LIST_INIT(&ports[i].flow_tunnel_list);
3763 /* Initialize ports NUMA structures */
3764 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3765 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3766 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3767 }
3768
3769 static void
force_quit(void)3770 force_quit(void)
3771 {
3772 pmd_test_exit();
3773 prompt_exit();
3774 }
3775
3776 static void
print_stats(void)3777 print_stats(void)
3778 {
3779 uint8_t i;
3780 const char clr[] = { 27, '[', '2', 'J', '\0' };
3781 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3782
3783 /* Clear screen and move to top left */
3784 printf("%s%s", clr, top_left);
3785
3786 printf("\nPort statistics ====================================");
3787 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3788 nic_stats_display(fwd_ports_ids[i]);
3789
3790 fflush(stdout);
3791 }
3792
3793 static void
signal_handler(int signum)3794 signal_handler(int signum)
3795 {
3796 if (signum == SIGINT || signum == SIGTERM) {
3797 printf("\nSignal %d received, preparing to exit...\n",
3798 signum);
3799 #ifdef RTE_LIB_PDUMP
3800 /* uninitialize packet capture framework */
3801 rte_pdump_uninit();
3802 #endif
3803 #ifdef RTE_LIB_LATENCYSTATS
3804 if (latencystats_enabled != 0)
3805 rte_latencystats_uninit();
3806 #endif
3807 force_quit();
3808 /* Set flag to indicate the force termination. */
3809 f_quit = 1;
3810 /* exit with the expected status */
3811 signal(signum, SIG_DFL);
3812 kill(getpid(), signum);
3813 }
3814 }
3815
3816 int
main(int argc,char ** argv)3817 main(int argc, char** argv)
3818 {
3819 int diag;
3820 portid_t port_id;
3821 uint16_t count;
3822 int ret;
3823
3824 signal(SIGINT, signal_handler);
3825 signal(SIGTERM, signal_handler);
3826
3827 testpmd_logtype = rte_log_register("testpmd");
3828 if (testpmd_logtype < 0)
3829 rte_exit(EXIT_FAILURE, "Cannot register log type");
3830 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3831
3832 diag = rte_eal_init(argc, argv);
3833 if (diag < 0)
3834 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3835 rte_strerror(rte_errno));
3836
3837 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3838 rte_exit(EXIT_FAILURE,
3839 "Secondary process type not supported.\n");
3840
3841 ret = register_eth_event_callback();
3842 if (ret != 0)
3843 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3844
3845 #ifdef RTE_LIB_PDUMP
3846 /* initialize packet capture framework */
3847 rte_pdump_init();
3848 #endif
3849
3850 count = 0;
3851 RTE_ETH_FOREACH_DEV(port_id) {
3852 ports_ids[count] = port_id;
3853 count++;
3854 }
3855 nb_ports = (portid_t) count;
3856 if (nb_ports == 0)
3857 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3858
3859 /* allocate port structures, and init them */
3860 init_port();
3861
3862 set_def_fwd_config();
3863 if (nb_lcores == 0)
3864 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3865 "Check the core mask argument\n");
3866
3867 /* Bitrate/latency stats disabled by default */
3868 #ifdef RTE_LIB_BITRATESTATS
3869 bitrate_enabled = 0;
3870 #endif
3871 #ifdef RTE_LIB_LATENCYSTATS
3872 latencystats_enabled = 0;
3873 #endif
3874
3875 /* on FreeBSD, mlockall() is disabled by default */
3876 #ifdef RTE_EXEC_ENV_FREEBSD
3877 do_mlockall = 0;
3878 #else
3879 do_mlockall = 1;
3880 #endif
3881
3882 argc -= diag;
3883 argv += diag;
3884 if (argc > 1)
3885 launch_args_parse(argc, argv);
3886
3887 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3888 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3889 strerror(errno));
3890 }
3891
3892 if (tx_first && interactive)
3893 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3894 "interactive mode.\n");
3895
3896 if (tx_first && lsc_interrupt) {
3897 printf("Warning: lsc_interrupt needs to be off when "
3898 " using tx_first. Disabling.\n");
3899 lsc_interrupt = 0;
3900 }
3901
3902 if (!nb_rxq && !nb_txq)
3903 printf("Warning: Either rx or tx queues should be non-zero\n");
3904
3905 if (nb_rxq > 1 && nb_rxq > nb_txq)
3906 printf("Warning: nb_rxq=%d enables RSS configuration, "
3907 "but nb_txq=%d will prevent to fully test it.\n",
3908 nb_rxq, nb_txq);
3909
3910 init_config();
3911
3912 if (hot_plug) {
3913 ret = rte_dev_hotplug_handle_enable();
3914 if (ret) {
3915 RTE_LOG(ERR, EAL,
3916 "fail to enable hotplug handling.");
3917 return -1;
3918 }
3919
3920 ret = rte_dev_event_monitor_start();
3921 if (ret) {
3922 RTE_LOG(ERR, EAL,
3923 "fail to start device event monitoring.");
3924 return -1;
3925 }
3926
3927 ret = rte_dev_event_callback_register(NULL,
3928 dev_event_callback, NULL);
3929 if (ret) {
3930 RTE_LOG(ERR, EAL,
3931 "fail to register device event callback\n");
3932 return -1;
3933 }
3934 }
3935
3936 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3937 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3938
3939 /* set all ports to promiscuous mode by default */
3940 RTE_ETH_FOREACH_DEV(port_id) {
3941 ret = rte_eth_promiscuous_enable(port_id);
3942 if (ret != 0)
3943 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3944 port_id, rte_strerror(-ret));
3945 }
3946
3947 /* Init metrics library */
3948 rte_metrics_init(rte_socket_id());
3949
3950 #ifdef RTE_LIB_LATENCYSTATS
3951 if (latencystats_enabled != 0) {
3952 int ret = rte_latencystats_init(1, NULL);
3953 if (ret)
3954 printf("Warning: latencystats init()"
3955 " returned error %d\n", ret);
3956 printf("Latencystats running on lcore %d\n",
3957 latencystats_lcore_id);
3958 }
3959 #endif
3960
3961 /* Setup bitrate stats */
3962 #ifdef RTE_LIB_BITRATESTATS
3963 if (bitrate_enabled != 0) {
3964 bitrate_data = rte_stats_bitrate_create();
3965 if (bitrate_data == NULL)
3966 rte_exit(EXIT_FAILURE,
3967 "Could not allocate bitrate data.\n");
3968 rte_stats_bitrate_reg(bitrate_data);
3969 }
3970 #endif
3971
3972 #ifdef RTE_LIB_CMDLINE
3973 if (strlen(cmdline_filename) != 0)
3974 cmdline_read_from_file(cmdline_filename);
3975
3976 if (interactive == 1) {
3977 if (auto_start) {
3978 printf("Start automatic packet forwarding\n");
3979 start_packet_forwarding(0);
3980 }
3981 prompt();
3982 pmd_test_exit();
3983 } else
3984 #endif
3985 {
3986 char c;
3987 int rc;
3988
3989 f_quit = 0;
3990
3991 printf("No commandline core given, start packet forwarding\n");
3992 start_packet_forwarding(tx_first);
3993 if (stats_period != 0) {
3994 uint64_t prev_time = 0, cur_time, diff_time = 0;
3995 uint64_t timer_period;
3996
3997 /* Convert to number of cycles */
3998 timer_period = stats_period * rte_get_timer_hz();
3999
4000 while (f_quit == 0) {
4001 cur_time = rte_get_timer_cycles();
4002 diff_time += cur_time - prev_time;
4003
4004 if (diff_time >= timer_period) {
4005 print_stats();
4006 /* Reset the timer */
4007 diff_time = 0;
4008 }
4009 /* Sleep to avoid unnecessary checks */
4010 prev_time = cur_time;
4011 sleep(1);
4012 }
4013 }
4014
4015 printf("Press enter to exit\n");
4016 rc = read(0, &c, 1);
4017 pmd_test_exit();
4018 if (rc < 0)
4019 return 1;
4020 }
4021
4022 ret = rte_eal_cleanup();
4023 if (ret != 0)
4024 rte_exit(EXIT_FAILURE,
4025 "EAL cleanup failed: %s\n", strerror(-ret));
4026
4027 return EXIT_SUCCESS;
4028 }
4029