1 /* 2 * iperf, Copyright (c) 2014-2020, The Regents of the University of 3 * California, through Lawrence Berkeley National Laboratory (subject 4 * to receipt of any required approvals from the U.S. Dept. of 5 * Energy). All rights reserved. 6 * 7 * If you have questions about your rights to use or distribute this 8 * software, please contact Berkeley Lab's Technology Transfer 9 * Department at [email protected]. 10 * 11 * NOTICE. This software is owned by the U.S. Department of Energy. 12 * As such, the U.S. Government has been granted for itself and others 13 * acting on its behalf a paid-up, nonexclusive, irrevocable, 14 * worldwide license in the Software to reproduce, prepare derivative 15 * works, and perform publicly and display publicly. Beginning five 16 * (5) years after the date permission to assert copyright is obtained 17 * from the U.S. Department of Energy, and subject to any subsequent 18 * five (5) year renewals, the U.S. Government is granted for itself 19 * and others acting on its behalf a paid-up, nonexclusive, 20 * irrevocable, worldwide license in the Software to reproduce, 21 * prepare derivative works, distribute copies to the public, perform 22 * publicly and display publicly, and to permit others to do so. 23 * 24 * This code is distributed under a BSD style license, see the LICENSE 25 * file for complete information. 26 */ 27 #ifndef __IPERF_H 28 #define __IPERF_H 29 30 #include "iperf_config.h" 31 32 #include <sys/time.h> 33 #include <sys/types.h> 34 #ifdef HAVE_STDINT_H 35 #include <stdint.h> 36 #endif 37 #include <sys/select.h> 38 #include <sys/socket.h> 39 #ifndef _GNU_SOURCE 40 # define _GNU_SOURCE 41 #endif 42 #ifdef HAVE_LINUX_TCP_H 43 #include <linux/tcp.h> 44 #else 45 #include <netinet/tcp.h> 46 #endif 47 #include <net/if.h> // for IFNAMSIZ 48 49 #if defined(HAVE_CPUSET_SETAFFINITY) 50 #include <sys/param.h> 51 #include <sys/cpuset.h> 52 #endif /* HAVE_CPUSET_SETAFFINITY */ 53 54 #if defined(HAVE_INTTYPES_H) 55 # include <inttypes.h> 56 #else 57 # ifndef PRIu64 58 # if sizeof(long) == 8 59 # define PRIu64 "lu" 60 # else 61 # define PRIu64 "llu" 62 # endif 63 # endif 64 #endif 65 66 #include "timer.h" 67 #include "queue.h" 68 #include "cjson.h" 69 #include "iperf_time.h" 70 71 #if defined(HAVE_SSL) 72 #include <openssl/bio.h> 73 #include <openssl/evp.h> 74 #endif // HAVE_SSL 75 76 #if !defined(__IPERF_API_H) 77 typedef uint64_t iperf_size_t; 78 #endif // __IPERF_API_H 79 80 struct iperf_interval_results 81 { 82 iperf_size_t bytes_transferred; /* bytes transferred in this interval */ 83 struct iperf_time interval_start_time; 84 struct iperf_time interval_end_time; 85 float interval_duration; 86 87 /* for UDP */ 88 int interval_packet_count; 89 int interval_outoforder_packets; 90 int interval_cnt_error; 91 int packet_count; 92 double jitter; 93 int outoforder_packets; 94 int cnt_error; 95 96 int omitted; 97 #if (defined(linux) || defined(__FreeBSD__) || defined(__NetBSD__)) && \ 98 defined(TCP_INFO) 99 struct tcp_info tcpInfo; /* getsockopt(TCP_INFO) for Linux, {Free,Net}BSD */ 100 #else 101 /* Just placeholders, never accessed. */ 102 char *tcpInfo; 103 #endif 104 int interval_retrans; 105 int interval_sacks; 106 int snd_cwnd; 107 int snd_wnd; 108 TAILQ_ENTRY(iperf_interval_results) irlistentries; 109 void *custom_data; 110 int rtt; 111 int rttvar; 112 int pmtu; 113 }; 114 115 struct iperf_stream_result 116 { 117 iperf_size_t bytes_received; 118 iperf_size_t bytes_sent; 119 iperf_size_t bytes_received_this_interval; 120 iperf_size_t bytes_sent_this_interval; 121 iperf_size_t bytes_sent_omit; 122 int stream_prev_total_retrans; 123 int stream_retrans; 124 int stream_prev_total_sacks; 125 int stream_sacks; 126 int stream_max_rtt; 127 int stream_min_rtt; 128 int stream_sum_rtt; 129 int stream_count_rtt; 130 int stream_max_snd_cwnd; 131 int stream_max_snd_wnd; 132 struct iperf_time start_time; 133 struct iperf_time end_time; 134 struct iperf_time start_time_fixed; 135 double sender_time; 136 double receiver_time; 137 TAILQ_HEAD(irlisthead, iperf_interval_results) interval_results; 138 void *data; 139 }; 140 141 #define COOKIE_SIZE 37 /* size of an ascii uuid */ 142 struct iperf_settings 143 { 144 int domain; /* AF_INET or AF_INET6 */ 145 int socket_bufsize; /* window size for TCP */ 146 int blksize; /* size of read/writes (-l) */ 147 iperf_size_t rate; /* target data rate for application pacing*/ 148 iperf_size_t bitrate_limit; /* server's maximum allowed total data rate for all streams*/ 149 double bitrate_limit_interval; /* interval for averaging total data rate */ 150 int bitrate_limit_stats_per_interval; /* calculated number of stats periods for averaging total data rate */ 151 uint64_t fqrate; /* target data rate for FQ pacing*/ 152 int pacing_timer; /* pacing timer in microseconds */ 153 int burst; /* packets per burst */ 154 int mss; /* for TCP MSS */ 155 int ttl; /* IP TTL option */ 156 int tos; /* type of service bit */ 157 int flowlabel; /* IPv6 flow label */ 158 iperf_size_t bytes; /* number of bytes to send */ 159 iperf_size_t blocks; /* number of blocks (packets) to send */ 160 char unit_format; /* -f */ 161 int num_ostreams; /* SCTP initmsg settings */ 162 int dont_fragment; /* Whether to set IP flag Do-Not_Fragment */ 163 #if defined(HAVE_SSL) 164 char *authtoken; /* Authentication token */ 165 char *client_username; 166 char *client_password; 167 EVP_PKEY *client_rsa_pubkey; 168 #endif // HAVE_SSL 169 int connect_timeout; /* socket connection timeout, in ms */ 170 int idle_timeout; /* server idle time timeout */ 171 unsigned int snd_timeout; /* Timeout for sending tcp messages in active mode, in us */ 172 struct iperf_time rcv_timeout; /* Timeout for receiving messages in active mode, in us */ 173 }; 174 175 struct iperf_test; 176 177 struct iperf_stream 178 { 179 struct iperf_test* test; 180 181 /* configurable members */ 182 int local_port; 183 int remote_port; 184 int socket; 185 int id; 186 int sender; 187 /* XXX: is settings just a pointer to the same struct in iperf_test? if not, 188 should it be? */ 189 struct iperf_settings *settings; /* pointer to structure settings */ 190 191 /* non configurable members */ 192 struct iperf_stream_result *result; /* structure pointer to result */ 193 Timer *send_timer; 194 int green_light; 195 int buffer_fd; /* data to send, file descriptor */ 196 char *buffer; /* data to send, mmapped */ 197 int pending_size; /* pending data to send */ 198 int diskfile_fd; /* file to send, file descriptor */ 199 int diskfile_left; /* remaining file data on disk */ 200 201 /* 202 * for udp measurements - This can be a structure outside stream, and 203 * stream can have a pointer to this 204 */ 205 int packet_count; 206 int peer_packet_count; 207 int omitted_packet_count; 208 double jitter; 209 double prev_transit; 210 int outoforder_packets; 211 int omitted_outoforder_packets; 212 int cnt_error; 213 int omitted_cnt_error; 214 uint64_t target; 215 216 struct sockaddr_storage local_addr; 217 struct sockaddr_storage remote_addr; 218 219 int (*rcv) (struct iperf_stream * stream); 220 int (*snd) (struct iperf_stream * stream); 221 222 /* chained send/receive routines for -F mode */ 223 int (*rcv2) (struct iperf_stream * stream); 224 int (*snd2) (struct iperf_stream * stream); 225 226 // struct iperf_stream *next; 227 SLIST_ENTRY(iperf_stream) streams; 228 229 void *data; 230 }; 231 232 struct protocol { 233 int id; 234 char *name; 235 int (*accept)(struct iperf_test *); 236 int (*listen)(struct iperf_test *); 237 int (*connect)(struct iperf_test *); 238 int (*send)(struct iperf_stream *); 239 int (*recv)(struct iperf_stream *); 240 int (*init)(struct iperf_test *); 241 SLIST_ENTRY(protocol) protocols; 242 }; 243 244 struct iperf_textline { 245 char *line; 246 TAILQ_ENTRY(iperf_textline) textlineentries; 247 }; 248 249 struct xbind_entry { 250 char *name; 251 struct addrinfo *ai; 252 TAILQ_ENTRY(xbind_entry) link; 253 }; 254 255 enum iperf_mode { 256 SENDER = 1, 257 RECEIVER = 0, 258 BIDIRECTIONAL = -1 259 }; 260 261 enum debug_level { 262 DEBUG_LEVEL_ERROR = 1, 263 DEBUG_LEVEL_WARN = 2, 264 DEBUG_LEVEL_INFO = 3, 265 DEBUG_LEVEL_DEBUG = 4, 266 DEBUG_LEVEL_MAX = 4 267 }; 268 269 270 struct iperf_test 271 { 272 char role; /* 'c' lient or 's' erver */ 273 enum iperf_mode mode; 274 int sender_has_retransmits; 275 int other_side_has_retransmits; /* used if mode == BIDIRECTIONAL */ 276 struct protocol *protocol; 277 signed char state; 278 char *server_hostname; /* -c option */ 279 char *tmp_template; 280 char *bind_address; /* first -B option */ 281 char *bind_dev; /* bind to network device */ 282 TAILQ_HEAD(xbind_addrhead, xbind_entry) xbind_addrs; /* all -X opts */ 283 int bind_port; /* --cport option */ 284 int server_port; 285 int omit; /* duration of omit period (-O flag) */ 286 int duration; /* total duration of test (-t flag) */ 287 char *diskfile_name; /* -F option */ 288 int affinity, server_affinity; /* -A option */ 289 #if defined(HAVE_CPUSET_SETAFFINITY) 290 cpuset_t cpumask; 291 #endif /* HAVE_CPUSET_SETAFFINITY */ 292 char *title; /* -T option */ 293 char *extra_data; /* --extra-data */ 294 char *congestion; /* -C option */ 295 char *congestion_used; /* what was actually used */ 296 char *remote_congestion_used; /* what the other side used */ 297 char *pidfile; /* -P option */ 298 299 char *logfile; /* --logfile option */ 300 FILE *outfile; 301 302 int ctrl_sck; 303 int listener; 304 int prot_listener; 305 306 int ctrl_sck_mss; /* MSS for the control channel */ 307 308 #if defined(HAVE_SSL) 309 char *server_authorized_users; 310 EVP_PKEY *server_rsa_private_key; 311 int server_skew_threshold; 312 #endif // HAVE_SSL 313 314 /* boolean variables for Options */ 315 int daemon; /* -D option */ 316 int one_off; /* -1 option */ 317 int no_delay; /* -N option */ 318 int reverse; /* -R option */ 319 int bidirectional; /* --bidirectional */ 320 int verbose; /* -V option - verbose mode */ 321 int json_output; /* -J option - JSON output */ 322 int zerocopy; /* -Z option - use sendfile */ 323 int debug; /* -d option - enable debug */ 324 enum debug_level debug_level; /* -d option option - level of debug messages to show */ 325 int get_server_output; /* --get-server-output */ 326 int udp_counters_64bit; /* --use-64-bit-udp-counters */ 327 int forceflush; /* --forceflush - flushing output at every interval */ 328 int multisend; 329 int repeating_payload; /* --repeating-payload */ 330 int timestamps; /* --timestamps */ 331 char *timestamp_format; 332 333 char *json_output_string; /* rendered JSON output if json_output is set */ 334 /* Select related parameters */ 335 int max_fd; 336 fd_set read_set; /* set of read sockets */ 337 fd_set write_set; /* set of write sockets */ 338 339 /* Interval related members */ 340 int omitting; 341 double stats_interval; 342 double reporter_interval; 343 void (*stats_callback) (struct iperf_test *); 344 void (*reporter_callback) (struct iperf_test *); 345 Timer *omit_timer; 346 Timer *timer; 347 int done; 348 Timer *stats_timer; 349 Timer *reporter_timer; 350 351 double cpu_util[3]; /* cpu utilization of the test - total, user, system */ 352 double remote_cpu_util[3]; /* cpu utilization for the remote host/client - total, user, system */ 353 354 int num_streams; /* total streams in the test (-P) */ 355 356 iperf_size_t bytes_sent; 357 iperf_size_t blocks_sent; 358 359 iperf_size_t bytes_received; 360 iperf_size_t blocks_received; 361 362 iperf_size_t bitrate_limit_stats_count; /* Number of stats periods accumulated for server's total bitrate average */ 363 iperf_size_t *bitrate_limit_intervals_traffic_bytes; /* Pointer to a cyclic array that includes the last interval's bytes transferred */ 364 iperf_size_t bitrate_limit_last_interval_index; /* Index of the last interval traffic inserted into the cyclic array */ 365 int bitrate_limit_exceeded; /* Set by callback routine when average data rate exceeded the server's bitrate limit */ 366 367 int server_last_run_rc; /* Save last server run rc for next test */ 368 uint server_forced_idle_restarts_count; /* count number of forced server restarts to make sure it is not stack */ 369 uint server_forced_no_msg_restarts_count; /* count number of forced server restarts to make sure it is not stack */ 370 uint server_test_number; /* count number of tests performed by a server */ 371 372 char cookie[COOKIE_SIZE]; 373 // struct iperf_stream *streams; /* pointer to list of struct stream */ 374 SLIST_HEAD(slisthead, iperf_stream) streams; 375 struct iperf_settings *settings; 376 377 SLIST_HEAD(plisthead, protocol) protocols; 378 379 /* callback functions */ 380 void (*on_new_stream)(struct iperf_stream *); 381 void (*on_test_start)(struct iperf_test *); 382 void (*on_connect)(struct iperf_test *); 383 void (*on_test_finish)(struct iperf_test *); 384 385 /* cJSON handles for use when in -J mode */\ 386 cJSON *json_top; 387 cJSON *json_start; 388 cJSON *json_connected; 389 cJSON *json_intervals; 390 cJSON *json_end; 391 392 /* Server output (use on client side only) */ 393 char *server_output_text; 394 cJSON *json_server_output; 395 396 /* Server output (use on server side only) */ 397 TAILQ_HEAD(iperf_textlisthead, iperf_textline) server_output_list; 398 399 }; 400 401 /* default settings */ 402 #define PORT 5201 /* default port to listen on (don't use the same port as iperf2) */ 403 #define uS_TO_NS 1000 404 #define mS_TO_US 1000 405 #define SEC_TO_mS 1000 406 #define SEC_TO_US 1000000LL 407 #define UDP_RATE (1024 * 1024) /* 1 Mbps */ 408 #define OMIT 0 /* seconds */ 409 #define DURATION 10 /* seconds */ 410 411 #define SEC_TO_NS 1000000000LL /* too big for enum/const on some platforms */ 412 #define MAX_RESULT_STRING 4096 413 414 #define UDP_BUFFER_EXTRA 1024 415 416 /* constants for command line arg sanity checks */ 417 #define MB (1024 * 1024) 418 #define MAX_TCP_BUFFER (512 * MB) 419 #define MAX_BLOCKSIZE MB 420 /* Minimum size UDP send is the size of two 32-bit ints followed by a 64-bit int */ 421 #define MIN_UDP_BLOCKSIZE (4 + 4 + 8) 422 /* Maximum size UDP send is (64K - 1) - IP and UDP header sizes */ 423 #define MAX_UDP_BLOCKSIZE (65535 - 8 - 20) 424 #define MIN_INTERVAL 0.1 425 #define MAX_INTERVAL 60.0 426 #define MAX_TIME 86400 427 #define MAX_BURST 1000 428 #define MAX_MSS (9 * 1024) 429 #define MAX_STREAMS 128 430 431 #define TIMESTAMP_FORMAT "%c " 432 433 extern int gerror; /* error value from getaddrinfo(3), for use in internal error handling */ 434 435 /* UDP "connect" message and reply (textual value for Wireshark, etc. readability - legacy was numeric) */ 436 #define UDP_CONNECT_MSG 0x36373839 // "6789" - legacy value was 123456789 437 #define UDP_CONNECT_REPLY 0x39383736 // "9876" - legacy value was 987654321 438 #define LEGACY_UDP_CONNECT_REPLY 987654321 // Old servers may still reply with the legacy value 439 440 /* In Reverse mode, maximum number of packets to wait for "accept" response - to handle out of order packets */ 441 #define MAX_REVERSE_OUT_OF_ORDER_PACKETS 2 442 443 #endif /* !__IPERF_H */ 444