1 /* 2 * iperf, Copyright (c) 2014-2020, The Regents of the University of 3 * California, through Lawrence Berkeley National Laboratory (subject 4 * to receipt of any required approvals from the U.S. Dept. of 5 * Energy). All rights reserved. 6 * 7 * If you have questions about your rights to use or distribute this 8 * software, please contact Berkeley Lab's Technology Transfer 9 * Department at [email protected]. 10 * 11 * NOTICE. This software is owned by the U.S. Department of Energy. 12 * As such, the U.S. Government has been granted for itself and others 13 * acting on its behalf a paid-up, nonexclusive, irrevocable, 14 * worldwide license in the Software to reproduce, prepare derivative 15 * works, and perform publicly and display publicly. Beginning five 16 * (5) years after the date permission to assert copyright is obtained 17 * from the U.S. Department of Energy, and subject to any subsequent 18 * five (5) year renewals, the U.S. Government is granted for itself 19 * and others acting on its behalf a paid-up, nonexclusive, 20 * irrevocable, worldwide license in the Software to reproduce, 21 * prepare derivative works, distribute copies to the public, perform 22 * publicly and display publicly, and to permit others to do so. 23 * 24 * This code is distributed under a BSD style license, see the LICENSE 25 * file for complete information. 26 */ 27 #include <errno.h> 28 #include <setjmp.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <unistd.h> 33 #include <signal.h> 34 #include <sys/types.h> 35 #include <netinet/in.h> 36 #include <sys/select.h> 37 #include <sys/uio.h> 38 #include <arpa/inet.h> 39 40 #include "iperf.h" 41 #include "iperf_api.h" 42 #include "iperf_util.h" 43 #include "iperf_locale.h" 44 #include "iperf_time.h" 45 #include "net.h" 46 #include "timer.h" 47 48 #if defined(HAVE_TCP_CONGESTION) 49 #if !defined(TCP_CA_NAME_MAX) 50 #define TCP_CA_NAME_MAX 16 51 #endif /* TCP_CA_NAME_MAX */ 52 #endif /* HAVE_TCP_CONGESTION */ 53 54 int 55 iperf_create_streams(struct iperf_test *test, int sender) 56 { 57 int i, s; 58 #if defined(HAVE_TCP_CONGESTION) 59 int saved_errno; 60 #endif /* HAVE_TCP_CONGESTION */ 61 struct iperf_stream *sp; 62 63 int orig_bind_port = test->bind_port; 64 for (i = 0; i < test->num_streams; ++i) { 65 66 test->bind_port = orig_bind_port; 67 if (orig_bind_port) 68 test->bind_port += i; 69 if ((s = test->protocol->connect(test)) < 0) 70 return -1; 71 72 #if defined(HAVE_TCP_CONGESTION) 73 if (test->protocol->id == Ptcp) { 74 if (test->congestion) { 75 if (setsockopt(s, IPPROTO_TCP, TCP_CONGESTION, test->congestion, strlen(test->congestion)) < 0) { 76 saved_errno = errno; 77 close(s); 78 errno = saved_errno; 79 i_errno = IESETCONGESTION; 80 return -1; 81 } 82 } 83 { 84 socklen_t len = TCP_CA_NAME_MAX; 85 char ca[TCP_CA_NAME_MAX + 1]; 86 if (getsockopt(s, IPPROTO_TCP, TCP_CONGESTION, ca, &len) < 0) { 87 saved_errno = errno; 88 close(s); 89 errno = saved_errno; 90 i_errno = IESETCONGESTION; 91 return -1; 92 } 93 test->congestion_used = strdup(ca); 94 if (test->debug) { 95 printf("Congestion algorithm is %s\n", test->congestion_used); 96 } 97 } 98 } 99 #endif /* HAVE_TCP_CONGESTION */ 100 101 if (sender) 102 FD_SET(s, &test->write_set); 103 else 104 FD_SET(s, &test->read_set); 105 if (s > test->max_fd) test->max_fd = s; 106 107 sp = iperf_new_stream(test, s, sender); 108 if (!sp) 109 return -1; 110 111 /* Perform the new stream callback */ 112 if (test->on_new_stream) 113 test->on_new_stream(sp); 114 } 115 116 return 0; 117 } 118 119 static void 120 test_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 121 { 122 struct iperf_test *test = client_data.p; 123 124 test->timer = NULL; 125 test->done = 1; 126 } 127 128 static void 129 client_stats_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 130 { 131 struct iperf_test *test = client_data.p; 132 133 if (test->done) 134 return; 135 if (test->stats_callback) 136 test->stats_callback(test); 137 } 138 139 static void 140 client_reporter_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 141 { 142 struct iperf_test *test = client_data.p; 143 144 if (test->done) 145 return; 146 if (test->reporter_callback) 147 test->reporter_callback(test); 148 } 149 150 static int 151 create_client_timers(struct iperf_test * test) 152 { 153 struct iperf_time now; 154 TimerClientData cd; 155 156 if (iperf_time_now(&now) < 0) { 157 i_errno = IEINITTEST; 158 return -1; 159 } 160 cd.p = test; 161 test->timer = test->stats_timer = test->reporter_timer = NULL; 162 if (test->duration != 0) { 163 test->done = 0; 164 test->timer = tmr_create(&now, test_timer_proc, cd, ( test->duration + test->omit ) * SEC_TO_US, 0); 165 if (test->timer == NULL) { 166 i_errno = IEINITTEST; 167 return -1; 168 } 169 } 170 if (test->stats_interval != 0) { 171 test->stats_timer = tmr_create(&now, client_stats_timer_proc, cd, test->stats_interval * SEC_TO_US, 1); 172 if (test->stats_timer == NULL) { 173 i_errno = IEINITTEST; 174 return -1; 175 } 176 } 177 if (test->reporter_interval != 0) { 178 test->reporter_timer = tmr_create(&now, client_reporter_timer_proc, cd, test->reporter_interval * SEC_TO_US, 1); 179 if (test->reporter_timer == NULL) { 180 i_errno = IEINITTEST; 181 return -1; 182 } 183 } 184 return 0; 185 } 186 187 static void 188 client_omit_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 189 { 190 struct iperf_test *test = client_data.p; 191 192 test->omit_timer = NULL; 193 test->omitting = 0; 194 iperf_reset_stats(test); 195 if (test->verbose && !test->json_output && test->reporter_interval == 0) 196 iperf_printf(test, "%s", report_omit_done); 197 198 /* Reset the timers. */ 199 if (test->stats_timer != NULL) 200 tmr_reset(nowP, test->stats_timer); 201 if (test->reporter_timer != NULL) 202 tmr_reset(nowP, test->reporter_timer); 203 } 204 205 static int 206 create_client_omit_timer(struct iperf_test * test) 207 { 208 struct iperf_time now; 209 TimerClientData cd; 210 211 if (test->omit == 0) { 212 test->omit_timer = NULL; 213 test->omitting = 0; 214 } else { 215 if (iperf_time_now(&now) < 0) { 216 i_errno = IEINITTEST; 217 return -1; 218 } 219 test->omitting = 1; 220 cd.p = test; 221 test->omit_timer = tmr_create(&now, client_omit_timer_proc, cd, test->omit * SEC_TO_US, 0); 222 if (test->omit_timer == NULL) { 223 i_errno = IEINITTEST; 224 return -1; 225 } 226 } 227 return 0; 228 } 229 230 int 231 iperf_handle_message_client(struct iperf_test *test) 232 { 233 int rval; 234 int32_t err; 235 236 /*!!! Why is this read() and not Nread()? */ 237 if ((rval = read(test->ctrl_sck, (char*) &test->state, sizeof(signed char))) <= 0) { 238 if (rval == 0) { 239 i_errno = IECTRLCLOSE; 240 return -1; 241 } else { 242 i_errno = IERECVMESSAGE; 243 return -1; 244 } 245 } 246 247 switch (test->state) { 248 case PARAM_EXCHANGE: 249 if (iperf_exchange_parameters(test) < 0) 250 return -1; 251 if (test->on_connect) 252 test->on_connect(test); 253 break; 254 case CREATE_STREAMS: 255 if (test->mode == BIDIRECTIONAL) 256 { 257 if (iperf_create_streams(test, 1) < 0) 258 return -1; 259 if (iperf_create_streams(test, 0) < 0) 260 return -1; 261 } 262 else if (iperf_create_streams(test, test->mode) < 0) 263 return -1; 264 break; 265 case TEST_START: 266 if (iperf_init_test(test) < 0) 267 return -1; 268 if (create_client_timers(test) < 0) 269 return -1; 270 if (create_client_omit_timer(test) < 0) 271 return -1; 272 if (test->mode) 273 if (iperf_create_send_timers(test) < 0) 274 return -1; 275 break; 276 case TEST_RUNNING: 277 break; 278 case EXCHANGE_RESULTS: 279 if (iperf_exchange_results(test) < 0) 280 return -1; 281 break; 282 case DISPLAY_RESULTS: 283 if (test->on_test_finish) 284 test->on_test_finish(test); 285 iperf_client_end(test); 286 break; 287 case IPERF_DONE: 288 break; 289 case SERVER_TERMINATE: 290 i_errno = IESERVERTERM; 291 292 /* 293 * Temporarily be in DISPLAY_RESULTS phase so we can get 294 * ending summary statistics. 295 */ 296 signed char oldstate = test->state; 297 cpu_util(test->cpu_util); 298 test->state = DISPLAY_RESULTS; 299 test->reporter_callback(test); 300 test->state = oldstate; 301 return -1; 302 case ACCESS_DENIED: 303 i_errno = IEACCESSDENIED; 304 return -1; 305 case SERVER_ERROR: 306 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 307 i_errno = IECTRLREAD; 308 return -1; 309 } 310 i_errno = ntohl(err); 311 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 312 i_errno = IECTRLREAD; 313 return -1; 314 } 315 errno = ntohl(err); 316 return -1; 317 default: 318 i_errno = IEMESSAGE; 319 return -1; 320 } 321 322 return 0; 323 } 324 325 326 327 /* iperf_connect -- client to server connection function */ 328 int 329 iperf_connect(struct iperf_test *test) 330 { 331 FD_ZERO(&test->read_set); 332 FD_ZERO(&test->write_set); 333 334 make_cookie(test->cookie); 335 336 /* Create and connect the control channel */ 337 if (test->ctrl_sck < 0) 338 // Create the control channel using an ephemeral port 339 test->ctrl_sck = netdial(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, 0, test->server_hostname, test->server_port, test->settings->connect_timeout); 340 if (test->ctrl_sck < 0) { 341 i_errno = IECONNECT; 342 return -1; 343 } 344 345 // set TCP_NODELAY for lower latency on control messages 346 int flag = 1; 347 if (setsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int))) { 348 i_errno = IESETNODELAY; 349 return -1; 350 } 351 352 if (Nwrite(test->ctrl_sck, test->cookie, COOKIE_SIZE, Ptcp) < 0) { 353 i_errno = IESENDCOOKIE; 354 return -1; 355 } 356 357 FD_SET(test->ctrl_sck, &test->read_set); 358 if (test->ctrl_sck > test->max_fd) test->max_fd = test->ctrl_sck; 359 360 int opt; 361 socklen_t len; 362 363 len = sizeof(opt); 364 if (getsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_MAXSEG, &opt, &len) < 0) { 365 test->ctrl_sck_mss = 0; 366 } 367 else { 368 if (opt > 0 && opt <= MAX_UDP_BLOCKSIZE) { 369 test->ctrl_sck_mss = opt; 370 } 371 else { 372 char str[128]; 373 snprintf(str, sizeof(str), 374 "Ignoring nonsense TCP MSS %d", opt); 375 warning(str); 376 377 test->ctrl_sck_mss = 0; 378 } 379 } 380 381 if (test->verbose) { 382 printf("Control connection MSS %d\n", test->ctrl_sck_mss); 383 } 384 385 /* 386 * If we're doing a UDP test and the block size wasn't explicitly 387 * set, then use the known MSS of the control connection to pick 388 * an appropriate default. If we weren't able to get the 389 * MSS for some reason, then default to something that should 390 * work on non-jumbo-frame Ethernet networks. The goal is to 391 * pick a reasonable default that is large but should get from 392 * sender to receiver without any IP fragmentation. 393 * 394 * We assume that the control connection is routed the same as the 395 * data packets (thus has the same PMTU). Also in the case of 396 * --reverse tests, we assume that the MTU is the same in both 397 * directions. Note that even if the algorithm guesses wrong, 398 * the user always has the option to override. 399 */ 400 if (test->protocol->id == Pudp) { 401 if (test->settings->blksize == 0) { 402 if (test->ctrl_sck_mss) { 403 test->settings->blksize = test->ctrl_sck_mss; 404 } 405 else { 406 test->settings->blksize = DEFAULT_UDP_BLKSIZE; 407 } 408 if (test->verbose) { 409 printf("Setting UDP block size to %d\n", test->settings->blksize); 410 } 411 } 412 413 /* 414 * Regardless of whether explicitly or implicitly set, if the 415 * block size is larger than the MSS, print a warning. 416 */ 417 if (test->ctrl_sck_mss > 0 && 418 test->settings->blksize > test->ctrl_sck_mss) { 419 char str[128]; 420 snprintf(str, sizeof(str), 421 "UDP block size %d exceeds TCP MSS %d, may result in fragmentation / drops", test->settings->blksize, test->ctrl_sck_mss); 422 warning(str); 423 } 424 } 425 426 return 0; 427 } 428 429 430 int 431 iperf_client_end(struct iperf_test *test) 432 { 433 struct iperf_stream *sp; 434 435 /* Close all stream sockets */ 436 SLIST_FOREACH(sp, &test->streams, streams) { 437 close(sp->socket); 438 } 439 440 /* show final summary */ 441 test->reporter_callback(test); 442 443 if (iperf_set_send_state(test, IPERF_DONE) != 0) 444 return -1; 445 446 /* Close control socket */ 447 if (test->ctrl_sck) 448 close(test->ctrl_sck); 449 450 return 0; 451 } 452 453 454 int 455 iperf_run_client(struct iperf_test * test) 456 { 457 int startup; 458 int result = 0; 459 fd_set read_set, write_set; 460 struct iperf_time now; 461 struct timeval* timeout = NULL; 462 struct iperf_stream *sp; 463 struct iperf_time last_receive_time; 464 struct iperf_time diff_time; 465 struct timeval used_timeout; 466 int64_t t_usecs; 467 int64_t timeout_us; 468 int64_t rcv_timeout_us; 469 470 if (test->logfile) 471 if (iperf_open_logfile(test) < 0) 472 return -1; 473 474 if (test->affinity != -1) 475 if (iperf_setaffinity(test, test->affinity) != 0) 476 return -1; 477 478 if (test->json_output) 479 if (iperf_json_start(test) < 0) 480 return -1; 481 482 if (test->json_output) { 483 cJSON_AddItemToObject(test->json_start, "version", cJSON_CreateString(version)); 484 cJSON_AddItemToObject(test->json_start, "system_info", cJSON_CreateString(get_system_info())); 485 } else if (test->verbose) { 486 iperf_printf(test, "%s\n", version); 487 iperf_printf(test, "%s", ""); 488 iperf_printf(test, "%s\n", get_system_info()); 489 iflush(test); 490 } 491 492 /* Start the client and connect to the server */ 493 if (iperf_connect(test) < 0) 494 goto cleanup_and_fail; 495 496 /* Begin calculating CPU utilization */ 497 cpu_util(NULL); 498 if (test->mode != SENDER) 499 rcv_timeout_us = (test->settings->rcv_timeout.secs * SEC_TO_US) + test->settings->rcv_timeout.usecs; 500 else 501 rcv_timeout_us = 0; 502 503 startup = 1; 504 while (test->state != IPERF_DONE) { 505 memcpy(&read_set, &test->read_set, sizeof(fd_set)); 506 memcpy(&write_set, &test->write_set, sizeof(fd_set)); 507 iperf_time_now(&now); 508 timeout = tmr_timeout(&now); 509 510 // In reverse active mode client ensures data is received 511 if (test->state == TEST_RUNNING && rcv_timeout_us > 0) { 512 timeout_us = -1; 513 if (timeout != NULL) { 514 used_timeout.tv_sec = timeout->tv_sec; 515 used_timeout.tv_usec = timeout->tv_usec; 516 timeout_us = (timeout->tv_sec * SEC_TO_US) + timeout->tv_usec; 517 } 518 if (timeout_us < 0 || timeout_us > rcv_timeout_us) { 519 used_timeout.tv_sec = test->settings->rcv_timeout.secs; 520 used_timeout.tv_usec = test->settings->rcv_timeout.usecs; 521 } 522 timeout = &used_timeout; 523 } 524 525 result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout); 526 if (result < 0 && errno != EINTR) { 527 i_errno = IESELECT; 528 goto cleanup_and_fail; 529 } else if (result == 0 && test->state == TEST_RUNNING && rcv_timeout_us > 0) { 530 // If nothing was received in non-reverse running state then probably something got stack - 531 // either client, server or network, and test should be terminated. 532 iperf_time_now(&now); 533 if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) { 534 t_usecs = iperf_time_in_usecs(&diff_time); 535 if (t_usecs > rcv_timeout_us) { 536 i_errno = IENOMSG; 537 goto cleanup_and_fail; 538 } 539 540 } 541 } 542 543 if (result > 0) { 544 if (rcv_timeout_us > 0) { 545 iperf_time_now(&last_receive_time); 546 } 547 if (FD_ISSET(test->ctrl_sck, &read_set)) { 548 if (iperf_handle_message_client(test) < 0) { 549 goto cleanup_and_fail; 550 } 551 FD_CLR(test->ctrl_sck, &read_set); 552 } 553 } 554 555 if (test->state == TEST_RUNNING) { 556 557 /* Is this our first time really running? */ 558 if (startup) { 559 startup = 0; 560 561 // Set non-blocking for non-UDP tests 562 if (test->protocol->id != Pudp) { 563 SLIST_FOREACH(sp, &test->streams, streams) { 564 setnonblocking(sp->socket, 1); 565 } 566 } 567 } 568 569 570 if (test->mode == BIDIRECTIONAL) 571 { 572 if (iperf_send(test, &write_set) < 0) 573 goto cleanup_and_fail; 574 if (iperf_recv(test, &read_set) < 0) 575 goto cleanup_and_fail; 576 } else if (test->mode == SENDER) { 577 // Regular mode. Client sends. 578 if (iperf_send(test, &write_set) < 0) 579 goto cleanup_and_fail; 580 } else { 581 // Reverse mode. Client receives. 582 if (iperf_recv(test, &read_set) < 0) 583 goto cleanup_and_fail; 584 } 585 586 587 /* Run the timers. */ 588 iperf_time_now(&now); 589 tmr_run(&now); 590 591 /* 592 * Is the test done yet? We have to be out of omitting 593 * mode, and then we have to have fulfilled one of the 594 * ending criteria, either by times, bytes, or blocks. 595 * The bytes and blocks tests needs to handle both the 596 * cases of the client being the sender and the client 597 * being the receiver. 598 */ 599 if ((!test->omitting) && 600 (test->done || 601 (test->settings->bytes != 0 && (test->bytes_sent >= test->settings->bytes || 602 test->bytes_received >= test->settings->bytes)) || 603 (test->settings->blocks != 0 && (test->blocks_sent >= test->settings->blocks || 604 test->blocks_received >= test->settings->blocks)))) { 605 606 // Unset non-blocking for non-UDP tests 607 if (test->protocol->id != Pudp) { 608 SLIST_FOREACH(sp, &test->streams, streams) { 609 setnonblocking(sp->socket, 0); 610 } 611 } 612 613 /* Yes, done! Send TEST_END. */ 614 test->done = 1; 615 cpu_util(test->cpu_util); 616 test->stats_callback(test); 617 if (iperf_set_send_state(test, TEST_END) != 0) 618 goto cleanup_and_fail; 619 } 620 } 621 // If we're in reverse mode, continue draining the data 622 // connection(s) even if test is over. This prevents a 623 // deadlock where the server side fills up its pipe(s) 624 // and gets blocked, so it can't receive state changes 625 // from the client side. 626 else if (test->mode == RECEIVER && test->state == TEST_END) { 627 if (iperf_recv(test, &read_set) < 0) 628 goto cleanup_and_fail; 629 } 630 } 631 632 if (test->json_output) { 633 if (iperf_json_finish(test) < 0) 634 return -1; 635 } else { 636 iperf_printf(test, "\n"); 637 iperf_printf(test, "%s", report_done); 638 } 639 640 iflush(test); 641 642 return 0; 643 644 cleanup_and_fail: 645 iperf_client_end(test); 646 if (test->json_output) 647 iperf_json_finish(test); 648 iflush(test); 649 return -1; 650 } 651