1 /* 2 * iperf, Copyright (c) 2014-2021, The Regents of the University of 3 * California, through Lawrence Berkeley National Laboratory (subject 4 * to receipt of any required approvals from the U.S. Dept. of 5 * Energy). All rights reserved. 6 * 7 * If you have questions about your rights to use or distribute this 8 * software, please contact Berkeley Lab's Technology Transfer 9 * Department at [email protected]. 10 * 11 * NOTICE. This software is owned by the U.S. Department of Energy. 12 * As such, the U.S. Government has been granted for itself and others 13 * acting on its behalf a paid-up, nonexclusive, irrevocable, 14 * worldwide license in the Software to reproduce, prepare derivative 15 * works, and perform publicly and display publicly. Beginning five 16 * (5) years after the date permission to assert copyright is obtained 17 * from the U.S. Department of Energy, and subject to any subsequent 18 * five (5) year renewals, the U.S. Government is granted for itself 19 * and others acting on its behalf a paid-up, nonexclusive, 20 * irrevocable, worldwide license in the Software to reproduce, 21 * prepare derivative works, distribute copies to the public, perform 22 * publicly and display publicly, and to permit others to do so. 23 * 24 * This code is distributed under a BSD style license, see the LICENSE 25 * file for complete information. 26 */ 27 #include <errno.h> 28 #include <setjmp.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <unistd.h> 33 #include <signal.h> 34 #include <sys/types.h> 35 #include <netinet/in.h> 36 #include <sys/select.h> 37 #include <sys/uio.h> 38 #include <arpa/inet.h> 39 40 #include "iperf.h" 41 #include "iperf_api.h" 42 #include "iperf_util.h" 43 #include "iperf_locale.h" 44 #include "iperf_time.h" 45 #include "net.h" 46 #include "timer.h" 47 48 #if defined(HAVE_TCP_CONGESTION) 49 #if !defined(TCP_CA_NAME_MAX) 50 #define TCP_CA_NAME_MAX 16 51 #endif /* TCP_CA_NAME_MAX */ 52 #endif /* HAVE_TCP_CONGESTION */ 53 54 int 55 iperf_create_streams(struct iperf_test *test, int sender) 56 { 57 if (NULL == test) 58 { 59 iperf_err(NULL, "No test\n"); 60 return -1; 61 } 62 int i, s; 63 #if defined(HAVE_TCP_CONGESTION) 64 int saved_errno; 65 #endif /* HAVE_TCP_CONGESTION */ 66 struct iperf_stream *sp; 67 68 int orig_bind_port = test->bind_port; 69 for (i = 0; i < test->num_streams; ++i) { 70 71 test->bind_port = orig_bind_port; 72 if (orig_bind_port) 73 test->bind_port += i; 74 if ((s = test->protocol->connect(test)) < 0) 75 return -1; 76 77 #if defined(HAVE_TCP_CONGESTION) 78 if (test->protocol->id == Ptcp) { 79 if (test->congestion) { 80 if (setsockopt(s, IPPROTO_TCP, TCP_CONGESTION, test->congestion, strlen(test->congestion)) < 0) { 81 saved_errno = errno; 82 close(s); 83 errno = saved_errno; 84 i_errno = IESETCONGESTION; 85 return -1; 86 } 87 } 88 { 89 socklen_t len = TCP_CA_NAME_MAX; 90 char ca[TCP_CA_NAME_MAX + 1]; 91 int rc; 92 rc = getsockopt(s, IPPROTO_TCP, TCP_CONGESTION, ca, &len); 93 if (rc < 0 && test->congestion) { 94 saved_errno = errno; 95 close(s); 96 errno = saved_errno; 97 i_errno = IESETCONGESTION; 98 return -1; 99 } 100 // Set actual used congestion alg, or set to unknown if could not get it 101 if (rc < 0) 102 test->congestion_used = strdup("unknown"); 103 else 104 test->congestion_used = strdup(ca); 105 if (test->debug) { 106 printf("Congestion algorithm is %s\n", test->congestion_used); 107 } 108 } 109 } 110 #endif /* HAVE_TCP_CONGESTION */ 111 112 if (sender) 113 FD_SET(s, &test->write_set); 114 else 115 FD_SET(s, &test->read_set); 116 if (s > test->max_fd) test->max_fd = s; 117 118 sp = iperf_new_stream(test, s, sender); 119 if (!sp) 120 return -1; 121 122 /* Perform the new stream callback */ 123 if (test->on_new_stream) 124 test->on_new_stream(sp); 125 } 126 127 return 0; 128 } 129 130 static void 131 test_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 132 { 133 struct iperf_test *test = client_data.p; 134 135 test->timer = NULL; 136 test->done = 1; 137 } 138 139 static void 140 client_stats_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 141 { 142 struct iperf_test *test = client_data.p; 143 144 if (test->done) 145 return; 146 if (test->stats_callback) 147 test->stats_callback(test); 148 } 149 150 static void 151 client_reporter_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 152 { 153 struct iperf_test *test = client_data.p; 154 155 if (test->done) 156 return; 157 if (test->reporter_callback) 158 test->reporter_callback(test); 159 } 160 161 static int 162 create_client_timers(struct iperf_test * test) 163 { 164 struct iperf_time now; 165 TimerClientData cd; 166 if (NULL == test) 167 { 168 iperf_err(NULL, "No test\n"); 169 i_errno = IEINITTEST; 170 return -1; 171 } 172 173 if (iperf_time_now(&now) < 0) { 174 i_errno = IEINITTEST; 175 return -1; 176 } 177 cd.p = test; 178 test->timer = test->stats_timer = test->reporter_timer = NULL; 179 if (test->duration != 0) { 180 test->done = 0; 181 test->timer = tmr_create(&now, test_timer_proc, cd, ( test->duration + test->omit ) * SEC_TO_US, 0); 182 if (test->timer == NULL) { 183 i_errno = IEINITTEST; 184 return -1; 185 } 186 } 187 if (test->stats_interval != 0) { 188 test->stats_timer = tmr_create(&now, client_stats_timer_proc, cd, test->stats_interval * SEC_TO_US, 1); 189 if (test->stats_timer == NULL) { 190 i_errno = IEINITTEST; 191 return -1; 192 } 193 } 194 if (test->reporter_interval != 0) { 195 test->reporter_timer = tmr_create(&now, client_reporter_timer_proc, cd, test->reporter_interval * SEC_TO_US, 1); 196 if (test->reporter_timer == NULL) { 197 i_errno = IEINITTEST; 198 return -1; 199 } 200 } 201 return 0; 202 } 203 204 static void 205 client_omit_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 206 { 207 struct iperf_test *test = client_data.p; 208 209 test->omit_timer = NULL; 210 test->omitting = 0; 211 iperf_reset_stats(test); 212 if (test->verbose && !test->json_output && test->reporter_interval == 0) 213 iperf_printf(test, "%s", report_omit_done); 214 215 /* Reset the timers. */ 216 if (test->stats_timer != NULL) 217 tmr_reset(nowP, test->stats_timer); 218 if (test->reporter_timer != NULL) 219 tmr_reset(nowP, test->reporter_timer); 220 } 221 222 static int 223 create_client_omit_timer(struct iperf_test * test) 224 { 225 struct iperf_time now; 226 TimerClientData cd; 227 if (NULL == test) 228 { 229 iperf_err(NULL, "No test\n"); 230 return -1; 231 } 232 233 if (test->omit == 0) { 234 test->omit_timer = NULL; 235 test->omitting = 0; 236 } else { 237 if (iperf_time_now(&now) < 0) { 238 i_errno = IEINITTEST; 239 return -1; 240 } 241 test->omitting = 1; 242 cd.p = test; 243 test->omit_timer = tmr_create(&now, client_omit_timer_proc, cd, test->omit * SEC_TO_US, 0); 244 if (test->omit_timer == NULL) { 245 i_errno = IEINITTEST; 246 return -1; 247 } 248 } 249 return 0; 250 } 251 252 int 253 iperf_handle_message_client(struct iperf_test *test) 254 { 255 int rval; 256 int32_t err; 257 258 if (NULL == test) 259 { 260 iperf_err(NULL, "No test\n"); 261 i_errno = IEINITTEST; 262 return -1; 263 } 264 /*!!! Why is this read() and not Nread()? */ 265 if ((rval = read(test->ctrl_sck, (char*) &test->state, sizeof(signed char))) <= 0) { 266 if (rval == 0) { 267 i_errno = IECTRLCLOSE; 268 return -1; 269 } else { 270 i_errno = IERECVMESSAGE; 271 return -1; 272 } 273 } 274 275 switch (test->state) { 276 case PARAM_EXCHANGE: 277 if (iperf_exchange_parameters(test) < 0) 278 return -1; 279 if (test->on_connect) 280 test->on_connect(test); 281 break; 282 case CREATE_STREAMS: 283 if (test->mode == BIDIRECTIONAL) 284 { 285 if (iperf_create_streams(test, 1) < 0) 286 return -1; 287 if (iperf_create_streams(test, 0) < 0) 288 return -1; 289 } 290 else if (iperf_create_streams(test, test->mode) < 0) 291 return -1; 292 break; 293 case TEST_START: 294 if (iperf_init_test(test) < 0) 295 return -1; 296 if (create_client_timers(test) < 0) 297 return -1; 298 if (create_client_omit_timer(test) < 0) 299 return -1; 300 if (test->mode) 301 if (iperf_create_send_timers(test) < 0) 302 return -1; 303 break; 304 case TEST_RUNNING: 305 break; 306 case EXCHANGE_RESULTS: 307 if (iperf_exchange_results(test) < 0) 308 return -1; 309 break; 310 case DISPLAY_RESULTS: 311 if (test->on_test_finish) 312 test->on_test_finish(test); 313 iperf_client_end(test); 314 break; 315 case IPERF_DONE: 316 break; 317 case SERVER_TERMINATE: 318 i_errno = IESERVERTERM; 319 320 /* 321 * Temporarily be in DISPLAY_RESULTS phase so we can get 322 * ending summary statistics. 323 */ 324 signed char oldstate = test->state; 325 cpu_util(test->cpu_util); 326 test->state = DISPLAY_RESULTS; 327 test->reporter_callback(test); 328 test->state = oldstate; 329 return -1; 330 case ACCESS_DENIED: 331 i_errno = IEACCESSDENIED; 332 return -1; 333 case SERVER_ERROR: 334 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 335 i_errno = IECTRLREAD; 336 return -1; 337 } 338 i_errno = ntohl(err); 339 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 340 i_errno = IECTRLREAD; 341 return -1; 342 } 343 errno = ntohl(err); 344 return -1; 345 default: 346 i_errno = IEMESSAGE; 347 return -1; 348 } 349 350 return 0; 351 } 352 353 354 355 /* iperf_connect -- client to server connection function */ 356 int 357 iperf_connect(struct iperf_test *test) 358 { 359 if (NULL == test) 360 { 361 iperf_err(NULL, "No test\n"); 362 return -1; 363 } 364 FD_ZERO(&test->read_set); 365 FD_ZERO(&test->write_set); 366 367 make_cookie(test->cookie); 368 369 /* Create and connect the control channel */ 370 if (test->ctrl_sck < 0) 371 // Create the control channel using an ephemeral port 372 test->ctrl_sck = netdial(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, 0, test->server_hostname, test->server_port, test->settings->connect_timeout); 373 if (test->ctrl_sck < 0) { 374 i_errno = IECONNECT; 375 return -1; 376 } 377 378 // set TCP_NODELAY for lower latency on control messages 379 int flag = 1; 380 if (setsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int))) { 381 i_errno = IESETNODELAY; 382 return -1; 383 } 384 385 if (Nwrite(test->ctrl_sck, test->cookie, COOKIE_SIZE, Ptcp) < 0) { 386 i_errno = IESENDCOOKIE; 387 return -1; 388 } 389 390 FD_SET(test->ctrl_sck, &test->read_set); 391 if (test->ctrl_sck > test->max_fd) test->max_fd = test->ctrl_sck; 392 393 int opt; 394 socklen_t len; 395 396 len = sizeof(opt); 397 if (getsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_MAXSEG, &opt, &len) < 0) { 398 test->ctrl_sck_mss = 0; 399 } 400 else { 401 if (opt > 0 && opt <= MAX_UDP_BLOCKSIZE) { 402 test->ctrl_sck_mss = opt; 403 } 404 else { 405 char str[WARN_STR_LEN]; 406 snprintf(str, sizeof(str), 407 "Ignoring nonsense TCP MSS %d", opt); 408 warning(str); 409 410 test->ctrl_sck_mss = 0; 411 } 412 } 413 414 if (test->verbose) { 415 printf("Control connection MSS %d\n", test->ctrl_sck_mss); 416 } 417 418 /* 419 * If we're doing a UDP test and the block size wasn't explicitly 420 * set, then use the known MSS of the control connection to pick 421 * an appropriate default. If we weren't able to get the 422 * MSS for some reason, then default to something that should 423 * work on non-jumbo-frame Ethernet networks. The goal is to 424 * pick a reasonable default that is large but should get from 425 * sender to receiver without any IP fragmentation. 426 * 427 * We assume that the control connection is routed the same as the 428 * data packets (thus has the same PMTU). Also in the case of 429 * --reverse tests, we assume that the MTU is the same in both 430 * directions. Note that even if the algorithm guesses wrong, 431 * the user always has the option to override. 432 */ 433 if (test->protocol->id == Pudp) { 434 if (test->settings->blksize == 0) { 435 if (test->ctrl_sck_mss) { 436 test->settings->blksize = test->ctrl_sck_mss; 437 } 438 else { 439 test->settings->blksize = DEFAULT_UDP_BLKSIZE; 440 } 441 if (test->verbose) { 442 printf("Setting UDP block size to %d\n", test->settings->blksize); 443 } 444 } 445 446 /* 447 * Regardless of whether explicitly or implicitly set, if the 448 * block size is larger than the MSS, print a warning. 449 */ 450 if (test->ctrl_sck_mss > 0 && 451 test->settings->blksize > test->ctrl_sck_mss) { 452 char str[WARN_STR_LEN]; 453 snprintf(str, sizeof(str), 454 "UDP block size %d exceeds TCP MSS %d, may result in fragmentation / drops", test->settings->blksize, test->ctrl_sck_mss); 455 warning(str); 456 } 457 } 458 459 return 0; 460 } 461 462 463 int 464 iperf_client_end(struct iperf_test *test) 465 { 466 if (NULL == test) 467 { 468 iperf_err(NULL, "No test\n"); 469 return -1; 470 } 471 struct iperf_stream *sp; 472 473 /* Close all stream sockets */ 474 SLIST_FOREACH(sp, &test->streams, streams) { 475 close(sp->socket); 476 } 477 478 /* show final summary */ 479 test->reporter_callback(test); 480 481 /* Send response only if no error in server */ 482 if (test->state > 0) { 483 if (iperf_set_send_state(test, IPERF_DONE) != 0) 484 return -1; 485 } 486 487 /* Close control socket */ 488 if (test->ctrl_sck >= 0) 489 close(test->ctrl_sck); 490 491 return 0; 492 } 493 494 495 int 496 iperf_run_client(struct iperf_test * test) 497 { 498 int startup; 499 int result = 0; 500 fd_set read_set, write_set; 501 struct iperf_time now; 502 struct timeval* timeout = NULL; 503 struct iperf_stream *sp; 504 struct iperf_time last_receive_time; 505 struct iperf_time diff_time; 506 struct timeval used_timeout; 507 int64_t t_usecs; 508 int64_t timeout_us; 509 int64_t rcv_timeout_us; 510 511 if (NULL == test) 512 { 513 iperf_err(NULL, "No test\n"); 514 return -1; 515 } 516 517 if (test->logfile) 518 if (iperf_open_logfile(test) < 0) 519 return -1; 520 521 if (test->affinity != -1) 522 if (iperf_setaffinity(test, test->affinity) != 0) 523 return -1; 524 525 if (test->json_output) 526 if (iperf_json_start(test) < 0) 527 return -1; 528 529 if (test->json_output) { 530 cJSON_AddItemToObject(test->json_start, "version", cJSON_CreateString(version)); 531 cJSON_AddItemToObject(test->json_start, "system_info", cJSON_CreateString(get_system_info())); 532 } else if (test->verbose) { 533 iperf_printf(test, "%s\n", version); 534 iperf_printf(test, "%s", ""); 535 iperf_printf(test, "%s\n", get_system_info()); 536 iflush(test); 537 } 538 539 /* Start the client and connect to the server */ 540 if (iperf_connect(test) < 0) 541 goto cleanup_and_fail; 542 543 /* Begin calculating CPU utilization */ 544 cpu_util(NULL); 545 if (test->mode != SENDER) 546 rcv_timeout_us = (test->settings->rcv_timeout.secs * SEC_TO_US) + test->settings->rcv_timeout.usecs; 547 else 548 rcv_timeout_us = 0; 549 550 startup = 1; 551 while (test->state != IPERF_DONE) { 552 memcpy(&read_set, &test->read_set, sizeof(fd_set)); 553 memcpy(&write_set, &test->write_set, sizeof(fd_set)); 554 iperf_time_now(&now); 555 timeout = tmr_timeout(&now); 556 557 // In reverse active mode client ensures data is received 558 if (test->state == TEST_RUNNING && rcv_timeout_us > 0) { 559 timeout_us = -1; 560 if (timeout != NULL) { 561 used_timeout.tv_sec = timeout->tv_sec; 562 used_timeout.tv_usec = timeout->tv_usec; 563 timeout_us = (timeout->tv_sec * SEC_TO_US) + timeout->tv_usec; 564 } 565 if (timeout_us < 0 || timeout_us > rcv_timeout_us) { 566 used_timeout.tv_sec = test->settings->rcv_timeout.secs; 567 used_timeout.tv_usec = test->settings->rcv_timeout.usecs; 568 } 569 timeout = &used_timeout; 570 } 571 572 result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout); 573 if (result < 0 && errno != EINTR) { 574 i_errno = IESELECT; 575 goto cleanup_and_fail; 576 } else if (result == 0 && test->state == TEST_RUNNING && rcv_timeout_us > 0) { 577 // If nothing was received in non-reverse running state then probably something got stack - 578 // either client, server or network, and test should be terminated. 579 iperf_time_now(&now); 580 if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) { 581 t_usecs = iperf_time_in_usecs(&diff_time); 582 if (t_usecs > rcv_timeout_us) { 583 i_errno = IENOMSG; 584 goto cleanup_and_fail; 585 } 586 587 } 588 } 589 590 if (result > 0) { 591 if (rcv_timeout_us > 0) { 592 iperf_time_now(&last_receive_time); 593 } 594 if (FD_ISSET(test->ctrl_sck, &read_set)) { 595 if (iperf_handle_message_client(test) < 0) { 596 goto cleanup_and_fail; 597 } 598 FD_CLR(test->ctrl_sck, &read_set); 599 } 600 } 601 602 if (test->state == TEST_RUNNING) { 603 604 /* Is this our first time really running? */ 605 if (startup) { 606 startup = 0; 607 608 // Set non-blocking for non-UDP tests 609 if (test->protocol->id != Pudp) { 610 SLIST_FOREACH(sp, &test->streams, streams) { 611 setnonblocking(sp->socket, 1); 612 } 613 } 614 } 615 616 617 if (test->mode == BIDIRECTIONAL) 618 { 619 if (iperf_send(test, &write_set) < 0) 620 goto cleanup_and_fail; 621 if (iperf_recv(test, &read_set) < 0) 622 goto cleanup_and_fail; 623 } else if (test->mode == SENDER) { 624 // Regular mode. Client sends. 625 if (iperf_send(test, &write_set) < 0) 626 goto cleanup_and_fail; 627 } else { 628 // Reverse mode. Client receives. 629 if (iperf_recv(test, &read_set) < 0) 630 goto cleanup_and_fail; 631 } 632 633 634 /* Run the timers. */ 635 iperf_time_now(&now); 636 tmr_run(&now); 637 638 /* 639 * Is the test done yet? We have to be out of omitting 640 * mode, and then we have to have fulfilled one of the 641 * ending criteria, either by times, bytes, or blocks. 642 * The bytes and blocks tests needs to handle both the 643 * cases of the client being the sender and the client 644 * being the receiver. 645 */ 646 if ((!test->omitting) && 647 (test->done || 648 (test->settings->bytes != 0 && (test->bytes_sent >= test->settings->bytes || 649 test->bytes_received >= test->settings->bytes)) || 650 (test->settings->blocks != 0 && (test->blocks_sent >= test->settings->blocks || 651 test->blocks_received >= test->settings->blocks)))) { 652 653 // Unset non-blocking for non-UDP tests 654 if (test->protocol->id != Pudp) { 655 SLIST_FOREACH(sp, &test->streams, streams) { 656 setnonblocking(sp->socket, 0); 657 } 658 } 659 660 /* Yes, done! Send TEST_END. */ 661 test->done = 1; 662 cpu_util(test->cpu_util); 663 test->stats_callback(test); 664 if (iperf_set_send_state(test, TEST_END) != 0) 665 goto cleanup_and_fail; 666 } 667 } 668 // If we're in reverse mode, continue draining the data 669 // connection(s) even if test is over. This prevents a 670 // deadlock where the server side fills up its pipe(s) 671 // and gets blocked, so it can't receive state changes 672 // from the client side. 673 else if (test->mode == RECEIVER && test->state == TEST_END) { 674 if (iperf_recv(test, &read_set) < 0) 675 goto cleanup_and_fail; 676 } 677 } 678 679 if (test->json_output) { 680 if (iperf_json_finish(test) < 0) 681 return -1; 682 } else { 683 iperf_printf(test, "\n"); 684 iperf_printf(test, "%s", report_done); 685 } 686 687 iflush(test); 688 689 return 0; 690 691 cleanup_and_fail: 692 iperf_client_end(test); 693 if (test->json_output) { 694 cJSON_AddStringToObject(test->json_top, "error", iperf_strerror(i_errno)); 695 iperf_json_finish(test); 696 iflush(test); 697 // Return 0 and not -1 since all terminating function were done here. 698 // Also prevents error message logging outside the already closed JSON output. 699 return 0; 700 } 701 iflush(test); 702 return -1; 703 } 704