1 /* 2 * iperf, Copyright (c) 2014-2021, The Regents of the University of 3 * California, through Lawrence Berkeley National Laboratory (subject 4 * to receipt of any required approvals from the U.S. Dept. of 5 * Energy). All rights reserved. 6 * 7 * If you have questions about your rights to use or distribute this 8 * software, please contact Berkeley Lab's Technology Transfer 9 * Department at [email protected]. 10 * 11 * NOTICE. This software is owned by the U.S. Department of Energy. 12 * As such, the U.S. Government has been granted for itself and others 13 * acting on its behalf a paid-up, nonexclusive, irrevocable, 14 * worldwide license in the Software to reproduce, prepare derivative 15 * works, and perform publicly and display publicly. Beginning five 16 * (5) years after the date permission to assert copyright is obtained 17 * from the U.S. Department of Energy, and subject to any subsequent 18 * five (5) year renewals, the U.S. Government is granted for itself 19 * and others acting on its behalf a paid-up, nonexclusive, 20 * irrevocable, worldwide license in the Software to reproduce, 21 * prepare derivative works, distribute copies to the public, perform 22 * publicly and display publicly, and to permit others to do so. 23 * 24 * This code is distributed under a BSD style license, see the LICENSE 25 * file for complete information. 26 */ 27 #include <errno.h> 28 #include <setjmp.h> 29 #include <stdio.h> 30 #include <stdlib.h> 31 #include <string.h> 32 #include <unistd.h> 33 #include <signal.h> 34 #include <sys/types.h> 35 #include <netinet/in.h> 36 #include <sys/select.h> 37 #include <sys/uio.h> 38 #include <arpa/inet.h> 39 40 #include "iperf.h" 41 #include "iperf_api.h" 42 #include "iperf_util.h" 43 #include "iperf_locale.h" 44 #include "iperf_time.h" 45 #include "net.h" 46 #include "timer.h" 47 48 #if defined(HAVE_TCP_CONGESTION) 49 #if !defined(TCP_CA_NAME_MAX) 50 #define TCP_CA_NAME_MAX 16 51 #endif /* TCP_CA_NAME_MAX */ 52 #endif /* HAVE_TCP_CONGESTION */ 53 54 int 55 iperf_create_streams(struct iperf_test *test, int sender) 56 { 57 int i, s; 58 #if defined(HAVE_TCP_CONGESTION) 59 int saved_errno; 60 #endif /* HAVE_TCP_CONGESTION */ 61 struct iperf_stream *sp; 62 63 int orig_bind_port = test->bind_port; 64 for (i = 0; i < test->num_streams; ++i) { 65 66 test->bind_port = orig_bind_port; 67 if (orig_bind_port) 68 test->bind_port += i; 69 if ((s = test->protocol->connect(test)) < 0) 70 return -1; 71 72 #if defined(HAVE_TCP_CONGESTION) 73 if (test->protocol->id == Ptcp) { 74 if (test->congestion) { 75 if (setsockopt(s, IPPROTO_TCP, TCP_CONGESTION, test->congestion, strlen(test->congestion)) < 0) { 76 saved_errno = errno; 77 close(s); 78 errno = saved_errno; 79 i_errno = IESETCONGESTION; 80 return -1; 81 } 82 } 83 { 84 socklen_t len = TCP_CA_NAME_MAX; 85 char ca[TCP_CA_NAME_MAX + 1]; 86 int rc; 87 rc = getsockopt(s, IPPROTO_TCP, TCP_CONGESTION, ca, &len); 88 if (rc < 0 && test->congestion) { 89 saved_errno = errno; 90 close(s); 91 errno = saved_errno; 92 i_errno = IESETCONGESTION; 93 return -1; 94 } 95 // Set actual used congestion alg, or set to unknown if could not get it 96 if (rc < 0) 97 test->congestion_used = strdup("unknown"); 98 else 99 test->congestion_used = strdup(ca); 100 if (test->debug) { 101 printf("Congestion algorithm is %s\n", test->congestion_used); 102 } 103 } 104 } 105 #endif /* HAVE_TCP_CONGESTION */ 106 107 if (sender) 108 FD_SET(s, &test->write_set); 109 else 110 FD_SET(s, &test->read_set); 111 if (s > test->max_fd) test->max_fd = s; 112 113 sp = iperf_new_stream(test, s, sender); 114 if (!sp) 115 return -1; 116 117 /* Perform the new stream callback */ 118 if (test->on_new_stream) 119 test->on_new_stream(sp); 120 } 121 122 return 0; 123 } 124 125 static void 126 test_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 127 { 128 struct iperf_test *test = client_data.p; 129 130 test->timer = NULL; 131 test->done = 1; 132 } 133 134 static void 135 client_stats_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 136 { 137 struct iperf_test *test = client_data.p; 138 139 if (test->done) 140 return; 141 if (test->stats_callback) 142 test->stats_callback(test); 143 } 144 145 static void 146 client_reporter_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 147 { 148 struct iperf_test *test = client_data.p; 149 150 if (test->done) 151 return; 152 if (test->reporter_callback) 153 test->reporter_callback(test); 154 } 155 156 static int 157 create_client_timers(struct iperf_test * test) 158 { 159 struct iperf_time now; 160 TimerClientData cd; 161 162 if (iperf_time_now(&now) < 0) { 163 i_errno = IEINITTEST; 164 return -1; 165 } 166 cd.p = test; 167 test->timer = test->stats_timer = test->reporter_timer = NULL; 168 if (test->duration != 0) { 169 test->done = 0; 170 test->timer = tmr_create(&now, test_timer_proc, cd, ( test->duration + test->omit ) * SEC_TO_US, 0); 171 if (test->timer == NULL) { 172 i_errno = IEINITTEST; 173 return -1; 174 } 175 } 176 if (test->stats_interval != 0) { 177 test->stats_timer = tmr_create(&now, client_stats_timer_proc, cd, test->stats_interval * SEC_TO_US, 1); 178 if (test->stats_timer == NULL) { 179 i_errno = IEINITTEST; 180 return -1; 181 } 182 } 183 if (test->reporter_interval != 0) { 184 test->reporter_timer = tmr_create(&now, client_reporter_timer_proc, cd, test->reporter_interval * SEC_TO_US, 1); 185 if (test->reporter_timer == NULL) { 186 i_errno = IEINITTEST; 187 return -1; 188 } 189 } 190 return 0; 191 } 192 193 static void 194 client_omit_timer_proc(TimerClientData client_data, struct iperf_time *nowP) 195 { 196 struct iperf_test *test = client_data.p; 197 198 test->omit_timer = NULL; 199 test->omitting = 0; 200 iperf_reset_stats(test); 201 if (test->verbose && !test->json_output && test->reporter_interval == 0) 202 iperf_printf(test, "%s", report_omit_done); 203 204 /* Reset the timers. */ 205 if (test->stats_timer != NULL) 206 tmr_reset(nowP, test->stats_timer); 207 if (test->reporter_timer != NULL) 208 tmr_reset(nowP, test->reporter_timer); 209 } 210 211 static int 212 create_client_omit_timer(struct iperf_test * test) 213 { 214 struct iperf_time now; 215 TimerClientData cd; 216 217 if (test->omit == 0) { 218 test->omit_timer = NULL; 219 test->omitting = 0; 220 } else { 221 if (iperf_time_now(&now) < 0) { 222 i_errno = IEINITTEST; 223 return -1; 224 } 225 test->omitting = 1; 226 cd.p = test; 227 test->omit_timer = tmr_create(&now, client_omit_timer_proc, cd, test->omit * SEC_TO_US, 0); 228 if (test->omit_timer == NULL) { 229 i_errno = IEINITTEST; 230 return -1; 231 } 232 } 233 return 0; 234 } 235 236 int 237 iperf_handle_message_client(struct iperf_test *test) 238 { 239 int rval; 240 int32_t err; 241 242 /*!!! Why is this read() and not Nread()? */ 243 if ((rval = read(test->ctrl_sck, (char*) &test->state, sizeof(signed char))) <= 0) { 244 if (rval == 0) { 245 i_errno = IECTRLCLOSE; 246 return -1; 247 } else { 248 i_errno = IERECVMESSAGE; 249 return -1; 250 } 251 } 252 253 switch (test->state) { 254 case PARAM_EXCHANGE: 255 if (iperf_exchange_parameters(test) < 0) 256 return -1; 257 if (test->on_connect) 258 test->on_connect(test); 259 break; 260 case CREATE_STREAMS: 261 if (test->mode == BIDIRECTIONAL) 262 { 263 if (iperf_create_streams(test, 1) < 0) 264 return -1; 265 if (iperf_create_streams(test, 0) < 0) 266 return -1; 267 } 268 else if (iperf_create_streams(test, test->mode) < 0) 269 return -1; 270 break; 271 case TEST_START: 272 if (iperf_init_test(test) < 0) 273 return -1; 274 if (create_client_timers(test) < 0) 275 return -1; 276 if (create_client_omit_timer(test) < 0) 277 return -1; 278 if (test->mode) 279 if (iperf_create_send_timers(test) < 0) 280 return -1; 281 break; 282 case TEST_RUNNING: 283 break; 284 case EXCHANGE_RESULTS: 285 if (iperf_exchange_results(test) < 0) 286 return -1; 287 break; 288 case DISPLAY_RESULTS: 289 if (test->on_test_finish) 290 test->on_test_finish(test); 291 iperf_client_end(test); 292 break; 293 case IPERF_DONE: 294 break; 295 case SERVER_TERMINATE: 296 i_errno = IESERVERTERM; 297 298 /* 299 * Temporarily be in DISPLAY_RESULTS phase so we can get 300 * ending summary statistics. 301 */ 302 signed char oldstate = test->state; 303 cpu_util(test->cpu_util); 304 test->state = DISPLAY_RESULTS; 305 test->reporter_callback(test); 306 test->state = oldstate; 307 return -1; 308 case ACCESS_DENIED: 309 i_errno = IEACCESSDENIED; 310 return -1; 311 case SERVER_ERROR: 312 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 313 i_errno = IECTRLREAD; 314 return -1; 315 } 316 i_errno = ntohl(err); 317 if (Nread(test->ctrl_sck, (char*) &err, sizeof(err), Ptcp) < 0) { 318 i_errno = IECTRLREAD; 319 return -1; 320 } 321 errno = ntohl(err); 322 return -1; 323 default: 324 i_errno = IEMESSAGE; 325 return -1; 326 } 327 328 return 0; 329 } 330 331 332 333 /* iperf_connect -- client to server connection function */ 334 int 335 iperf_connect(struct iperf_test *test) 336 { 337 FD_ZERO(&test->read_set); 338 FD_ZERO(&test->write_set); 339 340 make_cookie(test->cookie); 341 342 /* Create and connect the control channel */ 343 if (test->ctrl_sck < 0) 344 // Create the control channel using an ephemeral port 345 test->ctrl_sck = netdial(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, 0, test->server_hostname, test->server_port, test->settings->connect_timeout); 346 if (test->ctrl_sck < 0) { 347 i_errno = IECONNECT; 348 return -1; 349 } 350 351 // set TCP_NODELAY for lower latency on control messages 352 int flag = 1; 353 if (setsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int))) { 354 i_errno = IESETNODELAY; 355 return -1; 356 } 357 358 if (Nwrite(test->ctrl_sck, test->cookie, COOKIE_SIZE, Ptcp) < 0) { 359 i_errno = IESENDCOOKIE; 360 return -1; 361 } 362 363 FD_SET(test->ctrl_sck, &test->read_set); 364 if (test->ctrl_sck > test->max_fd) test->max_fd = test->ctrl_sck; 365 366 int opt; 367 socklen_t len; 368 369 len = sizeof(opt); 370 if (getsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_MAXSEG, &opt, &len) < 0) { 371 test->ctrl_sck_mss = 0; 372 } 373 else { 374 if (opt > 0 && opt <= MAX_UDP_BLOCKSIZE) { 375 test->ctrl_sck_mss = opt; 376 } 377 else { 378 char str[128]; 379 snprintf(str, sizeof(str), 380 "Ignoring nonsense TCP MSS %d", opt); 381 warning(str); 382 383 test->ctrl_sck_mss = 0; 384 } 385 } 386 387 if (test->verbose) { 388 printf("Control connection MSS %d\n", test->ctrl_sck_mss); 389 } 390 391 /* 392 * If we're doing a UDP test and the block size wasn't explicitly 393 * set, then use the known MSS of the control connection to pick 394 * an appropriate default. If we weren't able to get the 395 * MSS for some reason, then default to something that should 396 * work on non-jumbo-frame Ethernet networks. The goal is to 397 * pick a reasonable default that is large but should get from 398 * sender to receiver without any IP fragmentation. 399 * 400 * We assume that the control connection is routed the same as the 401 * data packets (thus has the same PMTU). Also in the case of 402 * --reverse tests, we assume that the MTU is the same in both 403 * directions. Note that even if the algorithm guesses wrong, 404 * the user always has the option to override. 405 */ 406 if (test->protocol->id == Pudp) { 407 if (test->settings->blksize == 0) { 408 if (test->ctrl_sck_mss) { 409 test->settings->blksize = test->ctrl_sck_mss; 410 } 411 else { 412 test->settings->blksize = DEFAULT_UDP_BLKSIZE; 413 } 414 if (test->verbose) { 415 printf("Setting UDP block size to %d\n", test->settings->blksize); 416 } 417 } 418 419 /* 420 * Regardless of whether explicitly or implicitly set, if the 421 * block size is larger than the MSS, print a warning. 422 */ 423 if (test->ctrl_sck_mss > 0 && 424 test->settings->blksize > test->ctrl_sck_mss) { 425 char str[128]; 426 snprintf(str, sizeof(str), 427 "UDP block size %d exceeds TCP MSS %d, may result in fragmentation / drops", test->settings->blksize, test->ctrl_sck_mss); 428 warning(str); 429 } 430 } 431 432 return 0; 433 } 434 435 436 int 437 iperf_client_end(struct iperf_test *test) 438 { 439 struct iperf_stream *sp; 440 441 /* Close all stream sockets */ 442 SLIST_FOREACH(sp, &test->streams, streams) { 443 close(sp->socket); 444 } 445 446 /* show final summary */ 447 test->reporter_callback(test); 448 449 /* Send response only if no error in server */ 450 if (test->state > 0) { 451 if (iperf_set_send_state(test, IPERF_DONE) != 0) 452 return -1; 453 } 454 455 /* Close control socket */ 456 if (test->ctrl_sck >= 0) 457 close(test->ctrl_sck); 458 459 return 0; 460 } 461 462 463 int 464 iperf_run_client(struct iperf_test * test) 465 { 466 int startup; 467 int result = 0; 468 fd_set read_set, write_set; 469 struct iperf_time now; 470 struct timeval* timeout = NULL; 471 struct iperf_stream *sp; 472 struct iperf_time last_receive_time; 473 struct iperf_time diff_time; 474 struct timeval used_timeout; 475 int64_t t_usecs; 476 int64_t timeout_us; 477 int64_t rcv_timeout_us; 478 479 if (test->logfile) 480 if (iperf_open_logfile(test) < 0) 481 return -1; 482 483 if (test->affinity != -1) 484 if (iperf_setaffinity(test, test->affinity) != 0) 485 return -1; 486 487 if (test->json_output) 488 if (iperf_json_start(test) < 0) 489 return -1; 490 491 if (test->json_output) { 492 cJSON_AddItemToObject(test->json_start, "version", cJSON_CreateString(version)); 493 cJSON_AddItemToObject(test->json_start, "system_info", cJSON_CreateString(get_system_info())); 494 } else if (test->verbose) { 495 iperf_printf(test, "%s\n", version); 496 iperf_printf(test, "%s", ""); 497 iperf_printf(test, "%s\n", get_system_info()); 498 iflush(test); 499 } 500 501 /* Start the client and connect to the server */ 502 if (iperf_connect(test) < 0) 503 goto cleanup_and_fail; 504 505 /* Begin calculating CPU utilization */ 506 cpu_util(NULL); 507 if (test->mode != SENDER) 508 rcv_timeout_us = (test->settings->rcv_timeout.secs * SEC_TO_US) + test->settings->rcv_timeout.usecs; 509 else 510 rcv_timeout_us = 0; 511 512 startup = 1; 513 while (test->state != IPERF_DONE) { 514 memcpy(&read_set, &test->read_set, sizeof(fd_set)); 515 memcpy(&write_set, &test->write_set, sizeof(fd_set)); 516 iperf_time_now(&now); 517 timeout = tmr_timeout(&now); 518 519 // In reverse active mode client ensures data is received 520 if (test->state == TEST_RUNNING && rcv_timeout_us > 0) { 521 timeout_us = -1; 522 if (timeout != NULL) { 523 used_timeout.tv_sec = timeout->tv_sec; 524 used_timeout.tv_usec = timeout->tv_usec; 525 timeout_us = (timeout->tv_sec * SEC_TO_US) + timeout->tv_usec; 526 } 527 if (timeout_us < 0 || timeout_us > rcv_timeout_us) { 528 used_timeout.tv_sec = test->settings->rcv_timeout.secs; 529 used_timeout.tv_usec = test->settings->rcv_timeout.usecs; 530 } 531 timeout = &used_timeout; 532 } 533 534 result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout); 535 if (result < 0 && errno != EINTR) { 536 i_errno = IESELECT; 537 goto cleanup_and_fail; 538 } else if (result == 0 && test->state == TEST_RUNNING && rcv_timeout_us > 0) { 539 // If nothing was received in non-reverse running state then probably something got stack - 540 // either client, server or network, and test should be terminated. 541 iperf_time_now(&now); 542 if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) { 543 t_usecs = iperf_time_in_usecs(&diff_time); 544 if (t_usecs > rcv_timeout_us) { 545 i_errno = IENOMSG; 546 goto cleanup_and_fail; 547 } 548 549 } 550 } 551 552 if (result > 0) { 553 if (rcv_timeout_us > 0) { 554 iperf_time_now(&last_receive_time); 555 } 556 if (FD_ISSET(test->ctrl_sck, &read_set)) { 557 if (iperf_handle_message_client(test) < 0) { 558 goto cleanup_and_fail; 559 } 560 FD_CLR(test->ctrl_sck, &read_set); 561 } 562 } 563 564 if (test->state == TEST_RUNNING) { 565 566 /* Is this our first time really running? */ 567 if (startup) { 568 startup = 0; 569 570 // Set non-blocking for non-UDP tests 571 if (test->protocol->id != Pudp) { 572 SLIST_FOREACH(sp, &test->streams, streams) { 573 setnonblocking(sp->socket, 1); 574 } 575 } 576 } 577 578 579 if (test->mode == BIDIRECTIONAL) 580 { 581 if (iperf_send(test, &write_set) < 0) 582 goto cleanup_and_fail; 583 if (iperf_recv(test, &read_set) < 0) 584 goto cleanup_and_fail; 585 } else if (test->mode == SENDER) { 586 // Regular mode. Client sends. 587 if (iperf_send(test, &write_set) < 0) 588 goto cleanup_and_fail; 589 } else { 590 // Reverse mode. Client receives. 591 if (iperf_recv(test, &read_set) < 0) 592 goto cleanup_and_fail; 593 } 594 595 596 /* Run the timers. */ 597 iperf_time_now(&now); 598 tmr_run(&now); 599 600 /* 601 * Is the test done yet? We have to be out of omitting 602 * mode, and then we have to have fulfilled one of the 603 * ending criteria, either by times, bytes, or blocks. 604 * The bytes and blocks tests needs to handle both the 605 * cases of the client being the sender and the client 606 * being the receiver. 607 */ 608 if ((!test->omitting) && 609 (test->done || 610 (test->settings->bytes != 0 && (test->bytes_sent >= test->settings->bytes || 611 test->bytes_received >= test->settings->bytes)) || 612 (test->settings->blocks != 0 && (test->blocks_sent >= test->settings->blocks || 613 test->blocks_received >= test->settings->blocks)))) { 614 615 // Unset non-blocking for non-UDP tests 616 if (test->protocol->id != Pudp) { 617 SLIST_FOREACH(sp, &test->streams, streams) { 618 setnonblocking(sp->socket, 0); 619 } 620 } 621 622 /* Yes, done! Send TEST_END. */ 623 test->done = 1; 624 cpu_util(test->cpu_util); 625 test->stats_callback(test); 626 if (iperf_set_send_state(test, TEST_END) != 0) 627 goto cleanup_and_fail; 628 } 629 } 630 // If we're in reverse mode, continue draining the data 631 // connection(s) even if test is over. This prevents a 632 // deadlock where the server side fills up its pipe(s) 633 // and gets blocked, so it can't receive state changes 634 // from the client side. 635 else if (test->mode == RECEIVER && test->state == TEST_END) { 636 if (iperf_recv(test, &read_set) < 0) 637 goto cleanup_and_fail; 638 } 639 } 640 641 if (test->json_output) { 642 if (iperf_json_finish(test) < 0) 643 return -1; 644 } else { 645 iperf_printf(test, "\n"); 646 iperf_printf(test, "%s", report_done); 647 } 648 649 iflush(test); 650 651 return 0; 652 653 cleanup_and_fail: 654 iperf_errexit(test, "error - %s", iperf_strerror(i_errno)); 655 iperf_client_end(test); 656 if (test->json_output) { 657 if (iperf_json_finish(test) < 0) 658 return -1; // It is o.k. that error will be logged later outside the JSON output since its creation failed 659 } 660 iflush(test); 661 return 0; // Return 0 and not -1 since all terminating function were done here. 662 // Also prevents error message logging outside the already closed JSON output. 663 } 664