xref: /iperf/src/iperf_server_api.c (revision 76bd67f6)
1 /*
2  * iperf, Copyright (c) 2014-2022 The Regents of the University of
3  * California, through Lawrence Berkeley National Laboratory (subject
4  * to receipt of any required approvals from the U.S. Dept. of
5  * Energy).  All rights reserved.
6  *
7  * If you have questions about your rights to use or distribute this
8  * software, please contact Berkeley Lab's Technology Transfer
9  * Department at [email protected].
10  *
11  * NOTICE.  This software is owned by the U.S. Department of Energy.
12  * As such, the U.S. Government has been granted for itself and others
13  * acting on its behalf a paid-up, nonexclusive, irrevocable,
14  * worldwide license in the Software to reproduce, prepare derivative
15  * works, and perform publicly and display publicly.  Beginning five
16  * (5) years after the date permission to assert copyright is obtained
17  * from the U.S. Department of Energy, and subject to any subsequent
18  * five (5) year renewals, the U.S. Government is granted for itself
19  * and others acting on its behalf a paid-up, nonexclusive,
20  * irrevocable, worldwide license in the Software to reproduce,
21  * prepare derivative works, distribute copies to the public, perform
22  * publicly and display publicly, and to permit others to do so.
23  *
24  * This code is distributed under a BSD style license, see the LICENSE
25  * file for complete information.
26  */
27 /* iperf_server_api.c: Functions to be used by an iperf server
28 */
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <getopt.h>
34 #include <errno.h>
35 #include <unistd.h>
36 #include <assert.h>
37 #include <fcntl.h>
38 #include <sys/socket.h>
39 #include <sys/types.h>
40 #include <netinet/in.h>
41 #include <arpa/inet.h>
42 #include <netdb.h>
43 #ifdef HAVE_STDINT_H
44 #include <stdint.h>
45 #endif
46 #include <sys/time.h>
47 #include <sys/resource.h>
48 #include <sched.h>
49 #include <setjmp.h>
50 
51 #include "iperf.h"
52 #include "iperf_api.h"
53 #include "iperf_udp.h"
54 #include "iperf_tcp.h"
55 #include "iperf_util.h"
56 #include "timer.h"
57 #include "iperf_time.h"
58 #include "net.h"
59 #include "units.h"
60 #include "iperf_util.h"
61 #include "iperf_locale.h"
62 
63 #if defined(HAVE_TCP_CONGESTION)
64 #if !defined(TCP_CA_NAME_MAX)
65 #define TCP_CA_NAME_MAX 16
66 #endif /* TCP_CA_NAME_MAX */
67 #endif /* HAVE_TCP_CONGESTION */
68 
69 int
70 iperf_server_listen(struct iperf_test *test)
71 {
72     retry:
73     if((test->listener = netannounce(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, test->server_port)) < 0) {
74 	if (errno == EAFNOSUPPORT && (test->settings->domain == AF_INET6 || test->settings->domain == AF_UNSPEC)) {
75 	    /* If we get "Address family not supported by protocol", that
76 	    ** probably means we were compiled with IPv6 but the running
77 	    ** kernel does not actually do IPv6.  This is not too unusual,
78 	    ** v6 support is and perhaps always will be spotty.
79 	    */
80 	    warning("this system does not seem to support IPv6 - trying IPv4");
81 	    test->settings->domain = AF_INET;
82 	    goto retry;
83 	} else {
84 	    i_errno = IELISTEN;
85 	    return -1;
86 	}
87     }
88 
89     if (!test->json_output) {
90         if (test->server_last_run_rc != 2)
91             test->server_test_number +=1;
92         if (test->debug || test->server_last_run_rc != 2) {
93 	    iperf_printf(test, "-----------------------------------------------------------\n");
94 	    iperf_printf(test, "Server listening on %d (test #%d)\n", test->server_port, test->server_test_number);
95 	    iperf_printf(test, "-----------------------------------------------------------\n");
96 	    if (test->forceflush)
97 	        iflush(test);
98         }
99     }
100 
101     FD_ZERO(&test->read_set);
102     FD_ZERO(&test->write_set);
103     FD_SET(test->listener, &test->read_set);
104     if (test->listener > test->max_fd) test->max_fd = test->listener;
105 
106     return 0;
107 }
108 
109 int
110 iperf_accept(struct iperf_test *test)
111 {
112     int s;
113     signed char rbuf = ACCESS_DENIED;
114     socklen_t len;
115     struct sockaddr_storage addr;
116 
117     len = sizeof(addr);
118     if ((s = accept(test->listener, (struct sockaddr *) &addr, &len)) < 0) {
119         i_errno = IEACCEPT;
120         return -1;
121     }
122 
123     if (test->ctrl_sck == -1) {
124         /* Server free, accept new client */
125         test->ctrl_sck = s;
126         // set TCP_NODELAY for lower latency on control messages
127         int flag = 1;
128         if (setsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int))) {
129             i_errno = IESETNODELAY;
130             return -1;
131         }
132 
133         if (Nread(test->ctrl_sck, test->cookie, COOKIE_SIZE, Ptcp) < 0) {
134             i_errno = IERECVCOOKIE;
135             return -1;
136         }
137 	FD_SET(test->ctrl_sck, &test->read_set);
138 	if (test->ctrl_sck > test->max_fd) test->max_fd = test->ctrl_sck;
139 
140 	if (iperf_set_send_state(test, PARAM_EXCHANGE) != 0)
141             return -1;
142         if (iperf_exchange_parameters(test) < 0)
143             return -1;
144 	if (test->server_affinity != -1)
145 	    if (iperf_setaffinity(test, test->server_affinity) != 0)
146 		return -1;
147         if (test->on_connect)
148             test->on_connect(test);
149     } else {
150 	/*
151 	 * Don't try to read from the socket.  It could block an ongoing test.
152 	 * Just send ACCESS_DENIED.
153          * Also, if sending failed, don't return an error, as the request is not related
154          * to the ongoing test, and returning an error will terminate the test.
155 	 */
156         if (Nwrite(s, (char*) &rbuf, sizeof(rbuf), Ptcp) < 0) {
157             if (test->debug)
158                 printf("failed to send ACCESS_DENIED to an unsolicited connection request during active test\n");
159         } else {
160             if (test->debug)
161                 printf("successfully sent ACCESS_DENIED to an unsolicited connection request during active test\n");
162         }
163         close(s);
164     }
165 
166     return 0;
167 }
168 
169 
170 /**************************************************************************/
171 int
172 iperf_handle_message_server(struct iperf_test *test)
173 {
174     int rval;
175     struct iperf_stream *sp;
176 
177     // XXX: Need to rethink how this behaves to fit API
178     if ((rval = Nread(test->ctrl_sck, (char*) &test->state, sizeof(signed char), Ptcp)) <= 0) {
179         if (rval == 0) {
180 	    iperf_err(test, "the client has unexpectedly closed the connection");
181             i_errno = IECTRLCLOSE;
182             test->state = IPERF_DONE;
183             return 0;
184         } else {
185             i_errno = IERECVMESSAGE;
186             return -1;
187         }
188     }
189 
190     switch(test->state) {
191         case TEST_START:
192             break;
193         case TEST_END:
194 	    test->done = 1;
195             cpu_util(test->cpu_util);
196             test->stats_callback(test);
197             SLIST_FOREACH(sp, &test->streams, streams) {
198                 FD_CLR(sp->socket, &test->read_set);
199                 FD_CLR(sp->socket, &test->write_set);
200                 close(sp->socket);
201             }
202             test->reporter_callback(test);
203 	    if (iperf_set_send_state(test, EXCHANGE_RESULTS) != 0)
204                 return -1;
205             if (iperf_exchange_results(test) < 0)
206                 return -1;
207 	    if (iperf_set_send_state(test, DISPLAY_RESULTS) != 0)
208                 return -1;
209             if (test->on_test_finish)
210                 test->on_test_finish(test);
211             break;
212         case IPERF_DONE:
213             break;
214         case CLIENT_TERMINATE:
215             i_errno = IECLIENTTERM;
216 
217 	    // Temporarily be in DISPLAY_RESULTS phase so we can get
218 	    // ending summary statistics.
219 	    signed char oldstate = test->state;
220 	    cpu_util(test->cpu_util);
221 	    test->state = DISPLAY_RESULTS;
222 	    test->reporter_callback(test);
223 	    test->state = oldstate;
224 
225             // XXX: Remove this line below!
226 	    iperf_err(test, "the client has terminated");
227             SLIST_FOREACH(sp, &test->streams, streams) {
228                 FD_CLR(sp->socket, &test->read_set);
229                 FD_CLR(sp->socket, &test->write_set);
230                 close(sp->socket);
231             }
232             test->state = IPERF_DONE;
233             break;
234         default:
235             i_errno = IEMESSAGE;
236             return -1;
237     }
238 
239     return 0;
240 }
241 
242 static void
243 server_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
244 {
245     struct iperf_test *test = client_data.p;
246     struct iperf_stream *sp;
247 
248     test->timer = NULL;
249     if (test->done)
250         return;
251     test->done = 1;
252     /* Free streams */
253     while (!SLIST_EMPTY(&test->streams)) {
254         sp = SLIST_FIRST(&test->streams);
255         SLIST_REMOVE_HEAD(&test->streams, streams);
256         close(sp->socket);
257         iperf_free_stream(sp);
258     }
259     close(test->ctrl_sck);
260 }
261 
262 static void
263 server_stats_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
264 {
265     struct iperf_test *test = client_data.p;
266 
267     if (test->done)
268         return;
269     if (test->stats_callback)
270 	test->stats_callback(test);
271 }
272 
273 static void
274 server_reporter_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
275 {
276     struct iperf_test *test = client_data.p;
277 
278     if (test->done)
279         return;
280     if (test->reporter_callback)
281 	test->reporter_callback(test);
282 }
283 
284 static int
285 create_server_timers(struct iperf_test * test)
286 {
287     struct iperf_time now;
288     TimerClientData cd;
289     int max_rtt = 4; /* seconds */
290     int state_transitions = 10; /* number of state transitions in iperf3 */
291     int grace_period = max_rtt * state_transitions;
292 
293     if (iperf_time_now(&now) < 0) {
294 	i_errno = IEINITTEST;
295 	return -1;
296     }
297     cd.p = test;
298     test->timer = test->stats_timer = test->reporter_timer = NULL;
299     if (test->duration != 0 ) {
300         test->done = 0;
301         test->timer = tmr_create(&now, server_timer_proc, cd, (test->duration + test->omit + grace_period) * SEC_TO_US, 0);
302         if (test->timer == NULL) {
303             i_errno = IEINITTEST;
304             return -1;
305         }
306     }
307 
308     test->stats_timer = test->reporter_timer = NULL;
309     if (test->stats_interval != 0) {
310         test->stats_timer = tmr_create(&now, server_stats_timer_proc, cd, test->stats_interval * SEC_TO_US, 1);
311         if (test->stats_timer == NULL) {
312             i_errno = IEINITTEST;
313             return -1;
314 	}
315     }
316     if (test->reporter_interval != 0) {
317         test->reporter_timer = tmr_create(&now, server_reporter_timer_proc, cd, test->reporter_interval * SEC_TO_US, 1);
318         if (test->reporter_timer == NULL) {
319             i_errno = IEINITTEST;
320             return -1;
321 	}
322     }
323     return 0;
324 }
325 
326 static void
327 server_omit_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
328 {
329     struct iperf_test *test = client_data.p;
330 
331     test->omit_timer = NULL;
332     test->omitting = 0;
333     iperf_reset_stats(test);
334     if (test->verbose && !test->json_output && test->reporter_interval == 0)
335 	iperf_printf(test, "%s", report_omit_done);
336 
337     /* Reset the timers. */
338     if (test->stats_timer != NULL)
339 	tmr_reset(nowP, test->stats_timer);
340     if (test->reporter_timer != NULL)
341 	tmr_reset(nowP, test->reporter_timer);
342 }
343 
344 static int
345 create_server_omit_timer(struct iperf_test * test)
346 {
347     struct iperf_time now;
348     TimerClientData cd;
349 
350     if (test->omit == 0) {
351 	test->omit_timer = NULL;
352 	test->omitting = 0;
353     } else {
354 	if (iperf_time_now(&now) < 0) {
355 	    i_errno = IEINITTEST;
356 	    return -1;
357 	}
358 	test->omitting = 1;
359 	cd.p = test;
360 	test->omit_timer = tmr_create(&now, server_omit_timer_proc, cd, test->omit * SEC_TO_US, 0);
361 	if (test->omit_timer == NULL) {
362 	    i_errno = IEINITTEST;
363 	    return -1;
364 	}
365     }
366 
367     return 0;
368 }
369 
370 static void
371 cleanup_server(struct iperf_test *test)
372 {
373     struct iperf_stream *sp;
374 
375     /* Close open streams */
376     SLIST_FOREACH(sp, &test->streams, streams) {
377 	FD_CLR(sp->socket, &test->read_set);
378 	FD_CLR(sp->socket, &test->write_set);
379 	close(sp->socket);
380     }
381 
382     /* Close open test sockets */
383     if (test->ctrl_sck) {
384 	close(test->ctrl_sck);
385     }
386     if (test->listener) {
387 	close(test->listener);
388     }
389     if (test->prot_listener > -1) {     // May remain open if create socket failed
390 	close(test->prot_listener);
391     }
392 
393     /* Cancel any remaining timers. */
394     if (test->stats_timer != NULL) {
395 	tmr_cancel(test->stats_timer);
396 	test->stats_timer = NULL;
397     }
398     if (test->reporter_timer != NULL) {
399 	tmr_cancel(test->reporter_timer);
400 	test->reporter_timer = NULL;
401     }
402     if (test->omit_timer != NULL) {
403 	tmr_cancel(test->omit_timer);
404 	test->omit_timer = NULL;
405     }
406     if (test->congestion_used != NULL) {
407         free(test->congestion_used);
408 	test->congestion_used = NULL;
409     }
410     if (test->timer != NULL) {
411         tmr_cancel(test->timer);
412         test->timer = NULL;
413     }
414 }
415 
416 
417 int
418 iperf_run_server(struct iperf_test *test)
419 {
420     int result, s;
421     int send_streams_accepted, rec_streams_accepted;
422     int streams_to_send = 0, streams_to_rec = 0;
423 #if defined(HAVE_TCP_CONGESTION)
424     int saved_errno;
425 #endif /* HAVE_TCP_CONGESTION */
426     fd_set read_set, write_set;
427     struct iperf_stream *sp;
428     struct iperf_time now;
429     struct iperf_time last_receive_time;
430     struct iperf_time diff_time;
431     struct timeval* timeout;
432     struct timeval used_timeout;
433     int flag;
434     int64_t t_usecs;
435     int64_t timeout_us;
436     int64_t rcv_timeout_us;
437 
438     if (test->logfile)
439         if (iperf_open_logfile(test) < 0)
440             return -1;
441 
442     if (test->affinity != -1)
443 	if (iperf_setaffinity(test, test->affinity) != 0)
444 	    return -2;
445 
446     if (test->json_output)
447 	if (iperf_json_start(test) < 0)
448 	    return -2;
449 
450     if (test->json_output) {
451 	cJSON_AddItemToObject(test->json_start, "version", cJSON_CreateString(version));
452 	cJSON_AddItemToObject(test->json_start, "system_info", cJSON_CreateString(get_system_info()));
453     } else if (test->verbose) {
454 	iperf_printf(test, "%s\n", version);
455 	iperf_printf(test, "%s", "");
456 	iperf_printf(test, "%s\n", get_system_info());
457 	iflush(test);
458     }
459 
460     // Open socket and listen
461     if (iperf_server_listen(test) < 0) {
462         return -2;
463     }
464 
465     iperf_time_now(&last_receive_time); // Initialize last time something was received
466 
467     test->state = IPERF_START;
468     send_streams_accepted = 0;
469     rec_streams_accepted = 0;
470     rcv_timeout_us = (test->settings->rcv_timeout.secs * SEC_TO_US) + test->settings->rcv_timeout.usecs;
471 
472     while (test->state != IPERF_DONE) {
473 
474         // Check if average transfer rate was exceeded (condition set in the callback routines)
475 	if (test->bitrate_limit_exceeded) {
476 	    cleanup_server(test);
477             i_errno = IETOTALRATE;
478             return -1;
479 	}
480 
481         memcpy(&read_set, &test->read_set, sizeof(fd_set));
482         memcpy(&write_set, &test->write_set, sizeof(fd_set));
483 
484 	iperf_time_now(&now);
485 	timeout = tmr_timeout(&now);
486 
487         // Ensure select() will timeout to allow handling error cases that require server restart
488         if (test->state == IPERF_START) {       // In idle mode server may need to restart
489             if (timeout == NULL && test->settings->idle_timeout > 0) {
490                 used_timeout.tv_sec = test->settings->idle_timeout;
491                 used_timeout.tv_usec = 0;
492                 timeout = &used_timeout;
493             }
494         } else if (test->mode != SENDER) {     // In non-reverse active mode server ensures data is received
495             timeout_us = -1;
496             if (timeout != NULL) {
497                 used_timeout.tv_sec = timeout->tv_sec;
498                 used_timeout.tv_usec = timeout->tv_usec;
499                 timeout_us = (timeout->tv_sec * SEC_TO_US) + timeout->tv_usec;
500             }
501             if (timeout_us < 0 || timeout_us > rcv_timeout_us) {
502                 used_timeout.tv_sec = test->settings->rcv_timeout.secs;
503                 used_timeout.tv_usec = test->settings->rcv_timeout.usecs;
504             }
505             timeout = &used_timeout;
506         }
507 
508         result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout);
509         if (result < 0 && errno != EINTR) {
510             cleanup_server(test);
511             i_errno = IESELECT;
512             return -1;
513         } else if (result == 0) {
514             // If nothing was received during the specified time (per state)
515             // then probably something got stack either at the client, server or network,
516             // and Test should be forced to end.
517             iperf_time_now(&now);
518             t_usecs = 0;
519             if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) {
520                 t_usecs = iperf_time_in_usecs(&diff_time);
521                 if (test->state == IPERF_START) {
522                     if (test->settings->idle_timeout > 0 && t_usecs >= test->settings->idle_timeout * SEC_TO_US) {
523                         test->server_forced_idle_restarts_count += 1;
524                         if (test->debug)
525                             printf("Server restart (#%d) in idle state as no connection request was received for %d sec\n",
526                                 test->server_forced_idle_restarts_count, test->settings->idle_timeout);
527                         cleanup_server(test);
528 			if ( iperf_get_test_one_off(test) ) {
529 			  if (test->debug)
530                             printf("No connection request was received for %d sec in one-off mode; exiting.\n",
531 				   test->settings->idle_timeout);
532 			  exit(0);
533 			}
534 
535                         return 2;
536                     }
537                 }
538                 else if (test->mode != SENDER && t_usecs > rcv_timeout_us) {
539                     test->server_forced_no_msg_restarts_count += 1;
540                     i_errno = IENOMSG;
541                     if (iperf_get_verbose(test))
542                         iperf_err(test, "Server restart (#%d) during active test due to idle data for receiving data",
543                                   test->server_forced_no_msg_restarts_count);
544                     cleanup_server(test);
545                     return -1;
546                 }
547 
548             }
549         }
550 
551 	if (result > 0) {
552             iperf_time_now(&last_receive_time);
553             if (FD_ISSET(test->listener, &read_set)) {
554                 if (test->state != CREATE_STREAMS) {
555                     if (iperf_accept(test) < 0) {
556 			cleanup_server(test);
557                         return -1;
558                     }
559                     FD_CLR(test->listener, &read_set);
560 
561                     // Set streams number
562                     if (test->mode == BIDIRECTIONAL) {
563                         streams_to_send = test->num_streams;
564                         streams_to_rec = test->num_streams;
565                     } else if (test->mode == RECEIVER) {
566                         streams_to_rec = test->num_streams;
567                         streams_to_send = 0;
568                     } else {
569                         streams_to_send = test->num_streams;
570                         streams_to_rec = 0;
571                     }
572                 }
573             }
574             if (FD_ISSET(test->ctrl_sck, &read_set)) {
575                 if (iperf_handle_message_server(test) < 0) {
576 		    cleanup_server(test);
577                     return -1;
578 		}
579                 FD_CLR(test->ctrl_sck, &read_set);
580             }
581 
582             if (test->state == CREATE_STREAMS) {
583                 if (FD_ISSET(test->prot_listener, &read_set)) {
584 
585                     if ((s = test->protocol->accept(test)) < 0) {
586 			cleanup_server(test);
587                         return -1;
588 		    }
589 
590                     if (!is_closed(s)) {
591 
592 #if defined(HAVE_TCP_CONGESTION)
593 		    if (test->protocol->id == Ptcp) {
594 			if (test->congestion) {
595 			    if (setsockopt(s, IPPROTO_TCP, TCP_CONGESTION, test->congestion, strlen(test->congestion)) < 0) {
596 				/*
597 				 * ENOENT means we tried to set the
598 				 * congestion algorithm but the algorithm
599 				 * specified doesn't exist.  This can happen
600 				 * if the client and server have different
601 				 * congestion algorithms available.  In this
602 				 * case, print a warning, but otherwise
603 				 * continue.
604 				 */
605 				if (errno == ENOENT) {
606 				    warning("TCP congestion control algorithm not supported");
607 				}
608 				else {
609 				    saved_errno = errno;
610 				    close(s);
611 				    cleanup_server(test);
612 				    errno = saved_errno;
613 				    i_errno = IESETCONGESTION;
614 				    return -1;
615 				}
616 			    }
617 			}
618 			{
619 			    socklen_t len = TCP_CA_NAME_MAX;
620 			    char ca[TCP_CA_NAME_MAX + 1];
621                             int rc;
622 			    rc = getsockopt(s, IPPROTO_TCP, TCP_CONGESTION, ca, &len);
623                             if (rc < 0 && test->congestion) {
624 				saved_errno = errno;
625 				close(s);
626 				cleanup_server(test);
627 				errno = saved_errno;
628 				i_errno = IESETCONGESTION;
629 				return -1;
630 			    }
631                             /*
632                              * If not the first connection, discard prior
633                              * congestion algorithm name so we don't leak
634                              * duplicated strings.  We probably don't need
635                              * the old string anyway.
636                              */
637                             if (test->congestion_used != NULL) {
638                                 free(test->congestion_used);
639                             }
640                             // Set actual used congestion alg, or set to unknown if could not get it
641                             if (rc < 0)
642                                 test->congestion_used = strdup("unknown");
643                             else
644                                 test->congestion_used = strdup(ca);
645 			    if (test->debug) {
646 				printf("Congestion algorithm is %s\n", test->congestion_used);
647 			    }
648 			}
649 		    }
650 #endif /* HAVE_TCP_CONGESTION */
651 
652                         if (rec_streams_accepted != streams_to_rec) {
653                             flag = 0;
654                             ++rec_streams_accepted;
655                         } else if (send_streams_accepted != streams_to_send) {
656                             flag = 1;
657                             ++send_streams_accepted;
658                         }
659 
660                         if (flag != -1) {
661                             sp = iperf_new_stream(test, s, flag);
662                             if (!sp) {
663                                 cleanup_server(test);
664                                 return -1;
665                             }
666 
667                             if (sp->sender)
668                                 FD_SET(s, &test->write_set);
669                             else
670                                 FD_SET(s, &test->read_set);
671 
672                             if (s > test->max_fd) test->max_fd = s;
673 
674                             /*
675                              * If the protocol isn't UDP, or even if it is but
676                              * we're the receiver, set nonblocking sockets.
677                              * We need this to allow a server receiver to
678                              * maintain interactivity with the control channel.
679                              */
680                             if (test->protocol->id != Pudp ||
681                                 !sp->sender) {
682                                 setnonblocking(s, 1);
683                             }
684 
685                             if (test->on_new_stream)
686                                 test->on_new_stream(sp);
687 
688                             flag = -1;
689                         }
690                     }
691                     FD_CLR(test->prot_listener, &read_set);
692                 }
693 
694 
695                 if (rec_streams_accepted == streams_to_rec && send_streams_accepted == streams_to_send) {
696                     if (test->protocol->id != Ptcp) {
697                         FD_CLR(test->prot_listener, &test->read_set);
698                         close(test->prot_listener);
699                     } else {
700                         if (test->no_delay || test->settings->mss || test->settings->socket_bufsize) {
701                             FD_CLR(test->listener, &test->read_set);
702                             close(test->listener);
703 			    test->listener = 0;
704                             if ((s = netannounce(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, test->server_port)) < 0) {
705 				cleanup_server(test);
706                                 i_errno = IELISTEN;
707                                 return -1;
708                             }
709                             test->listener = s;
710                             FD_SET(test->listener, &test->read_set);
711 			    if (test->listener > test->max_fd) test->max_fd = test->listener;
712                         }
713                     }
714                     test->prot_listener = -1;
715 
716 		    /* Ensure that total requested data rate is not above limit */
717 		    iperf_size_t total_requested_rate = test->num_streams * test->settings->rate * (test->mode == BIDIRECTIONAL? 2 : 1);
718 		    if (test->settings->bitrate_limit > 0 && total_requested_rate > test->settings->bitrate_limit) {
719                         if (iperf_get_verbose(test))
720                             iperf_err(test, "Client total requested throughput rate of %" PRIu64 " bps exceeded %" PRIu64 " bps limit",
721                                       total_requested_rate, test->settings->bitrate_limit);
722 			cleanup_server(test);
723 			i_errno = IETOTALRATE;
724 			return -1;
725 		    }
726 
727 		    // Begin calculating CPU utilization
728 		    cpu_util(NULL);
729 
730 		    if (iperf_set_send_state(test, TEST_START) != 0) {
731 			cleanup_server(test);
732                         return -1;
733 		    }
734                     if (iperf_init_test(test) < 0) {
735 			cleanup_server(test);
736                         return -1;
737 		    }
738 		    if (create_server_timers(test) < 0) {
739 			cleanup_server(test);
740                         return -1;
741 		    }
742 		    if (create_server_omit_timer(test) < 0) {
743 			cleanup_server(test);
744                         return -1;
745 		    }
746 		    if (test->mode != RECEIVER)
747 			if (iperf_create_send_timers(test) < 0) {
748 			    cleanup_server(test);
749 			    return -1;
750 			}
751 		    if (iperf_set_send_state(test, TEST_RUNNING) != 0) {
752 			cleanup_server(test);
753                         return -1;
754 		    }
755                 }
756             }
757 
758             if (test->state == TEST_RUNNING) {
759                 if (test->mode == BIDIRECTIONAL) {
760                     if (iperf_recv(test, &read_set) < 0) {
761                         cleanup_server(test);
762                         return -1;
763                     }
764                     if (iperf_send(test, &write_set) < 0) {
765                         cleanup_server(test);
766                         return -1;
767                     }
768                 } else if (test->mode == SENDER) {
769                     // Reverse mode. Server sends.
770                     if (iperf_send(test, &write_set) < 0) {
771 			cleanup_server(test);
772                         return -1;
773 		    }
774                 } else {
775                     // Regular mode. Server receives.
776                     if (iperf_recv(test, &read_set) < 0) {
777 			cleanup_server(test);
778                         return -1;
779 		    }
780                 }
781 	    }
782         }
783 
784 	if (result == 0 ||
785 	    (timeout != NULL && timeout->tv_sec == 0 && timeout->tv_usec == 0)) {
786 	    /* Run the timers. */
787 	    iperf_time_now(&now);
788 	    tmr_run(&now);
789 	}
790     }
791 
792     cleanup_server(test);
793 
794     if (test->json_output) {
795 	if (iperf_json_finish(test) < 0)
796 	    return -1;
797     }
798 
799     iflush(test);
800 
801     if (test->server_affinity != -1)
802 	if (iperf_clearaffinity(test) != 0)
803 	    return -1;
804 
805     return 0;
806 }
807