xref: /iperf/src/iperf_server_api.c (revision 85b25c2e)
1 /*
2  * iperf, Copyright (c) 2014-2022 The Regents of the University of
3  * California, through Lawrence Berkeley National Laboratory (subject
4  * to receipt of any required approvals from the U.S. Dept. of
5  * Energy).  All rights reserved.
6  *
7  * If you have questions about your rights to use or distribute this
8  * software, please contact Berkeley Lab's Technology Transfer
9  * Department at [email protected].
10  *
11  * NOTICE.  This software is owned by the U.S. Department of Energy.
12  * As such, the U.S. Government has been granted for itself and others
13  * acting on its behalf a paid-up, nonexclusive, irrevocable,
14  * worldwide license in the Software to reproduce, prepare derivative
15  * works, and perform publicly and display publicly.  Beginning five
16  * (5) years after the date permission to assert copyright is obtained
17  * from the U.S. Department of Energy, and subject to any subsequent
18  * five (5) year renewals, the U.S. Government is granted for itself
19  * and others acting on its behalf a paid-up, nonexclusive,
20  * irrevocable, worldwide license in the Software to reproduce,
21  * prepare derivative works, distribute copies to the public, perform
22  * publicly and display publicly, and to permit others to do so.
23  *
24  * This code is distributed under a BSD style license, see the LICENSE
25  * file for complete information.
26  */
27 /* iperf_server_api.c: Functions to be used by an iperf server
28 */
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <getopt.h>
34 #include <errno.h>
35 #include <unistd.h>
36 #include <assert.h>
37 #include <fcntl.h>
38 #include <sys/socket.h>
39 #include <sys/types.h>
40 #include <netinet/in.h>
41 #include <arpa/inet.h>
42 #include <netdb.h>
43 #ifdef HAVE_STDINT_H
44 #include <stdint.h>
45 #endif
46 #include <sys/time.h>
47 #include <sys/resource.h>
48 #include <sched.h>
49 #include <setjmp.h>
50 
51 #include "iperf.h"
52 #include "iperf_api.h"
53 #include "iperf_udp.h"
54 #include "iperf_tcp.h"
55 #include "iperf_util.h"
56 #include "timer.h"
57 #include "iperf_time.h"
58 #include "net.h"
59 #include "units.h"
60 #include "iperf_util.h"
61 #include "iperf_locale.h"
62 
63 #if defined(HAVE_TCP_CONGESTION)
64 #if !defined(TCP_CA_NAME_MAX)
65 #define TCP_CA_NAME_MAX 16
66 #endif /* TCP_CA_NAME_MAX */
67 #endif /* HAVE_TCP_CONGESTION */
68 
69 int
70 iperf_server_listen(struct iperf_test *test)
71 {
72     retry:
73     if((test->listener = netannounce(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, test->server_port)) < 0) {
74 	if (errno == EAFNOSUPPORT && (test->settings->domain == AF_INET6 || test->settings->domain == AF_UNSPEC)) {
75 	    /* If we get "Address family not supported by protocol", that
76 	    ** probably means we were compiled with IPv6 but the running
77 	    ** kernel does not actually do IPv6.  This is not too unusual,
78 	    ** v6 support is and perhaps always will be spotty.
79 	    */
80 	    warning("this system does not seem to support IPv6 - trying IPv4");
81 	    test->settings->domain = AF_INET;
82 	    goto retry;
83 	} else {
84 	    i_errno = IELISTEN;
85 	    return -1;
86 	}
87     }
88 
89     if (!test->json_output) {
90         if (test->server_last_run_rc != 2)
91             test->server_test_number +=1;
92         if (test->debug || test->server_last_run_rc != 2) {
93 	    iperf_printf(test, "-----------------------------------------------------------\n");
94 	    iperf_printf(test, "Server listening on %d (test #%d)\n", test->server_port, test->server_test_number);
95 	    iperf_printf(test, "-----------------------------------------------------------\n");
96 	    if (test->forceflush)
97 	        iflush(test);
98         }
99     }
100 
101     FD_ZERO(&test->read_set);
102     FD_ZERO(&test->write_set);
103     FD_SET(test->listener, &test->read_set);
104     if (test->listener > test->max_fd) test->max_fd = test->listener;
105 
106     return 0;
107 }
108 
109 int
110 iperf_accept(struct iperf_test *test)
111 {
112     int s;
113     signed char rbuf = ACCESS_DENIED;
114     socklen_t len;
115     struct sockaddr_storage addr;
116 
117     len = sizeof(addr);
118     if ((s = accept(test->listener, (struct sockaddr *) &addr, &len)) < 0) {
119         i_errno = IEACCEPT;
120         return -1;
121     }
122 
123     if (test->ctrl_sck == -1) {
124         /* Server free, accept new client */
125         test->ctrl_sck = s;
126         // set TCP_NODELAY for lower latency on control messages
127         int flag = 1;
128         if (setsockopt(test->ctrl_sck, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int))) {
129             i_errno = IESETNODELAY;
130             return -1;
131         }
132 
133 #if defined(HAVE_TCP_USER_TIMEOUT)
134         int opt;
135         if ((opt = test->settings->snd_timeout)) {
136             if (setsockopt(s, IPPROTO_TCP, TCP_USER_TIMEOUT, &opt, sizeof(opt)) < 0) {
137                 i_errno = IESETUSERTIMEOUT;
138                 return -1;
139             }
140         }
141 #endif /* HAVE_TCP_USER_TIMEOUT */
142 
143         if (Nread(test->ctrl_sck, test->cookie, COOKIE_SIZE, Ptcp) < 0) {
144             i_errno = IERECVCOOKIE;
145             return -1;
146         }
147 	FD_SET(test->ctrl_sck, &test->read_set);
148 	if (test->ctrl_sck > test->max_fd) test->max_fd = test->ctrl_sck;
149 
150 	if (iperf_set_send_state(test, PARAM_EXCHANGE) != 0)
151             return -1;
152         if (iperf_exchange_parameters(test) < 0)
153             return -1;
154 	if (test->server_affinity != -1)
155 	    if (iperf_setaffinity(test, test->server_affinity) != 0)
156 		return -1;
157         if (test->on_connect)
158             test->on_connect(test);
159     } else {
160 	/*
161 	 * Don't try to read from the socket.  It could block an ongoing test.
162 	 * Just send ACCESS_DENIED.
163          * Also, if sending failed, don't return an error, as the request is not related
164          * to the ongoing test, and returning an error will terminate the test.
165 	 */
166         if (Nwrite(s, (char*) &rbuf, sizeof(rbuf), Ptcp) < 0) {
167             if (test->debug)
168                 printf("failed to send ACCESS_DENIED to an unsolicited connection request during active test\n");
169         } else {
170             if (test->debug)
171                 printf("successfully sent ACCESS_DENIED to an unsolicited connection request during active test\n");
172         }
173         close(s);
174     }
175 
176     return 0;
177 }
178 
179 
180 /**************************************************************************/
181 int
182 iperf_handle_message_server(struct iperf_test *test)
183 {
184     int rval;
185     struct iperf_stream *sp;
186 
187     // XXX: Need to rethink how this behaves to fit API
188     if ((rval = Nread(test->ctrl_sck, (char*) &test->state, sizeof(signed char), Ptcp)) <= 0) {
189         if (rval == 0) {
190 	    iperf_err(test, "the client has unexpectedly closed the connection");
191             i_errno = IECTRLCLOSE;
192             test->state = IPERF_DONE;
193             return 0;
194         } else {
195             i_errno = IERECVMESSAGE;
196             return -1;
197         }
198     }
199 
200     switch(test->state) {
201         case TEST_START:
202             break;
203         case TEST_END:
204 	    test->done = 1;
205             cpu_util(test->cpu_util);
206             test->stats_callback(test);
207             SLIST_FOREACH(sp, &test->streams, streams) {
208                 FD_CLR(sp->socket, &test->read_set);
209                 FD_CLR(sp->socket, &test->write_set);
210                 close(sp->socket);
211             }
212             test->reporter_callback(test);
213 	    if (iperf_set_send_state(test, EXCHANGE_RESULTS) != 0)
214                 return -1;
215             if (iperf_exchange_results(test) < 0)
216                 return -1;
217 	    if (iperf_set_send_state(test, DISPLAY_RESULTS) != 0)
218                 return -1;
219             if (test->on_test_finish)
220                 test->on_test_finish(test);
221             break;
222         case IPERF_DONE:
223             break;
224         case CLIENT_TERMINATE:
225             i_errno = IECLIENTTERM;
226 
227 	    // Temporarily be in DISPLAY_RESULTS phase so we can get
228 	    // ending summary statistics.
229 	    signed char oldstate = test->state;
230 	    cpu_util(test->cpu_util);
231 	    test->state = DISPLAY_RESULTS;
232 	    test->reporter_callback(test);
233 	    test->state = oldstate;
234 
235             // XXX: Remove this line below!
236 	    iperf_err(test, "the client has terminated");
237             SLIST_FOREACH(sp, &test->streams, streams) {
238                 FD_CLR(sp->socket, &test->read_set);
239                 FD_CLR(sp->socket, &test->write_set);
240                 close(sp->socket);
241             }
242             test->state = IPERF_DONE;
243             break;
244         default:
245             i_errno = IEMESSAGE;
246             return -1;
247     }
248 
249     return 0;
250 }
251 
252 static void
253 server_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
254 {
255     struct iperf_test *test = client_data.p;
256     struct iperf_stream *sp;
257 
258     test->timer = NULL;
259     if (test->done)
260         return;
261     test->done = 1;
262     /* Free streams */
263     while (!SLIST_EMPTY(&test->streams)) {
264         sp = SLIST_FIRST(&test->streams);
265         SLIST_REMOVE_HEAD(&test->streams, streams);
266         close(sp->socket);
267         iperf_free_stream(sp);
268     }
269     close(test->ctrl_sck);
270 }
271 
272 static void
273 server_stats_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
274 {
275     struct iperf_test *test = client_data.p;
276 
277     if (test->done)
278         return;
279     if (test->stats_callback)
280 	test->stats_callback(test);
281 }
282 
283 static void
284 server_reporter_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
285 {
286     struct iperf_test *test = client_data.p;
287 
288     if (test->done)
289         return;
290     if (test->reporter_callback)
291 	test->reporter_callback(test);
292 }
293 
294 static int
295 create_server_timers(struct iperf_test * test)
296 {
297     struct iperf_time now;
298     TimerClientData cd;
299     int max_rtt = 4; /* seconds */
300     int state_transitions = 10; /* number of state transitions in iperf3 */
301     int grace_period = max_rtt * state_transitions;
302 
303     if (iperf_time_now(&now) < 0) {
304 	i_errno = IEINITTEST;
305 	return -1;
306     }
307     cd.p = test;
308     test->timer = test->stats_timer = test->reporter_timer = NULL;
309     if (test->duration != 0 ) {
310         test->done = 0;
311         test->timer = tmr_create(&now, server_timer_proc, cd, (test->duration + test->omit + grace_period) * SEC_TO_US, 0);
312         if (test->timer == NULL) {
313             i_errno = IEINITTEST;
314             return -1;
315         }
316     }
317 
318     test->stats_timer = test->reporter_timer = NULL;
319     if (test->stats_interval != 0) {
320         test->stats_timer = tmr_create(&now, server_stats_timer_proc, cd, test->stats_interval * SEC_TO_US, 1);
321         if (test->stats_timer == NULL) {
322             i_errno = IEINITTEST;
323             return -1;
324 	}
325     }
326     if (test->reporter_interval != 0) {
327         test->reporter_timer = tmr_create(&now, server_reporter_timer_proc, cd, test->reporter_interval * SEC_TO_US, 1);
328         if (test->reporter_timer == NULL) {
329             i_errno = IEINITTEST;
330             return -1;
331 	}
332     }
333     return 0;
334 }
335 
336 static void
337 server_omit_timer_proc(TimerClientData client_data, struct iperf_time *nowP)
338 {
339     struct iperf_test *test = client_data.p;
340 
341     test->omit_timer = NULL;
342     test->omitting = 0;
343     iperf_reset_stats(test);
344     if (test->verbose && !test->json_output && test->reporter_interval == 0)
345 	iperf_printf(test, "%s", report_omit_done);
346 
347     /* Reset the timers. */
348     if (test->stats_timer != NULL)
349 	tmr_reset(nowP, test->stats_timer);
350     if (test->reporter_timer != NULL)
351 	tmr_reset(nowP, test->reporter_timer);
352 }
353 
354 static int
355 create_server_omit_timer(struct iperf_test * test)
356 {
357     struct iperf_time now;
358     TimerClientData cd;
359 
360     if (test->omit == 0) {
361 	test->omit_timer = NULL;
362 	test->omitting = 0;
363     } else {
364 	if (iperf_time_now(&now) < 0) {
365 	    i_errno = IEINITTEST;
366 	    return -1;
367 	}
368 	test->omitting = 1;
369 	cd.p = test;
370 	test->omit_timer = tmr_create(&now, server_omit_timer_proc, cd, test->omit * SEC_TO_US, 0);
371 	if (test->omit_timer == NULL) {
372 	    i_errno = IEINITTEST;
373 	    return -1;
374 	}
375     }
376 
377     return 0;
378 }
379 
380 static void
381 cleanup_server(struct iperf_test *test)
382 {
383     struct iperf_stream *sp;
384 
385     /* Close open streams */
386     SLIST_FOREACH(sp, &test->streams, streams) {
387 	FD_CLR(sp->socket, &test->read_set);
388 	FD_CLR(sp->socket, &test->write_set);
389 	close(sp->socket);
390     }
391 
392     /* Close open test sockets */
393     if (test->ctrl_sck) {
394 	close(test->ctrl_sck);
395     }
396     if (test->listener) {
397 	close(test->listener);
398     }
399     if (test->prot_listener > -1) {     // May remain open if create socket failed
400 	close(test->prot_listener);
401     }
402 
403     /* Cancel any remaining timers. */
404     if (test->stats_timer != NULL) {
405 	tmr_cancel(test->stats_timer);
406 	test->stats_timer = NULL;
407     }
408     if (test->reporter_timer != NULL) {
409 	tmr_cancel(test->reporter_timer);
410 	test->reporter_timer = NULL;
411     }
412     if (test->omit_timer != NULL) {
413 	tmr_cancel(test->omit_timer);
414 	test->omit_timer = NULL;
415     }
416     if (test->congestion_used != NULL) {
417         free(test->congestion_used);
418 	test->congestion_used = NULL;
419     }
420     if (test->timer != NULL) {
421         tmr_cancel(test->timer);
422         test->timer = NULL;
423     }
424 }
425 
426 
427 int
428 iperf_run_server(struct iperf_test *test)
429 {
430     int result, s;
431     int send_streams_accepted, rec_streams_accepted;
432     int streams_to_send = 0, streams_to_rec = 0;
433 #if defined(HAVE_TCP_CONGESTION)
434     int saved_errno;
435 #endif /* HAVE_TCP_CONGESTION */
436     fd_set read_set, write_set;
437     struct iperf_stream *sp;
438     struct iperf_time now;
439     struct iperf_time last_receive_time;
440     struct iperf_time diff_time;
441     struct timeval* timeout;
442     struct timeval used_timeout;
443     int flag;
444     int64_t t_usecs;
445     int64_t timeout_us;
446     int64_t rcv_timeout_us;
447 
448     if (test->logfile)
449         if (iperf_open_logfile(test) < 0)
450             return -2;
451 
452     if (test->affinity != -1)
453 	if (iperf_setaffinity(test, test->affinity) != 0)
454 	    return -2;
455 
456     if (test->json_output)
457 	if (iperf_json_start(test) < 0)
458 	    return -2;
459 
460     if (test->json_output) {
461 	cJSON_AddItemToObject(test->json_start, "version", cJSON_CreateString(version));
462 	cJSON_AddItemToObject(test->json_start, "system_info", cJSON_CreateString(get_system_info()));
463     } else if (test->verbose) {
464 	iperf_printf(test, "%s\n", version);
465 	iperf_printf(test, "%s", "");
466 	iperf_printf(test, "%s\n", get_system_info());
467 	iflush(test);
468     }
469 
470     // Open socket and listen
471     if (iperf_server_listen(test) < 0) {
472         return -2;
473     }
474 
475     iperf_time_now(&last_receive_time); // Initialize last time something was received
476 
477     test->state = IPERF_START;
478     send_streams_accepted = 0;
479     rec_streams_accepted = 0;
480     rcv_timeout_us = (test->settings->rcv_timeout.secs * SEC_TO_US) + test->settings->rcv_timeout.usecs;
481 
482     while (test->state != IPERF_DONE) {
483 
484         // Check if average transfer rate was exceeded (condition set in the callback routines)
485 	if (test->bitrate_limit_exceeded) {
486 	    cleanup_server(test);
487             i_errno = IETOTALRATE;
488             return -1;
489 	}
490 
491         memcpy(&read_set, &test->read_set, sizeof(fd_set));
492         memcpy(&write_set, &test->write_set, sizeof(fd_set));
493 
494 	iperf_time_now(&now);
495 	timeout = tmr_timeout(&now);
496 
497         // Ensure select() will timeout to allow handling error cases that require server restart
498         if (test->state == IPERF_START) {       // In idle mode server may need to restart
499             if (timeout == NULL && test->settings->idle_timeout > 0) {
500                 used_timeout.tv_sec = test->settings->idle_timeout;
501                 used_timeout.tv_usec = 0;
502                 timeout = &used_timeout;
503             }
504         } else if (test->mode != SENDER) {     // In non-reverse active mode server ensures data is received
505             timeout_us = -1;
506             if (timeout != NULL) {
507                 used_timeout.tv_sec = timeout->tv_sec;
508                 used_timeout.tv_usec = timeout->tv_usec;
509                 timeout_us = (timeout->tv_sec * SEC_TO_US) + timeout->tv_usec;
510             }
511             if (timeout_us < 0 || timeout_us > rcv_timeout_us) {
512                 used_timeout.tv_sec = test->settings->rcv_timeout.secs;
513                 used_timeout.tv_usec = test->settings->rcv_timeout.usecs;
514             }
515             timeout = &used_timeout;
516         }
517 
518         result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout);
519         if (result < 0 && errno != EINTR) {
520             cleanup_server(test);
521             i_errno = IESELECT;
522             return -1;
523         } else if (result == 0) {
524             // If nothing was received during the specified time (per state)
525             // then probably something got stack either at the client, server or network,
526             // and Test should be forced to end.
527             iperf_time_now(&now);
528             t_usecs = 0;
529             if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) {
530                 t_usecs = iperf_time_in_usecs(&diff_time);
531                 if (test->state == IPERF_START) {
532                     if (test->settings->idle_timeout > 0 && t_usecs >= test->settings->idle_timeout * SEC_TO_US) {
533                         test->server_forced_idle_restarts_count += 1;
534                         if (test->debug)
535                             printf("Server restart (#%d) in idle state as no connection request was received for %d sec\n",
536                                 test->server_forced_idle_restarts_count, test->settings->idle_timeout);
537                         cleanup_server(test);
538 			if ( iperf_get_test_one_off(test) ) {
539 			  if (test->debug)
540                             printf("No connection request was received for %d sec in one-off mode; exiting.\n",
541 				   test->settings->idle_timeout);
542 			  exit(0);
543 			}
544 
545                         return 2;
546                     }
547                 }
548                 else if (test->mode != SENDER && t_usecs > rcv_timeout_us) {
549                     test->server_forced_no_msg_restarts_count += 1;
550                     i_errno = IENOMSG;
551                     if (iperf_get_verbose(test))
552                         iperf_err(test, "Server restart (#%d) during active test due to idle data for receiving data",
553                                   test->server_forced_no_msg_restarts_count);
554                     cleanup_server(test);
555                     return -1;
556                 }
557 
558             }
559         }
560 
561 	if (result > 0) {
562             iperf_time_now(&last_receive_time);
563             if (FD_ISSET(test->listener, &read_set)) {
564                 if (test->state != CREATE_STREAMS) {
565                     if (iperf_accept(test) < 0) {
566 			cleanup_server(test);
567                         return -1;
568                     }
569                     FD_CLR(test->listener, &read_set);
570 
571                     // Set streams number
572                     if (test->mode == BIDIRECTIONAL) {
573                         streams_to_send = test->num_streams;
574                         streams_to_rec = test->num_streams;
575                     } else if (test->mode == RECEIVER) {
576                         streams_to_rec = test->num_streams;
577                         streams_to_send = 0;
578                     } else {
579                         streams_to_send = test->num_streams;
580                         streams_to_rec = 0;
581                     }
582                 }
583             }
584             if (FD_ISSET(test->ctrl_sck, &read_set)) {
585                 if (iperf_handle_message_server(test) < 0) {
586 		    cleanup_server(test);
587                     return -1;
588 		}
589                 FD_CLR(test->ctrl_sck, &read_set);
590             }
591 
592             if (test->state == CREATE_STREAMS) {
593                 if (FD_ISSET(test->prot_listener, &read_set)) {
594 
595                     if ((s = test->protocol->accept(test)) < 0) {
596 			cleanup_server(test);
597                         return -1;
598 		    }
599 
600                     if (!is_closed(s)) {
601 
602 #if defined(HAVE_TCP_USER_TIMEOUT)
603 		    if (test->protocol->id == Ptcp) {
604                         int opt;
605                         if ((opt = test->settings->snd_timeout)) {
606                             if (setsockopt(s, IPPROTO_TCP, TCP_USER_TIMEOUT, &opt, sizeof(opt)) < 0) {
607                                 saved_errno = errno;
608                                 close(s);
609                                 cleanup_server(test);
610                                 errno = saved_errno;
611                                 i_errno = IESETUSERTIMEOUT;
612                                 return -1;
613                             }
614                         }
615                     }
616 #endif /* HAVE_TCP_USER_TIMEOUT */
617 
618 #if defined(HAVE_TCP_CONGESTION)
619 		    if (test->protocol->id == Ptcp) {
620 			if (test->congestion) {
621 			    if (setsockopt(s, IPPROTO_TCP, TCP_CONGESTION, test->congestion, strlen(test->congestion)) < 0) {
622 				/*
623 				 * ENOENT means we tried to set the
624 				 * congestion algorithm but the algorithm
625 				 * specified doesn't exist.  This can happen
626 				 * if the client and server have different
627 				 * congestion algorithms available.  In this
628 				 * case, print a warning, but otherwise
629 				 * continue.
630 				 */
631 				if (errno == ENOENT) {
632 				    warning("TCP congestion control algorithm not supported");
633 				}
634 				else {
635 				    saved_errno = errno;
636 				    close(s);
637 				    cleanup_server(test);
638 				    errno = saved_errno;
639 				    i_errno = IESETCONGESTION;
640 				    return -1;
641 				}
642 			    }
643 			}
644 			{
645 			    socklen_t len = TCP_CA_NAME_MAX;
646 			    char ca[TCP_CA_NAME_MAX + 1];
647                             int rc;
648 			    rc = getsockopt(s, IPPROTO_TCP, TCP_CONGESTION, ca, &len);
649                             if (rc < 0 && test->congestion) {
650 				saved_errno = errno;
651 				close(s);
652 				cleanup_server(test);
653 				errno = saved_errno;
654 				i_errno = IESETCONGESTION;
655 				return -1;
656 			    }
657                             /*
658                              * If not the first connection, discard prior
659                              * congestion algorithm name so we don't leak
660                              * duplicated strings.  We probably don't need
661                              * the old string anyway.
662                              */
663                             if (test->congestion_used != NULL) {
664                                 free(test->congestion_used);
665                             }
666                             // Set actual used congestion alg, or set to unknown if could not get it
667                             if (rc < 0)
668                                 test->congestion_used = strdup("unknown");
669                             else
670                                 test->congestion_used = strdup(ca);
671 			    if (test->debug) {
672 				printf("Congestion algorithm is %s\n", test->congestion_used);
673 			    }
674 			}
675 		    }
676 #endif /* HAVE_TCP_CONGESTION */
677 
678                         if (rec_streams_accepted != streams_to_rec) {
679                             flag = 0;
680                             ++rec_streams_accepted;
681                         } else if (send_streams_accepted != streams_to_send) {
682                             flag = 1;
683                             ++send_streams_accepted;
684                         }
685 
686                         if (flag != -1) {
687                             sp = iperf_new_stream(test, s, flag);
688                             if (!sp) {
689                                 cleanup_server(test);
690                                 return -1;
691                             }
692 
693                             if (sp->sender)
694                                 FD_SET(s, &test->write_set);
695                             else
696                                 FD_SET(s, &test->read_set);
697 
698                             if (s > test->max_fd) test->max_fd = s;
699 
700                             /*
701                              * If the protocol isn't UDP, or even if it is but
702                              * we're the receiver, set nonblocking sockets.
703                              * We need this to allow a server receiver to
704                              * maintain interactivity with the control channel.
705                              */
706                             if (test->protocol->id != Pudp ||
707                                 !sp->sender) {
708                                 setnonblocking(s, 1);
709                             }
710 
711                             if (test->on_new_stream)
712                                 test->on_new_stream(sp);
713 
714                             flag = -1;
715                         }
716                     }
717                     FD_CLR(test->prot_listener, &read_set);
718                 }
719 
720 
721                 if (rec_streams_accepted == streams_to_rec && send_streams_accepted == streams_to_send) {
722                     if (test->protocol->id != Ptcp) {
723                         FD_CLR(test->prot_listener, &test->read_set);
724                         close(test->prot_listener);
725                     } else {
726                         if (test->no_delay || test->settings->mss || test->settings->socket_bufsize) {
727                             FD_CLR(test->listener, &test->read_set);
728                             close(test->listener);
729 			    test->listener = 0;
730                             if ((s = netannounce(test->settings->domain, Ptcp, test->bind_address, test->bind_dev, test->server_port)) < 0) {
731 				cleanup_server(test);
732                                 i_errno = IELISTEN;
733                                 return -1;
734                             }
735                             test->listener = s;
736                             FD_SET(test->listener, &test->read_set);
737 			    if (test->listener > test->max_fd) test->max_fd = test->listener;
738                         }
739                     }
740                     test->prot_listener = -1;
741 
742 		    /* Ensure that total requested data rate is not above limit */
743 		    iperf_size_t total_requested_rate = test->num_streams * test->settings->rate * (test->mode == BIDIRECTIONAL? 2 : 1);
744 		    if (test->settings->bitrate_limit > 0 && total_requested_rate > test->settings->bitrate_limit) {
745                         if (iperf_get_verbose(test))
746                             iperf_err(test, "Client total requested throughput rate of %" PRIu64 " bps exceeded %" PRIu64 " bps limit",
747                                       total_requested_rate, test->settings->bitrate_limit);
748 			cleanup_server(test);
749 			i_errno = IETOTALRATE;
750 			return -1;
751 		    }
752 
753 		    // Begin calculating CPU utilization
754 		    cpu_util(NULL);
755 
756 		    if (iperf_set_send_state(test, TEST_START) != 0) {
757 			cleanup_server(test);
758                         return -1;
759 		    }
760                     if (iperf_init_test(test) < 0) {
761 			cleanup_server(test);
762                         return -1;
763 		    }
764 		    if (create_server_timers(test) < 0) {
765 			cleanup_server(test);
766                         return -1;
767 		    }
768 		    if (create_server_omit_timer(test) < 0) {
769 			cleanup_server(test);
770                         return -1;
771 		    }
772 		    if (test->mode != RECEIVER)
773 			if (iperf_create_send_timers(test) < 0) {
774 			    cleanup_server(test);
775 			    return -1;
776 			}
777 		    if (iperf_set_send_state(test, TEST_RUNNING) != 0) {
778 			cleanup_server(test);
779                         return -1;
780 		    }
781                 }
782             }
783 
784             if (test->state == TEST_RUNNING) {
785                 if (test->mode == BIDIRECTIONAL) {
786                     if (iperf_recv(test, &read_set) < 0) {
787                         cleanup_server(test);
788                         return -1;
789                     }
790                     if (iperf_send(test, &write_set) < 0) {
791                         cleanup_server(test);
792                         return -1;
793                     }
794                 } else if (test->mode == SENDER) {
795                     // Reverse mode. Server sends.
796                     if (iperf_send(test, &write_set) < 0) {
797 			cleanup_server(test);
798                         return -1;
799 		    }
800                 } else {
801                     // Regular mode. Server receives.
802                     if (iperf_recv(test, &read_set) < 0) {
803 			cleanup_server(test);
804                         return -1;
805 		    }
806                 }
807 	    }
808         }
809 
810 	if (result == 0 ||
811 	    (timeout != NULL && timeout->tv_sec == 0 && timeout->tv_usec == 0)) {
812 	    /* Run the timers. */
813 	    iperf_time_now(&now);
814 	    tmr_run(&now);
815 	}
816     }
817 
818     cleanup_server(test);
819 
820     if (test->json_output) {
821 	if (iperf_json_finish(test) < 0)
822 	    return -1;
823     }
824 
825     iflush(test);
826 
827     if (test->server_affinity != -1)
828 	if (iperf_clearaffinity(test) != 0)
829 	    return -1;
830 
831     return 0;
832 }
833