1 #include "first.h"
2
3 #include "network_write.h"
4
5 #include "base.h"
6 #include "ck.h"
7 #include "log.h"
8
9 #include <sys/types.h>
10 #include "sys-socket.h"
11
12 #include <errno.h>
13 #include <string.h>
14 #include <unistd.h>
15
16
17 /* on linux 2.4.x you get either sendfile or LFS */
18 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \
19 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \
20 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN
21 # ifdef NETWORK_WRITE_USE_SENDFILE
22 # error "can't have more than one sendfile implementation"
23 # endif
24 # define NETWORK_WRITE_USE_SENDFILE "linux-sendfile"
25 # define NETWORK_WRITE_USE_LINUX_SENDFILE
26 #endif
27
28 #if defined HAVE_SENDFILE && (defined(__FreeBSD__) || defined(__DragonFly__))
29 # ifdef NETWORK_WRITE_USE_SENDFILE
30 # error "can't have more than one sendfile implementation"
31 # endif
32 # define NETWORK_WRITE_USE_SENDFILE "freebsd-sendfile"
33 # define NETWORK_WRITE_USE_FREEBSD_SENDFILE
34 #endif
35
36 #if defined HAVE_SENDFILE && defined(__APPLE__)
37 # ifdef NETWORK_WRITE_USE_SENDFILE
38 # error "can't have more than one sendfile implementation"
39 # endif
40 # define NETWORK_WRITE_USE_SENDFILE "darwin-sendfile"
41 # define NETWORK_WRITE_USE_DARWIN_SENDFILE
42 #endif
43
44 #if defined(__APPLE__) && defined(__MACH__)
45 /* sendfile() on iOS/tvOS sendfile() raises SIGSYS insead of returning ENOSYS
46 * https://github.com/ndfred/iperf-ios/issues/17
47 * https://github.com/dotnet/runtime/pull/69436 */
48 #include <TargetConditionals.h> /* TARGET_OS_IPHONE, TARGET_OS_MAC */
49 #if TARGET_OS_IPHONE /* iOS, tvOS, or watchOS device */
50 #undef NETWORK_WRITE_USE_SENDFILE
51 #undef NETWORK_WRITE_USE_DARWIN_SENDFILE
52 #endif
53 #endif
54
55 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILEV && defined(__sun)
56 # ifdef NETWORK_WRITE_USE_SENDFILE
57 # error "can't have more than one sendfile implementation"
58 # endif
59 # define NETWORK_WRITE_USE_SENDFILE "solaris-sendfilev"
60 # define NETWORK_WRITE_USE_SOLARIS_SENDFILEV
61 #endif
62
63 /* not supported so far
64 #if defined HAVE_SEND_FILE && defined(__aix)
65 # ifdef NETWORK_WRITE_USE_SENDFILE
66 # error "can't have more than one sendfile implementation"
67 # endif
68 # define NETWORK_WRITE_USE_SENDFILE "aix-sendfile"
69 # define NETWORK_WRITE_USE_AIX_SENDFILE
70 #endif
71 */
72
73 #if defined HAVE_SYS_UIO_H && defined HAVE_WRITEV
74 # define NETWORK_WRITE_USE_WRITEV
75 #endif
76
77 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
78 #ifdef ENABLE_MMAP
79 # define NETWORK_WRITE_USE_MMAP
80 #endif
81 #endif
82
83
84 __attribute_cold__
network_write_error(int fd,log_error_st * errh)85 static int network_write_error(int fd, log_error_st *errh) {
86 #if defined(__WIN32)
87 int lastError = WSAGetLastError();
88 switch (lastError) {
89 case WSAEINTR:
90 case WSAEWOULDBLOCK:
91 return -3;
92 case WSAECONNRESET:
93 case WSAETIMEDOUT:
94 case WSAECONNABORTED:
95 return -2;
96 default:
97 log_error(errh,__FILE__,__LINE__,"send failed: %d %d",lastError,fd);
98 return -1;
99 }
100 #else /* __WIN32 */
101 switch (errno) {
102 case EAGAIN:
103 case EINTR:
104 return -3;
105 case EPIPE:
106 case ECONNRESET:
107 return -2;
108 default:
109 log_perror(errh,__FILE__,__LINE__,"write failed: %d",fd);
110 return -1;
111 }
112 #endif /* __WIN32 */
113 }
114
115 __attribute_cold__
network_remove_finished_chunks(chunkqueue * const cq,const off_t len)116 static int network_remove_finished_chunks(chunkqueue * const cq, const off_t len) {
117 force_assert(len >= 0);
118 chunkqueue_remove_finished_chunks(cq);
119 return 0;
120 }
121
122 inline
network_write_data_len(int fd,const char * data,off_t len)123 static ssize_t network_write_data_len(int fd, const char *data, off_t len) {
124 #if defined(__WIN32)
125 return send(fd, data, len, 0);
126 #else /* __WIN32 */
127 return write(fd, data, len);
128 #endif /* __WIN32 */
129 }
130
network_write_accounting(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh,const ssize_t wr,const off_t toSend)131 static int network_write_accounting(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh, const ssize_t wr, const off_t toSend) {
132 if (wr >= 0) {
133 *p_max_bytes -= wr;/*(toSend > 0 if we reach this func)*/
134 const int rc = (wr == toSend && *p_max_bytes > 0) ? 0 : -3;
135 chunkqueue_mark_written(cq, wr);
136 return rc;
137 }
138 else
139 return network_write_error(fd, errh);
140 }
141
142
143
144
145 /* write next chunk(s); finished chunks are removed afterwards after successful writes.
146 * return values: similar as backends (0 success, -1 error, -2 remote close, -3 try again later (EINTR/EAGAIN)) */
147 /* next chunk must be MEM_CHUNK. use write()/send() */
148 #if !defined(NETWORK_WRITE_USE_WRITEV)
network_write_mem_chunk(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh)149 static int network_write_mem_chunk(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
150 chunk* const c = cq->first;
151 off_t c_len = (off_t)buffer_clen(c->mem) - c->offset;
152 if (c_len > *p_max_bytes) c_len = *p_max_bytes;
153 if (c_len <= 0) return network_remove_finished_chunks(cq, c_len);
154
155 ssize_t wr = network_write_data_len(fd, c->mem->ptr + c->offset, c_len);
156 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, c_len);
157 }
158 #endif
159
160
161
162
163 __attribute_noinline__
network_write_file_chunk_no_mmap(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh)164 static int network_write_file_chunk_no_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
165 chunk* const c = cq->first;
166 off_t toSend = c->file.length - c->offset;
167 char buf[16384]; /* max read 16kb in one step */
168
169 if (toSend > *p_max_bytes) toSend = *p_max_bytes;
170 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend);
171 if (toSend > (off_t)sizeof(buf)) toSend = (off_t)sizeof(buf);
172
173 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1;
174
175 toSend = chunk_file_pread(c->file.fd, buf, toSend, c->offset);
176 if (toSend <= 0) {
177 log_perror(errh, __FILE__, __LINE__, "read");/* err or unexpected EOF */
178 return -1;
179 }
180
181 ssize_t wr = network_write_data_len(fd, buf, toSend);
182 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend);
183 }
184
185
186
187
188 #if defined(NETWORK_WRITE_USE_MMAP)
189
190 #include "sys-setjmp.h"
191
192 static off_t
network_write_setjmp_write_cb(void * fd,const void * data,off_t len)193 network_write_setjmp_write_cb (void *fd, const void *data, off_t len)
194 {
195 return network_write_data_len((int)(uintptr_t)fd, data, len);
196 }
197
198 /* next chunk must be FILE_CHUNK. send mmap()ed file with write() */
network_write_file_chunk_mmap(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh)199 static int network_write_file_chunk_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
200 chunk * const restrict c = cq->first;
201 const chunk_file_view * const restrict cfv = (!c->file.is_temp)
202 ? chunkqueue_chunk_file_view(cq->first, 0, errh)/*use default 512k block*/
203 : NULL;
204 if (NULL == cfv)
205 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh);
206
207 off_t toSend = c->file.length - c->offset;
208 if (toSend > *p_max_bytes) toSend = *p_max_bytes;
209 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend);
210
211 const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset);
212 const char * const data = chunk_file_view_dptr(cfv, c->offset);
213 if (toSend > mmap_avail) toSend = mmap_avail;
214 off_t wr = sys_setjmp_eval3(network_write_setjmp_write_cb,
215 (void *)(uintptr_t)fd, data, toSend);
216 return network_write_accounting(fd,cq,p_max_bytes,errh,(ssize_t)wr,toSend);
217 }
218
219 #endif /* NETWORK_WRITE_USE_MMAP */
220
221
222
223
224 #if defined(NETWORK_WRITE_USE_WRITEV)
225
226 #if defined(HAVE_SYS_UIO_H)
227 # include <sys/uio.h>
228 #endif
229
230 #if defined(UIO_MAXIOV)
231 # define SYS_MAX_CHUNKS UIO_MAXIOV
232 #elif defined(IOV_MAX)
233 /* new name for UIO_MAXIOV since IEEE Std 1003.1-2001 */
234 # define SYS_MAX_CHUNKS IOV_MAX
235 #elif defined(_XOPEN_IOV_MAX)
236 /* minimum value for sysconf(_SC_IOV_MAX); posix requires this to be at least 16, which is good enough - no need to call sysconf() */
237 # define SYS_MAX_CHUNKS _XOPEN_IOV_MAX
238 #else
239 # error neither UIO_MAXIOV nor IOV_MAX nor _XOPEN_IOV_MAX are defined
240 #endif
241
242 /* allocate iovec[MAX_CHUNKS] on stack, so pick a sane limit:
243 * - each entry will use 1 pointer + 1 size_t
244 * - 32 chunks -> 256 / 512 bytes (32-bit/64-bit pointers)
245 */
246 #define STACK_MAX_ALLOC_CHUNKS 32
247 #if SYS_MAX_CHUNKS > STACK_MAX_ALLOC_CHUNKS
248 # define MAX_CHUNKS STACK_MAX_ALLOC_CHUNKS
249 #else
250 # define MAX_CHUNKS SYS_MAX_CHUNKS
251 #endif
252
253 /* next chunk must be MEM_CHUNK. send multiple mem chunks using writev() */
network_writev_mem_chunks(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh)254 static int network_writev_mem_chunks(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
255 size_t num_chunks = 0;
256 off_t toSend = 0;
257 struct iovec chunks[MAX_CHUNKS];
258
259 for (const chunk *c = cq->first; c && MEM_CHUNK == c->type; c = c->next) {
260 const off_t c_len = (off_t)buffer_clen(c->mem) - c->offset;
261 if (c_len > 0) {
262 toSend += c_len;
263
264 chunks[num_chunks].iov_base = c->mem->ptr + c->offset;
265 chunks[num_chunks].iov_len = (size_t)c_len;
266
267 if (++num_chunks == MAX_CHUNKS || toSend >= *p_max_bytes) break;
268 }
269 else if (c_len < 0) /*(should not happen; trigger assert)*/
270 return network_remove_finished_chunks(cq, c_len);
271 }
272 if (0 == num_chunks) return network_remove_finished_chunks(cq, 0);
273
274 ssize_t wr = writev(fd, chunks, num_chunks);
275 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend);
276 }
277
278 #endif /* NETWORK_WRITE_USE_WRITEV */
279
280
281
282
283 #if defined(NETWORK_WRITE_USE_SENDFILE)
284
285 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE) \
286 || defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV)
287 #include <sys/sendfile.h>
288 #endif
289
290 #if defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE) \
291 || defined(NETWORK_WRITE_USE_DARWIN_SENDFILE)
292 #include <sys/uio.h>
293 #endif
294
network_write_file_chunk_sendfile(const int fd,chunkqueue * const cq,off_t * const p_max_bytes,log_error_st * const errh)295 static int network_write_file_chunk_sendfile(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
296 chunk * const c = cq->first;
297 ssize_t wr;
298 off_t offset;
299 off_t toSend;
300 off_t written = 0;
301
302 offset = c->offset;
303 toSend = c->file.length - c->offset;
304 if (toSend > *p_max_bytes) toSend = *p_max_bytes;
305 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend);
306
307 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1;
308
309 /* Darwin, FreeBSD, and Solaris variants support iovecs and could
310 * be optimized to send more than just file in single syscall */
311
312 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE)
313
314 wr = sendfile(fd, c->file.fd, &offset, toSend);
315 if (wr > 0) written = (off_t)wr;
316
317 #elif defined(NETWORK_WRITE_USE_DARWIN_SENDFILE)
318
319 written = toSend;
320 wr = sendfile(c->file.fd, fd, offset, &written, NULL, 0);
321 /* (for EAGAIN/EINTR written still contains the sent bytes) */
322
323 #elif defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE)
324
325 wr = sendfile(c->file.fd, fd, offset, toSend, NULL, &written, 0);
326 /* (for EAGAIN/EINTR written still contains the sent bytes) */
327
328 #elif defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV)
329 {
330 sendfilevec_t fvec;
331 fvec.sfv_fd = c->file.fd;
332 fvec.sfv_flag = 0;
333 fvec.sfv_off = offset;
334 fvec.sfv_len = toSend;
335
336 /* Solaris sendfilev() */
337 wr = sendfilev(fd, &fvec, 1, (size_t *)&written);
338 /* (for EAGAIN/EINTR written still contains the sent bytes) */
339 }
340 #else
341
342 wr = -1;
343 errno = ENOSYS;
344
345 #endif
346
347 if (-1 == wr) {
348 switch(errno) {
349 case EAGAIN:
350 case EINTR:
351 break; /* try again later */
352 case EPIPE:
353 case ECONNRESET:
354 case ENOTCONN:
355 return -2;
356 case EINVAL:
357 case ENOSYS:
358 #if defined(ENOTSUP) && (!defined(EOPNOTSUPP) || EOPNOTSUPP != ENOTSUP)
359 case ENOTSUP:
360 #endif
361 #ifdef EOPNOTSUPP
362 case EOPNOTSUPP:
363 #endif
364 #ifdef ESOCKTNOSUPPORT
365 case ESOCKTNOSUPPORT:
366 #endif
367 #ifdef EAFNOSUPPORT
368 case EAFNOSUPPORT:
369 #endif
370 #ifdef NETWORK_WRITE_USE_MMAP
371 return network_write_file_chunk_mmap(fd, cq, p_max_bytes, errh);
372 #else
373 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh);
374 #endif
375 default:
376 log_perror(errh, __FILE__, __LINE__, "sendfile(): fd: %d", fd);
377 return -1;
378 }
379 }
380
381 if (written > 0) {
382 chunkqueue_mark_written(cq, written);
383 *p_max_bytes -= written;
384 if (__builtin_expect( (*p_max_bytes <= 0), 0)) return -3;
385 }
386 else if (0 == wr) { /*(-1 != wr && 0 == written)*/
387 log_error(errh, __FILE__, __LINE__,
388 "sendfile(): fd: %d file truncated", fd);
389 return -1;
390 }
391
392 return (wr >= 0 && written == toSend) ? 0 : -3;
393 }
394
395 #endif
396
397
398
399
400 /* return values:
401 * >= 0 : no error
402 * -1 : error (on our side)
403 * -2 : remote close
404 */
405
network_write_chunkqueue_writev(const int fd,chunkqueue * const cq,off_t max_bytes,log_error_st * const errh)406 static int network_write_chunkqueue_writev(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) {
407 while (NULL != cq->first) {
408 int rc = -1;
409
410 switch (cq->first->type) {
411 case MEM_CHUNK:
412 #if defined(NETWORK_WRITE_USE_WRITEV)
413 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh);
414 #else
415 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh);
416 #endif
417 break;
418 case FILE_CHUNK:
419 #ifdef NETWORK_WRITE_USE_MMAP
420 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh);
421 #else
422 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh);
423 #endif
424 break;
425 }
426
427 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc;
428 }
429
430 return 0;
431 }
432
433 #if defined(NETWORK_WRITE_USE_SENDFILE)
network_write_chunkqueue_sendfile(const int fd,chunkqueue * const cq,off_t max_bytes,log_error_st * const errh)434 static int network_write_chunkqueue_sendfile(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) {
435 while (NULL != cq->first) {
436 int rc = -1;
437
438 switch (cq->first->type) {
439 case MEM_CHUNK:
440 #if defined(NETWORK_WRITE_USE_WRITEV)
441 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh);
442 #else
443 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh);
444 #endif
445 break;
446 case FILE_CHUNK:
447 #if defined(NETWORK_WRITE_USE_SENDFILE)
448 rc = network_write_file_chunk_sendfile(fd, cq, &max_bytes, errh);
449 #elif defined(NETWORK_WRITE_USE_MMAP)
450 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh);
451 #else
452 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh);
453 #endif
454 break;
455 }
456
457 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc;
458 }
459
460 return 0;
461 }
462 #endif
463
network_write_init(server * srv)464 int network_write_init(server *srv) {
465 typedef enum {
466 NETWORK_BACKEND_UNSET,
467 NETWORK_BACKEND_WRITE,
468 NETWORK_BACKEND_WRITEV,
469 NETWORK_BACKEND_SENDFILE,
470 } network_backend_t;
471
472 network_backend_t backend;
473
474 struct nb_map {
475 network_backend_t nb;
476 const char *name;
477 } network_backends[] = {
478 /* lowest id wins */
479 { NETWORK_BACKEND_SENDFILE, "sendfile" },
480 { NETWORK_BACKEND_SENDFILE, "linux-sendfile" },
481 { NETWORK_BACKEND_SENDFILE, "freebsd-sendfile" },
482 { NETWORK_BACKEND_SENDFILE, "solaris-sendfilev" },
483 { NETWORK_BACKEND_WRITEV, "writev" },
484 { NETWORK_BACKEND_WRITE, "write" },
485 { NETWORK_BACKEND_UNSET, NULL }
486 };
487
488 /* get a useful default */
489 backend = network_backends[0].nb;
490
491 /* match name against known types */
492 if (srv->srvconf.network_backend) {
493 const char *name, *confname = srv->srvconf.network_backend->ptr;
494 for (size_t i = 0; NULL != (name = network_backends[i].name); ++i) {
495 if (0 == strcmp(confname, name)) {
496 backend = network_backends[i].nb;
497 break;
498 }
499 }
500 if (NULL == name) {
501 log_error(srv->errh, __FILE__, __LINE__,
502 "server.network-backend has an unknown value: %s", confname);
503 return -1;
504 }
505 }
506
507 switch(backend) {
508 case NETWORK_BACKEND_SENDFILE:
509 #if defined(NETWORK_WRITE_USE_SENDFILE)
510 srv->network_backend_write = network_write_chunkqueue_sendfile;
511 break;
512 #endif
513 case NETWORK_BACKEND_WRITEV:
514 case NETWORK_BACKEND_WRITE:
515 srv->network_backend_write = network_write_chunkqueue_writev;
516 break;
517 default:
518 return -1;
519 }
520
521 return 0;
522 }
523
network_write_show_handlers(void)524 const char * network_write_show_handlers(void) {
525 return
526 "\nNetwork handler:\n\n"
527 #if defined NETWORK_WRITE_USE_LINUX_SENDFILE
528 "\t+ linux-sendfile\n"
529 #else
530 "\t- linux-sendfile\n"
531 #endif
532 #if defined NETWORK_WRITE_USE_FREEBSD_SENDFILE
533 "\t+ freebsd-sendfile\n"
534 #else
535 "\t- freebsd-sendfile\n"
536 #endif
537 #if defined NETWORK_WRITE_USE_DARWIN_SENDFILE
538 "\t+ darwin-sendfile\n"
539 #else
540 "\t- darwin-sendfile\n"
541 #endif
542 #if defined NETWORK_WRITE_USE_SOLARIS_SENDFILEV
543 "\t+ solaris-sendfilev\n"
544 #else
545 "\t- solaris-sendfilev\n"
546 #endif
547 #if defined NETWORK_WRITE_USE_WRITEV
548 "\t+ writev\n"
549 #else
550 "\t- writev\n"
551 #endif
552 "\t+ write\n"
553 #ifdef NETWORK_WRITE_USE_MMAP
554 "\t+ mmap support\n"
555 #else
556 "\t- mmap support\n"
557 #endif
558 ;
559 }
560