1 #include "first.h" 2 3 #include "network_write.h" 4 5 #include "base.h" 6 #include "ck.h" 7 #include "log.h" 8 9 #include <sys/types.h> 10 #include "sys-socket.h" 11 12 #include <errno.h> 13 #include <string.h> 14 #include <unistd.h> 15 16 17 /* on linux 2.4.x you get either sendfile or LFS */ 18 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 19 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 20 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 21 # ifdef NETWORK_WRITE_USE_SENDFILE 22 # error "can't have more than one sendfile implementation" 23 # endif 24 # define NETWORK_WRITE_USE_SENDFILE "linux-sendfile" 25 # define NETWORK_WRITE_USE_LINUX_SENDFILE 26 #endif 27 28 #if defined HAVE_SENDFILE && (defined(__FreeBSD__) || defined(__DragonFly__)) 29 # ifdef NETWORK_WRITE_USE_SENDFILE 30 # error "can't have more than one sendfile implementation" 31 # endif 32 # define NETWORK_WRITE_USE_SENDFILE "freebsd-sendfile" 33 # define NETWORK_WRITE_USE_FREEBSD_SENDFILE 34 #endif 35 36 #if defined HAVE_SENDFILE && defined(__APPLE__) 37 # ifdef NETWORK_WRITE_USE_SENDFILE 38 # error "can't have more than one sendfile implementation" 39 # endif 40 # define NETWORK_WRITE_USE_SENDFILE "darwin-sendfile" 41 # define NETWORK_WRITE_USE_DARWIN_SENDFILE 42 #endif 43 44 #if defined(__APPLE__) && defined(__MACH__) 45 /* sendfile() on iOS/tvOS sendfile() raises SIGSYS insead of returning ENOSYS 46 * https://github.com/ndfred/iperf-ios/issues/17 47 * https://github.com/dotnet/runtime/pull/69436 */ 48 #include <TargetConditionals.h> /* TARGET_OS_IPHONE, TARGET_OS_MAC */ 49 #if TARGET_OS_IPHONE /* iOS, tvOS, or watchOS device */ 50 #undef NETWORK_WRITE_USE_SENDFILE 51 #undef NETWORK_WRITE_USE_DARWIN_SENDFILE 52 #endif 53 #endif 54 55 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILEV && defined(__sun) 56 # ifdef NETWORK_WRITE_USE_SENDFILE 57 # error "can't have more than one sendfile implementation" 58 # endif 59 # define NETWORK_WRITE_USE_SENDFILE "solaris-sendfilev" 60 # define NETWORK_WRITE_USE_SOLARIS_SENDFILEV 61 #endif 62 63 /* not supported so far 64 #if defined HAVE_SEND_FILE && defined(__aix) 65 # ifdef NETWORK_WRITE_USE_SENDFILE 66 # error "can't have more than one sendfile implementation" 67 # endif 68 # define NETWORK_WRITE_USE_SENDFILE "aix-sendfile" 69 # define NETWORK_WRITE_USE_AIX_SENDFILE 70 #endif 71 */ 72 73 #if defined HAVE_SYS_UIO_H && defined HAVE_WRITEV 74 # define NETWORK_WRITE_USE_WRITEV 75 #endif 76 77 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ 78 #ifdef ENABLE_MMAP 79 # define NETWORK_WRITE_USE_MMAP 80 #endif 81 #endif 82 83 84 __attribute_cold__ 85 static int network_write_error(int fd, log_error_st *errh) { 86 #if defined(__WIN32) 87 int lastError = WSAGetLastError(); 88 switch (lastError) { 89 case WSAEINTR: 90 case WSAEWOULDBLOCK: 91 return -3; 92 case WSAECONNRESET: 93 case WSAETIMEDOUT: 94 case WSAECONNABORTED: 95 return -2; 96 default: 97 log_error(errh,__FILE__,__LINE__,"send failed: %d %d",lastError,fd); 98 return -1; 99 } 100 #else /* __WIN32 */ 101 switch (errno) { 102 case EAGAIN: 103 case EINTR: 104 return -3; 105 case EPIPE: 106 case ECONNRESET: 107 return -2; 108 default: 109 log_perror(errh,__FILE__,__LINE__,"write failed: %d",fd); 110 return -1; 111 } 112 #endif /* __WIN32 */ 113 } 114 115 __attribute_cold__ 116 static int network_remove_finished_chunks(chunkqueue * const cq, const off_t len) { 117 force_assert(len >= 0); 118 chunkqueue_remove_finished_chunks(cq); 119 return 0; 120 } 121 122 inline 123 static ssize_t network_write_data_len(int fd, const char *data, off_t len) { 124 #if defined(__WIN32) 125 return send(fd, data, len, 0); 126 #else /* __WIN32 */ 127 return write(fd, data, len); 128 #endif /* __WIN32 */ 129 } 130 131 static int network_write_accounting(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh, const ssize_t wr, const off_t toSend) { 132 if (wr >= 0) { 133 *p_max_bytes -= wr;/*(toSend > 0 if we reach this func)*/ 134 const int rc = (wr == toSend && *p_max_bytes > 0) ? 0 : -3; 135 chunkqueue_mark_written(cq, wr); 136 return rc; 137 } 138 else 139 return network_write_error(fd, errh); 140 } 141 142 143 144 145 /* write next chunk(s); finished chunks are removed afterwards after successful writes. 146 * return values: similar as backends (0 success, -1 error, -2 remote close, -3 try again later (EINTR/EAGAIN)) */ 147 /* next chunk must be MEM_CHUNK. use write()/send() */ 148 #if !defined(NETWORK_WRITE_USE_WRITEV) 149 static int network_write_mem_chunk(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 150 chunk* const c = cq->first; 151 off_t c_len = (off_t)buffer_clen(c->mem) - c->offset; 152 if (c_len > *p_max_bytes) c_len = *p_max_bytes; 153 if (c_len <= 0) return network_remove_finished_chunks(cq, c_len); 154 155 ssize_t wr = network_write_data_len(fd, c->mem->ptr + c->offset, c_len); 156 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, c_len); 157 } 158 #endif 159 160 161 162 163 __attribute_noinline__ 164 static int network_write_file_chunk_no_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 165 chunk* const c = cq->first; 166 off_t toSend = c->file.length - c->offset; 167 char buf[16384]; /* max read 16kb in one step */ 168 169 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 170 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 171 if (toSend > (off_t)sizeof(buf)) toSend = (off_t)sizeof(buf); 172 173 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1; 174 175 toSend = chunk_file_pread(c->file.fd, buf, toSend, c->offset); 176 if (toSend <= 0) { 177 log_perror(errh, __FILE__, __LINE__, "read");/* err or unexpected EOF */ 178 return -1; 179 } 180 181 ssize_t wr = network_write_data_len(fd, buf, toSend); 182 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend); 183 } 184 185 186 187 188 #if defined(NETWORK_WRITE_USE_MMAP) 189 190 #include "sys-setjmp.h" 191 192 static off_t 193 network_write_setjmp_write_cb (void *fd, const void *data, off_t len) 194 { 195 return network_write_data_len((int)(uintptr_t)fd, data, len); 196 } 197 198 /* next chunk must be FILE_CHUNK. send mmap()ed file with write() */ 199 static int network_write_file_chunk_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 200 chunk * const restrict c = cq->first; 201 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 202 ? chunkqueue_chunk_file_view(cq->first, 0, errh)/*use default 512k block*/ 203 : NULL; 204 if (NULL == cfv) 205 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); 206 207 off_t toSend = c->file.length - c->offset; 208 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 209 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 210 211 const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); 212 const char * const data = chunk_file_view_dptr(cfv, c->offset); 213 if (toSend > mmap_avail) toSend = mmap_avail; 214 off_t wr = sys_setjmp_eval3(network_write_setjmp_write_cb, 215 (void *)(uintptr_t)fd, data, toSend); 216 return network_write_accounting(fd,cq,p_max_bytes,errh,(ssize_t)wr,toSend); 217 } 218 219 #endif /* NETWORK_WRITE_USE_MMAP */ 220 221 222 223 224 #if defined(NETWORK_WRITE_USE_WRITEV) 225 226 #if defined(HAVE_SYS_UIO_H) 227 # include <sys/uio.h> 228 #endif 229 230 #if defined(UIO_MAXIOV) 231 # define SYS_MAX_CHUNKS UIO_MAXIOV 232 #elif defined(IOV_MAX) 233 /* new name for UIO_MAXIOV since IEEE Std 1003.1-2001 */ 234 # define SYS_MAX_CHUNKS IOV_MAX 235 #elif defined(_XOPEN_IOV_MAX) 236 /* minimum value for sysconf(_SC_IOV_MAX); posix requires this to be at least 16, which is good enough - no need to call sysconf() */ 237 # define SYS_MAX_CHUNKS _XOPEN_IOV_MAX 238 #else 239 # error neither UIO_MAXIOV nor IOV_MAX nor _XOPEN_IOV_MAX are defined 240 #endif 241 242 /* allocate iovec[MAX_CHUNKS] on stack, so pick a sane limit: 243 * - each entry will use 1 pointer + 1 size_t 244 * - 32 chunks -> 256 / 512 bytes (32-bit/64-bit pointers) 245 */ 246 #define STACK_MAX_ALLOC_CHUNKS 32 247 #if SYS_MAX_CHUNKS > STACK_MAX_ALLOC_CHUNKS 248 # define MAX_CHUNKS STACK_MAX_ALLOC_CHUNKS 249 #else 250 # define MAX_CHUNKS SYS_MAX_CHUNKS 251 #endif 252 253 /* next chunk must be MEM_CHUNK. send multiple mem chunks using writev() */ 254 static int network_writev_mem_chunks(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 255 size_t num_chunks = 0; 256 off_t toSend = 0; 257 struct iovec chunks[MAX_CHUNKS]; 258 259 for (const chunk *c = cq->first; c && MEM_CHUNK == c->type; c = c->next) { 260 const off_t c_len = (off_t)buffer_clen(c->mem) - c->offset; 261 if (c_len > 0) { 262 toSend += c_len; 263 264 chunks[num_chunks].iov_base = c->mem->ptr + c->offset; 265 chunks[num_chunks].iov_len = (size_t)c_len; 266 267 if (++num_chunks == MAX_CHUNKS || toSend >= *p_max_bytes) break; 268 } 269 else if (c_len < 0) /*(should not happen; trigger assert)*/ 270 return network_remove_finished_chunks(cq, c_len); 271 } 272 if (0 == num_chunks) return network_remove_finished_chunks(cq, 0); 273 274 ssize_t wr = writev(fd, chunks, num_chunks); 275 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend); 276 } 277 278 #endif /* NETWORK_WRITE_USE_WRITEV */ 279 280 281 282 283 #if defined(NETWORK_WRITE_USE_SENDFILE) 284 285 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE) \ 286 || defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV) 287 #include <sys/sendfile.h> 288 #endif 289 290 #if defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE) \ 291 || defined(NETWORK_WRITE_USE_DARWIN_SENDFILE) 292 #include <sys/uio.h> 293 #endif 294 295 static int network_write_file_chunk_sendfile(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 296 chunk * const c = cq->first; 297 ssize_t wr; 298 off_t offset; 299 off_t toSend; 300 off_t written = 0; 301 302 offset = c->offset; 303 toSend = c->file.length - c->offset; 304 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 305 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 306 307 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1; 308 309 /* Darwin, FreeBSD, and Solaris variants support iovecs and could 310 * be optimized to send more than just file in single syscall */ 311 312 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE) 313 314 wr = sendfile(fd, c->file.fd, &offset, toSend); 315 if (wr > 0) written = (off_t)wr; 316 317 #elif defined(NETWORK_WRITE_USE_DARWIN_SENDFILE) 318 319 written = toSend; 320 wr = sendfile(c->file.fd, fd, offset, &written, NULL, 0); 321 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 322 323 #elif defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE) 324 325 wr = sendfile(c->file.fd, fd, offset, toSend, NULL, &written, 0); 326 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 327 328 #elif defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV) 329 { 330 sendfilevec_t fvec; 331 fvec.sfv_fd = c->file.fd; 332 fvec.sfv_flag = 0; 333 fvec.sfv_off = offset; 334 fvec.sfv_len = toSend; 335 336 /* Solaris sendfilev() */ 337 wr = sendfilev(fd, &fvec, 1, (size_t *)&written); 338 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 339 } 340 #else 341 342 wr = -1; 343 errno = ENOSYS; 344 345 #endif 346 347 if (-1 == wr) { 348 switch(errno) { 349 case EAGAIN: 350 case EINTR: 351 break; /* try again later */ 352 case EPIPE: 353 case ECONNRESET: 354 case ENOTCONN: 355 return -2; 356 case EINVAL: 357 case ENOSYS: 358 #if defined(ENOTSUP) && (!defined(EOPNOTSUPP) || EOPNOTSUPP != ENOTSUP) 359 case ENOTSUP: 360 #endif 361 #ifdef EOPNOTSUPP 362 case EOPNOTSUPP: 363 #endif 364 #ifdef ESOCKTNOSUPPORT 365 case ESOCKTNOSUPPORT: 366 #endif 367 #ifdef EAFNOSUPPORT 368 case EAFNOSUPPORT: 369 #endif 370 #ifdef NETWORK_WRITE_USE_MMAP 371 return network_write_file_chunk_mmap(fd, cq, p_max_bytes, errh); 372 #else 373 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); 374 #endif 375 default: 376 log_perror(errh, __FILE__, __LINE__, "sendfile(): fd: %d", fd); 377 return -1; 378 } 379 } 380 381 if (written > 0) { 382 chunkqueue_mark_written(cq, written); 383 *p_max_bytes -= written; 384 if (__builtin_expect( (*p_max_bytes <= 0), 0)) return -3; 385 } 386 else if (0 == wr) { /*(-1 != wr && 0 == written)*/ 387 log_error(errh, __FILE__, __LINE__, 388 "sendfile(): fd: %d file truncated", fd); 389 return -1; 390 } 391 392 return (wr >= 0 && written == toSend) ? 0 : -3; 393 } 394 395 #endif 396 397 398 399 400 /* return values: 401 * >= 0 : no error 402 * -1 : error (on our side) 403 * -2 : remote close 404 */ 405 406 static int network_write_chunkqueue_writev(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) { 407 while (NULL != cq->first) { 408 int rc = -1; 409 410 switch (cq->first->type) { 411 case MEM_CHUNK: 412 #if defined(NETWORK_WRITE_USE_WRITEV) 413 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh); 414 #else 415 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh); 416 #endif 417 break; 418 case FILE_CHUNK: 419 #ifdef NETWORK_WRITE_USE_MMAP 420 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh); 421 #else 422 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh); 423 #endif 424 break; 425 } 426 427 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc; 428 } 429 430 return 0; 431 } 432 433 #if defined(NETWORK_WRITE_USE_SENDFILE) 434 static int network_write_chunkqueue_sendfile(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) { 435 while (NULL != cq->first) { 436 int rc = -1; 437 438 switch (cq->first->type) { 439 case MEM_CHUNK: 440 #if defined(NETWORK_WRITE_USE_WRITEV) 441 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh); 442 #else 443 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh); 444 #endif 445 break; 446 case FILE_CHUNK: 447 #if defined(NETWORK_WRITE_USE_SENDFILE) 448 rc = network_write_file_chunk_sendfile(fd, cq, &max_bytes, errh); 449 #elif defined(NETWORK_WRITE_USE_MMAP) 450 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh); 451 #else 452 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh); 453 #endif 454 break; 455 } 456 457 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc; 458 } 459 460 return 0; 461 } 462 #endif 463 464 int network_write_init(server *srv) { 465 typedef enum { 466 NETWORK_BACKEND_UNSET, 467 NETWORK_BACKEND_WRITE, 468 NETWORK_BACKEND_WRITEV, 469 NETWORK_BACKEND_SENDFILE, 470 } network_backend_t; 471 472 network_backend_t backend; 473 474 struct nb_map { 475 network_backend_t nb; 476 const char *name; 477 } network_backends[] = { 478 /* lowest id wins */ 479 { NETWORK_BACKEND_SENDFILE, "sendfile" }, 480 { NETWORK_BACKEND_SENDFILE, "linux-sendfile" }, 481 { NETWORK_BACKEND_SENDFILE, "freebsd-sendfile" }, 482 { NETWORK_BACKEND_SENDFILE, "solaris-sendfilev" }, 483 { NETWORK_BACKEND_WRITEV, "writev" }, 484 { NETWORK_BACKEND_WRITE, "write" }, 485 { NETWORK_BACKEND_UNSET, NULL } 486 }; 487 488 /* get a useful default */ 489 backend = network_backends[0].nb; 490 491 /* match name against known types */ 492 if (srv->srvconf.network_backend) { 493 const char *name, *confname = srv->srvconf.network_backend->ptr; 494 for (size_t i = 0; NULL != (name = network_backends[i].name); ++i) { 495 if (0 == strcmp(confname, name)) { 496 backend = network_backends[i].nb; 497 break; 498 } 499 } 500 if (NULL == name) { 501 log_error(srv->errh, __FILE__, __LINE__, 502 "server.network-backend has an unknown value: %s", confname); 503 return -1; 504 } 505 } 506 507 switch(backend) { 508 case NETWORK_BACKEND_SENDFILE: 509 #if defined(NETWORK_WRITE_USE_SENDFILE) 510 srv->network_backend_write = network_write_chunkqueue_sendfile; 511 break; 512 #endif 513 case NETWORK_BACKEND_WRITEV: 514 case NETWORK_BACKEND_WRITE: 515 srv->network_backend_write = network_write_chunkqueue_writev; 516 break; 517 default: 518 return -1; 519 } 520 521 return 0; 522 } 523 524 const char * network_write_show_handlers(void) { 525 return 526 "\nNetwork handler:\n\n" 527 #if defined NETWORK_WRITE_USE_LINUX_SENDFILE 528 "\t+ linux-sendfile\n" 529 #else 530 "\t- linux-sendfile\n" 531 #endif 532 #if defined NETWORK_WRITE_USE_FREEBSD_SENDFILE 533 "\t+ freebsd-sendfile\n" 534 #else 535 "\t- freebsd-sendfile\n" 536 #endif 537 #if defined NETWORK_WRITE_USE_DARWIN_SENDFILE 538 "\t+ darwin-sendfile\n" 539 #else 540 "\t- darwin-sendfile\n" 541 #endif 542 #if defined NETWORK_WRITE_USE_SOLARIS_SENDFILEV 543 "\t+ solaris-sendfilev\n" 544 #else 545 "\t- solaris-sendfilev\n" 546 #endif 547 #if defined NETWORK_WRITE_USE_WRITEV 548 "\t+ writev\n" 549 #else 550 "\t- writev\n" 551 #endif 552 "\t+ write\n" 553 #ifdef NETWORK_WRITE_USE_MMAP 554 "\t+ mmap support\n" 555 #else 556 "\t- mmap support\n" 557 #endif 558 ; 559 } 560