1 #include "first.h" 2 3 #include "network_write.h" 4 5 #include "base.h" 6 #include "ck.h" 7 #include "log.h" 8 9 #include <sys/types.h> 10 #include "sys-socket.h" 11 12 #include <errno.h> 13 #include <string.h> 14 #include <unistd.h> 15 16 17 /* on linux 2.4.x you get either sendfile or LFS */ 18 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 19 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 20 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 21 # ifdef NETWORK_WRITE_USE_SENDFILE 22 # error "can't have more than one sendfile implementation" 23 # endif 24 # define NETWORK_WRITE_USE_SENDFILE "linux-sendfile" 25 # define NETWORK_WRITE_USE_LINUX_SENDFILE 26 #endif 27 28 #if defined HAVE_SENDFILE && (defined(__FreeBSD__) || defined(__DragonFly__)) 29 # ifdef NETWORK_WRITE_USE_SENDFILE 30 # error "can't have more than one sendfile implementation" 31 # endif 32 # define NETWORK_WRITE_USE_SENDFILE "freebsd-sendfile" 33 # define NETWORK_WRITE_USE_FREEBSD_SENDFILE 34 #endif 35 36 #if defined HAVE_SENDFILE && defined(__APPLE__) 37 # ifdef NETWORK_WRITE_USE_SENDFILE 38 # error "can't have more than one sendfile implementation" 39 # endif 40 # define NETWORK_WRITE_USE_SENDFILE "darwin-sendfile" 41 # define NETWORK_WRITE_USE_DARWIN_SENDFILE 42 #endif 43 44 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILEV && defined(__sun) 45 # ifdef NETWORK_WRITE_USE_SENDFILE 46 # error "can't have more than one sendfile implementation" 47 # endif 48 # define NETWORK_WRITE_USE_SENDFILE "solaris-sendfilev" 49 # define NETWORK_WRITE_USE_SOLARIS_SENDFILEV 50 #endif 51 52 /* not supported so far 53 #if defined HAVE_SEND_FILE && defined(__aix) 54 # ifdef NETWORK_WRITE_USE_SENDFILE 55 # error "can't have more than one sendfile implementation" 56 # endif 57 # define NETWORK_WRITE_USE_SENDFILE "aix-sendfile" 58 # define NETWORK_WRITE_USE_AIX_SENDFILE 59 #endif 60 */ 61 62 #if defined HAVE_SYS_UIO_H && defined HAVE_WRITEV 63 # define NETWORK_WRITE_USE_WRITEV 64 #endif 65 66 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ 67 #ifdef ENABLE_MMAP 68 # define NETWORK_WRITE_USE_MMAP 69 #endif 70 #endif 71 72 73 __attribute_cold__ 74 static int network_write_error(int fd, log_error_st *errh) { 75 #if defined(__WIN32) 76 int lastError = WSAGetLastError(); 77 switch (lastError) { 78 case WSAEINTR: 79 case WSAEWOULDBLOCK: 80 return -3; 81 case WSAECONNRESET: 82 case WSAETIMEDOUT: 83 case WSAECONNABORTED: 84 return -2; 85 default: 86 log_error(errh,__FILE__,__LINE__,"send failed: %d %d",lastError,fd); 87 return -1; 88 } 89 #else /* __WIN32 */ 90 switch (errno) { 91 case EAGAIN: 92 case EINTR: 93 return -3; 94 case EPIPE: 95 case ECONNRESET: 96 return -2; 97 default: 98 log_perror(errh,__FILE__,__LINE__,"write failed: %d",fd); 99 return -1; 100 } 101 #endif /* __WIN32 */ 102 } 103 104 __attribute_cold__ 105 static int network_remove_finished_chunks(chunkqueue * const cq, const off_t len) { 106 force_assert(len >= 0); 107 chunkqueue_remove_finished_chunks(cq); 108 return 0; 109 } 110 111 inline 112 static ssize_t network_write_data_len(int fd, const char *data, off_t len) { 113 #if defined(__WIN32) 114 return send(fd, data, len, 0); 115 #else /* __WIN32 */ 116 return write(fd, data, len); 117 #endif /* __WIN32 */ 118 } 119 120 static int network_write_accounting(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh, const ssize_t wr, const off_t toSend) { 121 if (wr >= 0) { 122 *p_max_bytes -= wr;/*(toSend > 0 if we reach this func)*/ 123 const int rc = (wr == toSend && *p_max_bytes > 0) ? 0 : -3; 124 chunkqueue_mark_written(cq, wr); 125 return rc; 126 } 127 else 128 return network_write_error(fd, errh); 129 } 130 131 132 133 134 /* write next chunk(s); finished chunks are removed afterwards after successful writes. 135 * return values: similar as backends (0 success, -1 error, -2 remote close, -3 try again later (EINTR/EAGAIN)) */ 136 /* next chunk must be MEM_CHUNK. use write()/send() */ 137 #if !defined(NETWORK_WRITE_USE_WRITEV) 138 static int network_write_mem_chunk(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 139 chunk* const c = cq->first; 140 off_t c_len = (off_t)buffer_clen(c->mem) - c->offset; 141 if (c_len > *p_max_bytes) c_len = *p_max_bytes; 142 if (c_len <= 0) return network_remove_finished_chunks(cq, c_len); 143 144 ssize_t wr = network_write_data_len(fd, c->mem->ptr + c->offset, c_len); 145 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, c_len); 146 } 147 #endif 148 149 150 151 152 __attribute_noinline__ 153 static int network_write_file_chunk_no_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 154 chunk* const c = cq->first; 155 off_t toSend = c->file.length - c->offset; 156 char buf[16384]; /* max read 16kb in one step */ 157 158 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 159 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 160 if (toSend > (off_t)sizeof(buf)) toSend = (off_t)sizeof(buf); 161 162 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1; 163 164 toSend = chunk_file_pread(c->file.fd, buf, toSend, c->offset); 165 if (toSend <= 0) { 166 log_perror(errh, __FILE__, __LINE__, "read");/* err or unexpected EOF */ 167 return -1; 168 } 169 170 ssize_t wr = network_write_data_len(fd, buf, toSend); 171 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend); 172 } 173 174 175 176 177 #if defined(NETWORK_WRITE_USE_MMAP) 178 179 #include "sys-setjmp.h" 180 181 static off_t 182 network_write_setjmp_write_cb (void *fd, const void *data, off_t len) 183 { 184 return network_write_data_len((int)(uintptr_t)fd, data, len); 185 } 186 187 /* next chunk must be FILE_CHUNK. send mmap()ed file with write() */ 188 static int network_write_file_chunk_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 189 chunk * const restrict c = cq->first; 190 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 191 ? chunkqueue_chunk_file_view(cq->first, 0, errh)/*use default 512k block*/ 192 : NULL; 193 if (NULL == cfv) 194 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); 195 196 off_t toSend = c->file.length - c->offset; 197 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 198 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 199 200 const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); 201 const char * const data = chunk_file_view_dptr(cfv, c->offset); 202 if (toSend > mmap_avail) toSend = mmap_avail; 203 off_t wr = sys_setjmp_eval3(network_write_setjmp_write_cb, 204 (void *)(uintptr_t)fd, data, toSend); 205 return network_write_accounting(fd,cq,p_max_bytes,errh,(ssize_t)wr,toSend); 206 } 207 208 #endif /* NETWORK_WRITE_USE_MMAP */ 209 210 211 212 213 #if defined(NETWORK_WRITE_USE_WRITEV) 214 215 #if defined(HAVE_SYS_UIO_H) 216 # include <sys/uio.h> 217 #endif 218 219 #if defined(UIO_MAXIOV) 220 # define SYS_MAX_CHUNKS UIO_MAXIOV 221 #elif defined(IOV_MAX) 222 /* new name for UIO_MAXIOV since IEEE Std 1003.1-2001 */ 223 # define SYS_MAX_CHUNKS IOV_MAX 224 #elif defined(_XOPEN_IOV_MAX) 225 /* minimum value for sysconf(_SC_IOV_MAX); posix requires this to be at least 16, which is good enough - no need to call sysconf() */ 226 # define SYS_MAX_CHUNKS _XOPEN_IOV_MAX 227 #else 228 # error neither UIO_MAXIOV nor IOV_MAX nor _XOPEN_IOV_MAX are defined 229 #endif 230 231 /* allocate iovec[MAX_CHUNKS] on stack, so pick a sane limit: 232 * - each entry will use 1 pointer + 1 size_t 233 * - 32 chunks -> 256 / 512 bytes (32-bit/64-bit pointers) 234 */ 235 #define STACK_MAX_ALLOC_CHUNKS 32 236 #if SYS_MAX_CHUNKS > STACK_MAX_ALLOC_CHUNKS 237 # define MAX_CHUNKS STACK_MAX_ALLOC_CHUNKS 238 #else 239 # define MAX_CHUNKS SYS_MAX_CHUNKS 240 #endif 241 242 /* next chunk must be MEM_CHUNK. send multiple mem chunks using writev() */ 243 static int network_writev_mem_chunks(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 244 size_t num_chunks = 0; 245 off_t toSend = 0; 246 struct iovec chunks[MAX_CHUNKS]; 247 248 for (const chunk *c = cq->first; c && MEM_CHUNK == c->type; c = c->next) { 249 const off_t c_len = (off_t)buffer_clen(c->mem) - c->offset; 250 if (c_len > 0) { 251 toSend += c_len; 252 253 chunks[num_chunks].iov_base = c->mem->ptr + c->offset; 254 chunks[num_chunks].iov_len = (size_t)c_len; 255 256 if (++num_chunks == MAX_CHUNKS || toSend >= *p_max_bytes) break; 257 } 258 else if (c_len < 0) /*(should not happen; trigger assert)*/ 259 return network_remove_finished_chunks(cq, c_len); 260 } 261 if (0 == num_chunks) return network_remove_finished_chunks(cq, 0); 262 263 ssize_t wr = writev(fd, chunks, num_chunks); 264 return network_write_accounting(fd, cq, p_max_bytes, errh, wr, toSend); 265 } 266 267 #endif /* NETWORK_WRITE_USE_WRITEV */ 268 269 270 271 272 #if defined(NETWORK_WRITE_USE_SENDFILE) 273 274 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE) \ 275 || defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV) 276 #include <sys/sendfile.h> 277 #endif 278 279 #if defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE) \ 280 || defined(NETWORK_WRITE_USE_DARWIN_SENDFILE) 281 #include <sys/uio.h> 282 #endif 283 284 static int network_write_file_chunk_sendfile(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) { 285 chunk * const c = cq->first; 286 ssize_t wr; 287 off_t offset; 288 off_t toSend; 289 off_t written = 0; 290 291 offset = c->offset; 292 toSend = c->file.length - c->offset; 293 if (toSend > *p_max_bytes) toSend = *p_max_bytes; 294 if (toSend <= 0) return network_remove_finished_chunks(cq, toSend); 295 296 if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1; 297 298 /* Darwin, FreeBSD, and Solaris variants support iovecs and could 299 * be optimized to send more than just file in single syscall */ 300 301 #if defined(NETWORK_WRITE_USE_LINUX_SENDFILE) 302 303 wr = sendfile(fd, c->file.fd, &offset, toSend); 304 if (wr > 0) written = (off_t)wr; 305 306 #elif defined(NETWORK_WRITE_USE_DARWIN_SENDFILE) 307 308 written = toSend; 309 wr = sendfile(c->file.fd, fd, offset, &written, NULL, 0); 310 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 311 312 #elif defined(NETWORK_WRITE_USE_FREEBSD_SENDFILE) 313 314 wr = sendfile(c->file.fd, fd, offset, toSend, NULL, &written, 0); 315 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 316 317 #elif defined(NETWORK_WRITE_USE_SOLARIS_SENDFILEV) 318 { 319 sendfilevec_t fvec; 320 fvec.sfv_fd = c->file.fd; 321 fvec.sfv_flag = 0; 322 fvec.sfv_off = offset; 323 fvec.sfv_len = toSend; 324 325 /* Solaris sendfilev() */ 326 wr = sendfilev(fd, &fvec, 1, (size_t *)&written); 327 /* (for EAGAIN/EINTR written still contains the sent bytes) */ 328 } 329 #else 330 331 wr = -1; 332 errno = ENOSYS; 333 334 #endif 335 336 if (-1 == wr) { 337 switch(errno) { 338 case EAGAIN: 339 case EINTR: 340 break; /* try again later */ 341 case EPIPE: 342 case ECONNRESET: 343 case ENOTCONN: 344 return -2; 345 case EINVAL: 346 case ENOSYS: 347 #if defined(ENOTSUP) && (!defined(EOPNOTSUPP) || EOPNOTSUPP != ENOTSUP) 348 case ENOTSUP: 349 #endif 350 #ifdef EOPNOTSUPP 351 case EOPNOTSUPP: 352 #endif 353 #ifdef ESOCKTNOSUPPORT 354 case ESOCKTNOSUPPORT: 355 #endif 356 #ifdef EAFNOSUPPORT 357 case EAFNOSUPPORT: 358 #endif 359 #ifdef NETWORK_WRITE_USE_MMAP 360 return network_write_file_chunk_mmap(fd, cq, p_max_bytes, errh); 361 #else 362 return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh); 363 #endif 364 default: 365 log_perror(errh, __FILE__, __LINE__, "sendfile(): fd: %d", fd); 366 return -1; 367 } 368 } 369 370 if (written > 0) { 371 chunkqueue_mark_written(cq, written); 372 *p_max_bytes -= written; 373 if (__builtin_expect( (*p_max_bytes <= 0), 0)) return -3; 374 } 375 else if (0 == wr) { /*(-1 != wr && 0 == written)*/ 376 log_error(errh, __FILE__, __LINE__, 377 "sendfile(): fd: %d file truncated", fd); 378 return -1; 379 } 380 381 return (wr >= 0 && written == toSend) ? 0 : -3; 382 } 383 384 #endif 385 386 387 388 389 /* return values: 390 * >= 0 : no error 391 * -1 : error (on our side) 392 * -2 : remote close 393 */ 394 395 static int network_write_chunkqueue_writev(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) { 396 while (NULL != cq->first) { 397 int rc = -1; 398 399 switch (cq->first->type) { 400 case MEM_CHUNK: 401 #if defined(NETWORK_WRITE_USE_WRITEV) 402 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh); 403 #else 404 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh); 405 #endif 406 break; 407 case FILE_CHUNK: 408 #ifdef NETWORK_WRITE_USE_MMAP 409 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh); 410 #else 411 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh); 412 #endif 413 break; 414 } 415 416 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc; 417 } 418 419 return 0; 420 } 421 422 #if defined(NETWORK_WRITE_USE_SENDFILE) 423 static int network_write_chunkqueue_sendfile(const int fd, chunkqueue * const cq, off_t max_bytes, log_error_st * const errh) { 424 while (NULL != cq->first) { 425 int rc = -1; 426 427 switch (cq->first->type) { 428 case MEM_CHUNK: 429 #if defined(NETWORK_WRITE_USE_WRITEV) 430 rc = network_writev_mem_chunks(fd, cq, &max_bytes, errh); 431 #else 432 rc = network_write_mem_chunk(fd, cq, &max_bytes, errh); 433 #endif 434 break; 435 case FILE_CHUNK: 436 #if defined(NETWORK_WRITE_USE_SENDFILE) 437 rc = network_write_file_chunk_sendfile(fd, cq, &max_bytes, errh); 438 #elif defined(NETWORK_WRITE_USE_MMAP) 439 rc = network_write_file_chunk_mmap(fd, cq, &max_bytes, errh); 440 #else 441 rc = network_write_file_chunk_no_mmap(fd, cq, &max_bytes, errh); 442 #endif 443 break; 444 } 445 446 if (__builtin_expect( (0 != rc), 0)) return (-3 == rc) ? 0 : rc; 447 } 448 449 return 0; 450 } 451 #endif 452 453 int network_write_init(server *srv) { 454 typedef enum { 455 NETWORK_BACKEND_UNSET, 456 NETWORK_BACKEND_WRITE, 457 NETWORK_BACKEND_WRITEV, 458 NETWORK_BACKEND_SENDFILE, 459 } network_backend_t; 460 461 network_backend_t backend; 462 463 struct nb_map { 464 network_backend_t nb; 465 const char *name; 466 } network_backends[] = { 467 /* lowest id wins */ 468 { NETWORK_BACKEND_SENDFILE, "sendfile" }, 469 { NETWORK_BACKEND_SENDFILE, "linux-sendfile" }, 470 { NETWORK_BACKEND_SENDFILE, "freebsd-sendfile" }, 471 { NETWORK_BACKEND_SENDFILE, "solaris-sendfilev" }, 472 { NETWORK_BACKEND_WRITEV, "writev" }, 473 { NETWORK_BACKEND_WRITE, "write" }, 474 { NETWORK_BACKEND_UNSET, NULL } 475 }; 476 477 /* get a useful default */ 478 backend = network_backends[0].nb; 479 480 /* match name against known types */ 481 if (srv->srvconf.network_backend) { 482 const char *name, *confname = srv->srvconf.network_backend->ptr; 483 for (size_t i = 0; NULL != (name = network_backends[i].name); ++i) { 484 if (0 == strcmp(confname, name)) { 485 backend = network_backends[i].nb; 486 break; 487 } 488 } 489 if (NULL == name) { 490 log_error(srv->errh, __FILE__, __LINE__, 491 "server.network-backend has an unknown value: %s", confname); 492 return -1; 493 } 494 } 495 496 switch(backend) { 497 case NETWORK_BACKEND_SENDFILE: 498 #if defined(NETWORK_WRITE_USE_SENDFILE) 499 srv->network_backend_write = network_write_chunkqueue_sendfile; 500 break; 501 #endif 502 case NETWORK_BACKEND_WRITEV: 503 case NETWORK_BACKEND_WRITE: 504 srv->network_backend_write = network_write_chunkqueue_writev; 505 break; 506 default: 507 return -1; 508 } 509 510 return 0; 511 } 512 513 const char * network_write_show_handlers(void) { 514 return 515 "\nNetwork handler:\n\n" 516 #if defined NETWORK_WRITE_USE_LINUX_SENDFILE 517 "\t+ linux-sendfile\n" 518 #else 519 "\t- linux-sendfile\n" 520 #endif 521 #if defined NETWORK_WRITE_USE_FREEBSD_SENDFILE 522 "\t+ freebsd-sendfile\n" 523 #else 524 "\t- freebsd-sendfile\n" 525 #endif 526 #if defined NETWORK_WRITE_USE_DARWIN_SENDFILE 527 "\t+ darwin-sendfile\n" 528 #else 529 "\t- darwin-sendfile\n" 530 #endif 531 #if defined NETWORK_WRITE_USE_SOLARIS_SENDFILEV 532 "\t+ solaris-sendfilev\n" 533 #else 534 "\t- solaris-sendfilev\n" 535 #endif 536 #if defined NETWORK_WRITE_USE_WRITEV 537 "\t+ writev\n" 538 #else 539 "\t- writev\n" 540 #endif 541 "\t+ write\n" 542 #ifdef NETWORK_WRITE_USE_MMAP 543 "\t+ mmap support\n" 544 #else 545 "\t- mmap support\n" 546 #endif 547 ; 548 } 549