1 #include "first.h" 2 3 /** 4 * the network chunk-API 5 * 6 * 7 */ 8 9 #include "chunk.h" 10 #include "fdevent.h" 11 #include "log.h" 12 13 #include <sys/types.h> 14 #include <sys/stat.h> 15 #include "sys-mmap.h" 16 #include "sys-setjmp.h" 17 18 #include <stdlib.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 22 #include <errno.h> 23 #include <string.h> 24 25 26 #ifdef HAVE_MMAP 27 28 #define MMAP_CHUNK_SIZE (512*1024) 29 30 __attribute_cold__ 31 /*__attribute_noinline__*/ 32 static off_t 33 mmap_pagemask (void) 34 { 35 #ifndef _WIN32 36 long pagesize = sysconf(_SC_PAGESIZE); 37 #else 38 long pagesize = -1; /*(not implemented (yet))*/ 39 #endif 40 if (-1 == pagesize) pagesize = 4096; 41 force_assert(pagesize < MMAP_CHUNK_SIZE); 42 return ~((off_t)pagesize - 1); /* pagesize always power-of-2 */ 43 } 44 45 #if 0 46 static off_t 47 mmap_align_offset (off_t start) 48 { 49 static off_t pagemask = 0; 50 if (0 == pagemask) 51 pagemask = mmap_pagemask(); 52 return (start & pagemask); 53 } 54 #endif 55 56 #define mmap_align_offset(offset) ((offset) & chunk_pagemask) 57 static off_t chunk_pagemask = 0; 58 static int chunk_mmap_flags = MAP_SHARED; 59 60 #endif /* HAVE_MMAP */ 61 62 63 /* default 1 MB */ 64 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024) 65 66 static size_t chunk_buf_sz = 8192; 67 static chunk *chunks, *chunks_oversized, *chunks_filechunk; 68 static chunk *chunk_buffers; 69 static int chunks_oversized_n; 70 static const array *chunkqueue_default_tempdirs = NULL; 71 static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; 72 73 void chunkqueue_set_chunk_size (size_t sz) 74 { 75 size_t x = 1024; 76 while (x < sz && x < (1u << 30)) x <<= 1; 77 chunk_buf_sz = sz > 0 ? x : 8192; 78 } 79 80 void chunkqueue_set_tempdirs_default_reset (void) 81 { 82 chunk_buf_sz = 8192; 83 chunkqueue_default_tempdirs = NULL; 84 chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; 85 86 #ifdef HAVE_MMAP /*(abuse this func to initialize statics as startup)*/ 87 if (0 == chunk_pagemask) 88 chunk_pagemask = mmap_pagemask(); 89 chunk_mmap_flags = MAP_SHARED; 90 #endif 91 } 92 93 chunkqueue *chunkqueue_init(chunkqueue *cq) { 94 /* (if caller passes non-NULL cq, it must be 0-init) */ 95 if (NULL == cq) { 96 cq = calloc(1, sizeof(*cq)); 97 force_assert(NULL != cq); 98 } 99 100 cq->first = NULL; 101 cq->last = NULL; 102 103 cq->tempdirs = chunkqueue_default_tempdirs; 104 cq->upload_temp_file_size = chunkqueue_default_tempfile_size; 105 106 return cq; 107 } 108 109 __attribute_returns_nonnull__ 110 static chunk *chunk_init(void) { 111 chunk * const restrict c = calloc(1, sizeof(*c)); 112 force_assert(NULL != c); 113 114 #if 0 /*(zeroed by calloc())*/ 115 c->type = MEM_CHUNK; 116 c->next = NULL; 117 c->offset = 0; 118 c->file.length = 0; 119 c->file.is_temp = 0; 120 c->file.view = NULL; 121 #endif 122 c->file.fd = -1; 123 124 c->mem = buffer_init(); 125 return c; 126 } 127 128 __attribute_noinline__ 129 __attribute_returns_nonnull__ 130 static chunk *chunk_init_sz(size_t sz) { 131 chunk * const restrict c = chunk_init(); 132 buffer_string_prepare_copy(c->mem, sz-1); 133 return c; 134 } 135 136 #ifdef HAVE_MMAP 137 138 __attribute_malloc__ 139 __attribute_returns_nonnull__ 140 static void * chunk_file_view_init (void) { 141 chunk_file_view * const restrict cfv = calloc(1, sizeof(*cfv)); 142 force_assert(NULL != cfv); 143 cfv->mptr = MAP_FAILED; 144 #if 0 /*(zeroed by calloc())*/ 145 cfv->mlen = 0; 146 cfv->foff = 0; 147 #endif 148 cfv->refcnt = 1; 149 return cfv; 150 } 151 152 __attribute_nonnull__() 153 static chunk_file_view * chunk_file_view_release (chunk_file_view *cfv) { 154 if (0 == --cfv->refcnt) { 155 if (MAP_FAILED != cfv->mptr) 156 munmap(cfv->mptr, (size_t)cfv->mlen); 157 free(cfv); 158 } 159 return NULL; 160 } 161 162 __attribute_cold__ 163 __attribute_noinline__ 164 __attribute_nonnull__() 165 static chunk_file_view * chunk_file_view_failed (chunk_file_view *cfv) { 166 return chunk_file_view_release(cfv); 167 } 168 169 #endif /* HAVE_MMAP */ 170 171 ssize_t 172 chunk_file_pread (int fd, void *buf, size_t count, off_t offset) 173 { 174 /*(expects open file for FILE_CHUNK)*/ 175 #ifndef HAVE_PREAD 176 /*(On systems without pread() or equivalent, lseek() is repeated if this 177 * func is called in a loop, but this func is generally used on small files, 178 * or reading a small bit at a time. Even in the case of mod_deflate, files 179 * are not expected to be excessively large.) */ 180 if (-1 == lseek(fd, offset, SEEK_SET)) 181 return -1; 182 #endif 183 184 ssize_t rd; 185 do { 186 #ifdef HAVE_PREAD 187 rd =pread(fd, buf, count, offset); 188 #else 189 rd = read(fd, buf, count); 190 #endif 191 } while (-1 == rd && errno == EINTR); 192 return rd; 193 } 194 195 static void chunk_reset_file_chunk(chunk *c) { 196 if (c->file.is_temp) { 197 c->file.is_temp = 0; 198 if (!buffer_is_blank(c->mem)) 199 unlink(c->mem->ptr); 200 } 201 if (c->file.refchg) { 202 c->file.refchg(c->file.ref, -1); 203 c->file.refchg = 0; /* NULL fn ptr */ 204 c->file.ref = NULL; 205 } 206 else if (c->file.fd != -1) { 207 close(c->file.fd); 208 } 209 #ifdef HAVE_MMAP 210 if (c->file.view) 211 c->file.view = chunk_file_view_release(c->file.view); 212 #endif 213 c->file.fd = -1; 214 c->file.length = 0; 215 c->type = MEM_CHUNK; 216 } 217 218 static void chunk_reset(chunk *c) { 219 if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c); 220 221 buffer_clear(c->mem); 222 c->offset = 0; 223 } 224 225 static void chunk_free(chunk *c) { 226 if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c); 227 buffer_free(c->mem); 228 free(c); 229 } 230 231 static chunk * chunk_pop_oversized(size_t sz) { 232 /* future: might have buckets of certain sizes, up to socket buf sizes */ 233 if (chunks_oversized && chunks_oversized->mem->size >= sz) { 234 --chunks_oversized_n; 235 chunk *c = chunks_oversized; 236 chunks_oversized = c->next; 237 return c; 238 } 239 return NULL; 240 } 241 242 static void chunk_push_oversized(chunk * const c, const size_t sz) { 243 /* XXX: chunk_buffer_yield() may have removed need for list size limit */ 244 if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) { 245 ++chunks_oversized_n; 246 chunk **co = &chunks_oversized; 247 while (*co && sz < (*co)->mem->size) co = &(*co)->next; 248 c->next = *co; 249 *co = c; 250 } 251 else { 252 buffer * const tb = chunks_oversized ? chunks_oversized->mem : NULL; 253 if (tb && tb->size < sz) { 254 /* swap larger mem block onto head of list; free smaller mem */ 255 chunks_oversized->mem = c->mem; 256 c->mem = tb; 257 } 258 chunk_free(c); 259 } 260 } 261 262 __attribute_noinline__ 263 __attribute_returns_nonnull__ 264 static buffer * chunk_buffer_acquire_sz(const size_t sz) { 265 chunk *c; 266 buffer *b; 267 if (sz <= (chunk_buf_sz|1)) { 268 if (chunks) { 269 c = chunks; 270 chunks = c->next; 271 } 272 else 273 c = chunk_init_sz(chunk_buf_sz); 274 } 275 else { 276 c = chunk_pop_oversized(sz); 277 if (NULL == c) { 278 /*(round up to nearest chunk_buf_sz)*/ 279 /* NB: round down power-2 + 1 to avoid excess allocation 280 * (sz & ~1uL) relies on buffer_realloc() adding +1 *and* on callers 281 * of this func never passing power-2 + 1 sz unless direct caller 282 * adds +1 for '\0', as is done in chunk_buffer_prepare_append() */ 283 c = chunk_init_sz(((sz&~1uL)+(chunk_buf_sz-1)) & ~(chunk_buf_sz-1)); 284 } 285 } 286 c->next = chunk_buffers; 287 chunk_buffers = c; 288 b = c->mem; 289 c->mem = NULL; 290 return b; 291 } 292 293 buffer * chunk_buffer_acquire(void) { 294 return chunk_buffer_acquire_sz(chunk_buf_sz); 295 } 296 297 void chunk_buffer_release(buffer *b) { 298 if (NULL == b) return; 299 if (chunk_buffers) { 300 chunk *c = chunk_buffers; 301 chunk_buffers = c->next; 302 c->mem = b; 303 buffer_clear(b); 304 if (b->size == (chunk_buf_sz|1)) { 305 c->next = chunks; 306 chunks = c; 307 } 308 else if (b->size > chunk_buf_sz) 309 chunk_push_oversized(c, b->size); 310 else 311 chunk_free(c); 312 } 313 else { 314 buffer_free(b); 315 } 316 } 317 318 void chunk_buffer_yield(buffer *b) { 319 if (b->size == (chunk_buf_sz|1)) return; 320 321 buffer * const cb = chunk_buffer_acquire_sz(chunk_buf_sz); 322 buffer tb = *b; 323 *b = *cb; 324 *cb = tb; 325 chunk_buffer_release(cb); 326 } 327 328 size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) { 329 if (sz > buffer_string_space(b)) { 330 sz += b->used ? b->used : 1; 331 buffer * const cb = chunk_buffer_acquire_sz(sz); 332 /* swap buffer contents and copy original b->ptr into larger b->ptr */ 333 /*(this does more than buffer_move())*/ 334 buffer tb = *b; 335 *b = *cb; 336 *cb = tb; 337 if ((b->used = tb.used)) 338 memcpy(b->ptr, tb.ptr, tb.used); 339 chunk_buffer_release(cb); 340 } 341 return buffer_string_space(b); 342 } 343 344 __attribute_noinline__ 345 __attribute_returns_nonnull__ 346 static chunk * chunk_acquire(size_t sz) { 347 if (sz <= (chunk_buf_sz|1)) { 348 if (chunks) { 349 chunk *c = chunks; 350 chunks = c->next; 351 return c; 352 } 353 sz = chunk_buf_sz; 354 } 355 else { 356 /*(round up to nearest chunk_buf_sz)*/ 357 sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); 358 chunk *c = chunk_pop_oversized(sz); 359 if (c) return c; 360 } 361 362 return chunk_init_sz(sz); 363 } 364 365 static void chunk_release(chunk *c) { 366 const size_t sz = c->mem->size; 367 if (sz == (chunk_buf_sz|1)) { 368 chunk_reset(c); 369 c->next = chunks; 370 chunks = c; 371 } 372 else if (sz > chunk_buf_sz) { 373 chunk_reset(c); 374 chunk_push_oversized(c, sz); 375 } 376 else if (c->type == FILE_CHUNK) { 377 chunk_reset(c); 378 c->next = chunks_filechunk; 379 chunks_filechunk = c; 380 } 381 else { 382 chunk_free(c); 383 } 384 } 385 386 __attribute_returns_nonnull__ 387 static chunk * chunk_acquire_filechunk(void) { 388 if (chunks_filechunk) { 389 chunk *c = chunks_filechunk; 390 chunks_filechunk = c->next; 391 return c; 392 } 393 return chunk_init(); 394 } 395 396 void chunkqueue_chunk_pool_clear(void) 397 { 398 for (chunk *next, *c = chunks; c; c = next) { 399 next = c->next; 400 chunk_free(c); 401 } 402 chunks = NULL; 403 for (chunk *next, *c = chunks_oversized; c; c = next) { 404 next = c->next; 405 chunk_free(c); 406 } 407 chunks_oversized = NULL; 408 chunks_oversized_n = 0; 409 for (chunk *next, *c = chunks_filechunk; c; c = next) { 410 next = c->next; 411 chunk_free(c); 412 } 413 chunks_filechunk = NULL; 414 } 415 416 void chunkqueue_chunk_pool_free(void) 417 { 418 chunkqueue_chunk_pool_clear(); 419 for (chunk *next, *c = chunk_buffers; c; c = next) { 420 next = c->next; 421 #if 1 /*(chunk_buffers contains MEM_CHUNK with (c->mem == NULL))*/ 422 free(c); 423 #else /*(c->mem = buffer_init() is no longer necessary below)*/ 424 c->mem = buffer_init(); /*(chunk_reset() expects c->mem != NULL)*/ 425 chunk_free(c); 426 #endif 427 } 428 chunk_buffers = NULL; 429 } 430 431 __attribute_pure__ 432 static off_t chunk_remaining_length(const chunk *c) { 433 /* MEM_CHUNK or FILE_CHUNK */ 434 return (c->type == MEM_CHUNK 435 ? (off_t)buffer_clen(c->mem) 436 : c->file.length) 437 - c->offset; 438 } 439 440 static void chunkqueue_release_chunks(chunkqueue *cq) { 441 cq->last = NULL; 442 for (chunk *c; (c = cq->first); ) { 443 cq->first = c->next; 444 chunk_release(c); 445 } 446 } 447 448 void chunkqueue_free(chunkqueue *cq) { 449 if (NULL == cq) return; 450 chunkqueue_release_chunks(cq); 451 free(cq); 452 } 453 454 static void chunkqueue_prepend_chunk(chunkqueue * const restrict cq, chunk * const restrict c) { 455 if (NULL == (c->next = cq->first)) cq->last = c; 456 cq->first = c; 457 } 458 459 static void chunkqueue_append_chunk(chunkqueue * const restrict cq, chunk * const restrict c) { 460 c->next = NULL; 461 *(cq->last ? &cq->last->next : &cq->first) = c; 462 cq->last = c; 463 } 464 465 __attribute_returns_nonnull__ 466 static chunk * chunkqueue_prepend_mem_chunk(chunkqueue *cq, size_t sz) { 467 chunk *c = chunk_acquire(sz); 468 chunkqueue_prepend_chunk(cq, c); 469 return c; 470 } 471 472 __attribute_returns_nonnull__ 473 static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq, size_t sz) { 474 chunk *c = chunk_acquire(sz); 475 chunkqueue_append_chunk(cq, c); 476 return c; 477 } 478 479 __attribute_nonnull__() 480 __attribute_returns_nonnull__ 481 static chunk * chunkqueue_append_file_chunk(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) { 482 chunk * const c = chunk_acquire_filechunk(); 483 chunkqueue_append_chunk(cq, c); 484 c->type = FILE_CHUNK; 485 c->offset = offset; 486 c->file.length = offset + len; 487 cq->bytes_in += len; 488 buffer_copy_buffer(c->mem, fn); 489 return c; 490 } 491 492 void chunkqueue_reset(chunkqueue *cq) { 493 chunkqueue_release_chunks(cq); 494 cq->bytes_in = 0; 495 cq->bytes_out = 0; 496 cq->tempdir_idx = 0; 497 } 498 499 void chunkqueue_append_file_fd(chunkqueue * const restrict cq, const buffer * const restrict fn, int fd, off_t offset, off_t len) { 500 if (len > 0) { 501 (chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd; 502 } 503 else { 504 close(fd); 505 } 506 } 507 508 void chunkqueue_append_file(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) { 509 if (len > 0) { 510 chunkqueue_append_file_chunk(cq, fn, offset, len); 511 } 512 } 513 514 515 static int chunkqueue_append_mem_extend_chunk(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 516 chunk *c = cq->last; 517 if (0 == len) return 1; 518 if (c != NULL && c->type == MEM_CHUNK 519 && buffer_string_space(c->mem) >= len) { 520 buffer_append_string_len(c->mem, mem, len); 521 cq->bytes_in += len; 522 return 1; 523 } 524 return 0; 525 } 526 527 528 void chunkqueue_append_buffer(chunkqueue * const restrict cq, buffer * const restrict mem) { 529 chunk *c; 530 const size_t len = buffer_clen(mem); 531 if (len < 1024 && chunkqueue_append_mem_extend_chunk(cq, mem->ptr, len)) { 532 buffer_clear(mem); 533 return; 534 } 535 536 c = chunkqueue_append_mem_chunk(cq, chunk_buf_sz); 537 cq->bytes_in += len; 538 buffer_move(c->mem, mem); 539 } 540 541 542 void chunkqueue_append_mem(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 543 chunk *c; 544 if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len)) 545 return; 546 547 c = chunkqueue_append_mem_chunk(cq, len+1); 548 cq->bytes_in += len; 549 buffer_copy_string_len(c->mem, mem, len); 550 } 551 552 553 void chunkqueue_append_mem_min(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 554 chunk *c; 555 if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len)) 556 return; 557 558 c = chunk_init_sz(len+1); 559 chunkqueue_append_chunk(cq, c); 560 cq->bytes_in += len; 561 buffer_copy_string_len(c->mem, mem, len); 562 } 563 564 565 void chunkqueue_append_chunkqueue(chunkqueue * const restrict cq, chunkqueue * const restrict src) { 566 if (NULL == src->first) return; 567 568 if (NULL == cq->first) { 569 cq->first = src->first; 570 } else { 571 cq->last->next = src->first; 572 } 573 cq->last = src->last; 574 cq->bytes_in += chunkqueue_length(src); 575 576 src->first = NULL; 577 src->last = NULL; 578 src->bytes_out = src->bytes_in; 579 } 580 581 582 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz) { 583 chunk * const c = chunkqueue_prepend_mem_chunk(cq, sz); 584 return c->mem; 585 } 586 587 588 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq) { 589 return chunkqueue_prepend_buffer_open_sz(cq, chunk_buf_sz); 590 } 591 592 593 void chunkqueue_prepend_buffer_commit(chunkqueue *cq) { 594 cq->bytes_in += buffer_clen(cq->first->mem); 595 } 596 597 598 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz) { 599 chunk * const c = chunkqueue_append_mem_chunk(cq, sz); 600 return c->mem; 601 } 602 603 604 buffer * chunkqueue_append_buffer_open(chunkqueue *cq) { 605 return chunkqueue_append_buffer_open_sz(cq, chunk_buf_sz); 606 } 607 608 609 void chunkqueue_append_buffer_commit(chunkqueue *cq) { 610 cq->bytes_in += buffer_clen(cq->last->mem); 611 } 612 613 614 char * chunkqueue_get_memory(chunkqueue * const restrict cq, size_t * const restrict len) { 615 size_t sz = *len ? *len : (chunk_buf_sz >> 1); 616 buffer *b; 617 chunk *c = cq->last; 618 if (NULL != c && MEM_CHUNK == c->type) { 619 /* return pointer into existing buffer if large enough */ 620 size_t avail = buffer_string_space(c->mem); 621 if (avail >= sz) { 622 *len = avail; 623 b = c->mem; 624 return b->ptr + buffer_clen(b); 625 } 626 } 627 628 /* allocate new chunk */ 629 b = chunkqueue_append_buffer_open_sz(cq, sz); 630 *len = buffer_string_space(b); 631 return b->ptr; 632 } 633 634 void chunkqueue_use_memory(chunkqueue * const restrict cq, chunk *ckpt, size_t len) { 635 buffer *b = cq->last->mem; 636 637 if (len > 0) { 638 buffer_commit(b, len); 639 cq->bytes_in += len; 640 if (cq->last == ckpt || NULL == ckpt || MEM_CHUNK != ckpt->type 641 || len > buffer_string_space(ckpt->mem)) return; 642 643 buffer_append_string_buffer(ckpt->mem, b); 644 } 645 else if (!buffer_is_blank(b)) { /*(cq->last == ckpt)*/ 646 return; /* last chunk is not empty */ 647 } 648 649 /* remove empty last chunk */ 650 chunk_release(cq->last); 651 cq->last = ckpt; 652 *(ckpt ? &ckpt->next : &cq->first) = NULL; 653 } 654 655 void chunkqueue_update_file(chunkqueue * const restrict cq, chunk *c, off_t len) { 656 /*assert(c->type == FILE_CHUNK);*/ 657 c->file.length += len; 658 cq->bytes_in += len; 659 if (0 == chunk_remaining_length(c)) 660 chunkqueue_remove_empty_chunks(cq); 661 } 662 663 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size) { 664 if (upload_temp_file_size == 0) 665 upload_temp_file_size = DEFAULT_TEMPFILE_SIZE; 666 chunkqueue_default_tempdirs = tempdirs; 667 chunkqueue_default_tempfile_size = upload_temp_file_size; 668 } 669 670 void chunkqueue_set_tempdirs(chunkqueue * const restrict cq, const array * const restrict tempdirs, off_t upload_temp_file_size) { 671 if (upload_temp_file_size == 0) 672 upload_temp_file_size = chunkqueue_default_tempfile_size; 673 cq->tempdirs = tempdirs; 674 cq->upload_temp_file_size = upload_temp_file_size; 675 cq->tempdir_idx = 0; 676 } 677 678 __attribute_noinline__ 679 static void chunkqueue_dup_file_chunk_fd (chunk * const restrict d, const chunk * const restrict c) { 680 /*assert(d != c);*/ 681 /*assert(d->type == FILE_CHUNK);*/ 682 /*assert(c->type == FILE_CHUNK);*/ 683 if (c->file.fd >= 0) { 684 if (c->file.refchg) { 685 d->file.fd = c->file.fd; 686 d->file.ref = c->file.ref; 687 d->file.refchg = c->file.refchg; 688 d->file.refchg(d->file.ref, 1); 689 } 690 else 691 d->file.fd = fdevent_dup_cloexec(c->file.fd); 692 #ifdef HAVE_MMAP 693 if ((d->file.view = c->file.view)) 694 ++d->file.view->refcnt; 695 #endif 696 } 697 } 698 699 __attribute_noinline__ 700 static void chunkqueue_steal_partial_file_chunk(chunkqueue * const restrict dest, const chunk * const restrict c, const off_t len) { 701 chunkqueue_append_file(dest, c->mem, c->offset, len); 702 chunkqueue_dup_file_chunk_fd(dest->last, c); 703 } 704 705 void chunkqueue_steal(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len) { 706 /*(0-length first chunk (unexpected) is removed from src even if len == 0; 707 * progress is made when caller loops on this func)*/ 708 off_t clen; 709 do { 710 chunk * const c = src->first; 711 if (__builtin_expect( (NULL == c), 0)) break; 712 713 clen = chunk_remaining_length(c); 714 715 if (len >= clen) { 716 /* move complete chunk */ 717 src->first = c->next; 718 if (c == src->last) src->last = NULL; 719 720 if (__builtin_expect( (0 != clen), 1)) { 721 chunkqueue_append_chunk(dest, c); 722 dest->bytes_in += clen; 723 } 724 else /* drop empty chunk */ 725 chunk_release(c); 726 } else { 727 /* copy partial chunk */ 728 729 switch (c->type) { 730 case MEM_CHUNK: 731 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, len); 732 break; 733 case FILE_CHUNK: 734 /* tempfile flag is in "last" chunk after the split */ 735 chunkqueue_steal_partial_file_chunk(dest, c, len); 736 break; 737 } 738 739 c->offset += len; 740 clen = len; 741 } 742 743 src->bytes_out += clen; 744 } while ((len -= clen)); 745 } 746 747 static int chunkqueue_get_append_mkstemp(buffer * const b, const char *path, const uint32_t len) { 748 buffer_copy_path_len2(b,path,len,CONST_STR_LEN("lighttpd-upload-XXXXXX")); 749 #if defined(HAVE_SPLICE) && defined(HAVE_PWRITE) 750 /*(splice() rejects O_APPEND target; omit flag if also using pwrite())*/ 751 return fdevent_mkostemp(b->ptr, 0); 752 #else 753 return fdevent_mkostemp(b->ptr, O_APPEND); 754 #endif 755 } 756 757 static chunk *chunkqueue_get_append_newtempfile(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 758 static const buffer emptyb = { "", 0, 0 }; 759 chunk * const restrict last = cq->last; 760 chunk * const restrict c = chunkqueue_append_file_chunk(cq, &emptyb, 0, 0); 761 buffer * const restrict template = c->mem; 762 c->file.is_temp = 1; 763 764 if (cq->tempdirs && cq->tempdirs->used) { 765 /* we have several tempdirs, only if all of them fail we jump out */ 766 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) { 767 data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx]; 768 c->file.fd = 769 chunkqueue_get_append_mkstemp(template, BUF_PTR_LEN(&ds->value)); 770 if (-1 != c->file.fd) return c; 771 } 772 } 773 else { 774 c->file.fd = 775 chunkqueue_get_append_mkstemp(template, CONST_STR_LEN("/var/tmp")); 776 if (-1 != c->file.fd) return c; 777 } 778 779 /* (report only last error to mkstemp() even if multiple temp dirs tried) */ 780 log_perror(errh, __FILE__, __LINE__, 781 "opening temp-file failed: %s", template->ptr); 782 /* remove (failed) final chunk */ 783 c->file.is_temp = 0; 784 if ((cq->last = last)) 785 last->next = NULL; 786 else 787 cq->first = NULL; 788 chunk_release(c); 789 return NULL; 790 } 791 792 static chunk *chunkqueue_get_append_tempfile(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 793 /* 794 * if the last chunk is 795 * - smaller than cq->upload_temp_file_size 796 * -> append to it (and it then might exceed cq->upload_temp_file_size) 797 * otherwise 798 * -> create a new chunk 799 */ 800 801 chunk * const c = cq->last; 802 if (NULL != c && c->file.is_temp && c->file.fd >= 0) { 803 804 if (c->file.length < (off_t)cq->upload_temp_file_size) 805 return c; /* ok, take the last chunk for our job */ 806 807 /* the chunk is too large now, close it */ 808 force_assert(0 == c->file.refchg); /*(else should not happen)*/ 809 int rc = close(c->file.fd); 810 c->file.fd = -1; 811 if (0 != rc) { 812 log_perror(errh, __FILE__, __LINE__, 813 "close() temp-file %s failed", c->mem->ptr); 814 return NULL; 815 } 816 } 817 return chunkqueue_get_append_newtempfile(cq, errh); 818 } 819 820 __attribute_cold__ 821 static int chunkqueue_append_tempfile_err(chunkqueue * const cq, log_error_st * const restrict errh, chunk * const c) { 822 const int errnum = errno; 823 if (errnum == EINTR) return 1; /* retry */ 824 825 int retry = (errnum == ENOSPC && cq->tempdirs 826 && ++cq->tempdir_idx < cq->tempdirs->used); 827 if (!retry) 828 log_perror(errh, __FILE__, __LINE__, 829 "write() temp-file %s failed", c->mem->ptr); 830 831 if (0 == chunk_remaining_length(c)) { 832 /*(remove empty chunk and unlink tempfile)*/ 833 chunkqueue_remove_empty_chunks(cq); 834 } 835 else {/*(close tempfile; avoid later attempts to append)*/ 836 force_assert(0 == c->file.refchg); /*(else should not happen)*/ 837 int rc = close(c->file.fd); 838 c->file.fd = -1; 839 if (0 != rc) { 840 log_perror(errh, __FILE__, __LINE__, 841 "close() temp-file %s failed", c->mem->ptr); 842 retry = 0; 843 } 844 } 845 return retry; 846 } 847 848 __attribute_cold__ 849 __attribute_noinline__ 850 static int chunkqueue_to_tempfiles(chunkqueue * const restrict dest, log_error_st * const restrict errh) { 851 /* transfer chunks from dest to src, adjust dest->bytes_in, and then call 852 * chunkqueue_steal_with_tempfiles() to write chunks from src back into 853 * dest, but into tempfiles. chunkqueue_steal_with_tempfiles() calls back 854 * into chunkqueue_append_mem_to_tempfile(), but will not re-enter this func 855 * since chunks moved to src, and dest made empty before recursive call */ 856 const off_t cqlen = chunkqueue_length(dest); 857 chunkqueue src = *dest; /*(copy struct)*/ 858 dest->first = dest->last = NULL; 859 dest->bytes_in -= cqlen; 860 if (0 == chunkqueue_steal_with_tempfiles(dest, &src, cqlen, errh)) 861 return 0; 862 else { 863 const int errnum = errno; 864 chunkqueue_release_chunks(&src); 865 return -errnum; 866 } 867 } 868 869 int chunkqueue_append_mem_to_tempfile(chunkqueue * const restrict dest, const char * restrict mem, size_t len, log_error_st * const restrict errh) { 870 chunk *dst_c = dest->first; 871 872 /* check if prior MEM_CHUNK(s) exist and write to tempfile 873 * (check first chunk only, since if we are using tempfiles, then 874 * we expect further chunks to be tempfiles after starting tempfiles)*/ 875 if (dst_c && dst_c->type == MEM_CHUNK 876 && 0 != chunkqueue_to_tempfiles(dest, errh)) { 877 return -1; 878 } 879 880 do { 881 /*(aside: arg len is permitted to be 0 and creates tempfile as a 882 * side effect. This is used by mod_ssi for ssi exec, as the func 883 * chunkqueue_get_append_tempfile() is not public. The result is 884 * an empty chunk at the end of the chunkqueue, which typically 885 * should be avoided)*/ 886 dst_c = chunkqueue_get_append_tempfile(dest, errh); 887 if (NULL == dst_c) 888 return -1; 889 #ifdef __COVERITY__ 890 if (dst_c->file.fd < 0) return -1; 891 #endif 892 #ifdef HAVE_PWRITE 893 /* coverity[negative_returns : FALSE] */ 894 const ssize_t written =pwrite(dst_c->file.fd, mem, len, dst_c->file.length); 895 #else 896 /* coverity[negative_returns : FALSE] */ 897 const ssize_t written = write(dst_c->file.fd, mem, len); 898 #endif 899 900 if ((size_t) written == len) { 901 dst_c->file.length += len; 902 dest->bytes_in += len; 903 return 0; 904 } else if (written >= 0) { 905 /*(assume EINTR if partial write and retry write(); 906 * retry write() might fail with ENOSPC if no more space on volume)*/ 907 dest->bytes_in += written; 908 mem += written; 909 len -= (size_t)written; 910 dst_c->file.length += (size_t)written; 911 /* continue; retry */ 912 } else if (!chunkqueue_append_tempfile_err(dest, errh, dst_c)) { 913 break; /* return -1; */ 914 } /* else continue; retry */ 915 } while (len); 916 917 return -1; 918 } 919 920 #ifdef HAVE_PWRITEV 921 922 #ifdef HAVE_SYS_UIO_H 923 #include <sys/uio.h> 924 #endif 925 926 __attribute_cold__ 927 __attribute_noinline__ 928 static ssize_t chunkqueue_append_cqmem_to_tempfile_partial(chunkqueue * const dest, chunk * const c, ssize_t wr, log_error_st * const restrict errh) { 929 /* recover from partial write of existing dest MEM_CHUNK to tempfile */ 930 chunk *ckpt = dest->first; 931 while (ckpt->next != c) ckpt = ckpt->next; 932 ckpt->next = NULL; 933 dest->last = ckpt; 934 dest->bytes_in -= wr; /*(avoid double count in dest cq)*/ 935 dest->bytes_out -= wr; 936 chunkqueue_mark_written(dest, wr);/*(remove MEM_CHUNK written to tempfile)*/ 937 938 c->next = dest->first; /*(place tempfile at beginning of dest cq)*/ 939 dest->first = c; 940 return (0 == chunkqueue_to_tempfiles(dest, errh)) ? 0 : -1; 941 } 942 943 static ssize_t chunkqueue_append_cqmem_to_tempfile(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len, log_error_st * const restrict errh) { 944 /* write multiple MEM_CHUNKs to tempfile in single pwritev() syscall */ 945 /*(could lseek() and writev() if pwritev() is not available, 946 * but if writev() is available, pwritev() is likely available, 947 * e.g. any modern Linux or *BSD, and possibly anything not Windows)*/ 948 unsigned int iovcnt = 0; 949 struct iovec iov[16]; 950 951 off_t dlen = 0; 952 chunk *c; 953 for (c = dest->first; c && c->type == MEM_CHUNK; c = c->next) { 954 const off_t clen = chunk_remaining_length(c); 955 iov[iovcnt].iov_base = c->mem->ptr + c->offset; 956 iov[iovcnt].iov_len = (size_t)clen; 957 dlen += clen; 958 ++iovcnt; 959 if (__builtin_expect( (iovcnt == sizeof(iov)/sizeof(*iov)), 0)) 960 break; /*(not expecting large number of MEM_CHUNK)*/ 961 } 962 if (__builtin_expect( (c != NULL), 0) && dest->first->type == MEM_CHUNK) { 963 /*(expecting only MEM_CHUNK if dest cq starts w/ MEM_CHUNK)*/ 964 /*(use less efficient fallback if that assumption does not hold true)*/ 965 if (0 != chunkqueue_to_tempfiles(dest, errh)) 966 return -1; 967 dlen = 0; 968 iovcnt = 0; 969 } 970 971 if (__builtin_expect( (iovcnt < sizeof(iov)/sizeof(*iov)), 1)) { 972 for (c = src->first; c && c->type == MEM_CHUNK; c = c->next) { 973 off_t clen = chunk_remaining_length(c); 974 if (clen > len) clen = len; 975 iov[iovcnt].iov_base = c->mem->ptr + c->offset; 976 iov[iovcnt].iov_len = (size_t)clen; 977 len -= clen; 978 ++iovcnt; 979 if (0 == len) break; 980 if (__builtin_expect( (iovcnt == sizeof(iov)/sizeof(*iov)), 0)) 981 break; /*(not expecting large number of MEM_CHUNK)*/ 982 } 983 } 984 985 if (__builtin_expect( (0 == iovcnt), 0)) return 0; /*(should not happen)*/ 986 987 c = chunkqueue_get_append_tempfile(dest, errh); 988 if (NULL == c) 989 return -1; 990 #ifdef __COVERITY__ 991 if (c->file.fd < 0) return -1; 992 #endif 993 /* coverity[negative_returns : FALSE] */ 994 ssize_t wr = pwritev(c->file.fd, iov, (int)iovcnt, c->file.length); 995 996 /*(memory use in chunkqueues is expected to be limited before spilling 997 * to tempfiles, so common case will write entire iovec to tempfile, 998 * and we return amount written *from src cq*, even if partial write; 999 * (not looping here to retry writing more, but caller might loop))*/ 1000 1001 if (wr >= 0) { 1002 c->file.length += wr; 1003 dest->bytes_in += wr; 1004 if (dlen) { 1005 if (__builtin_expect( (wr < dlen), 0)) 1006 return 1007 chunkqueue_append_cqmem_to_tempfile_partial(dest,c,wr,errh); 1008 wr -= (ssize_t)dlen; 1009 dest->bytes_in -= dlen; /*(avoid double count in dest cq)*/ 1010 dest->bytes_out -= dlen; 1011 chunkqueue_mark_written(dest, dlen); 1012 } 1013 } 1014 1015 return wr; 1016 } 1017 1018 #endif /* HAVE_PWRITEV */ 1019 1020 #ifdef HAVE_SPLICE 1021 1022 __attribute_cold__ 1023 __attribute_noinline__ 1024 static ssize_t chunkqueue_append_drain_pipe_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1025 /* attempt to drain full 'len' from pipe 1026 * (even if len not reduced to opts->max_per_read limit) 1027 * since data may have already been moved from socket to pipe 1028 *(returns 0 on success, or -errno (negative errno) if error, 1029 * even if partial write occurred)*/ 1030 char buf[16384]; 1031 ssize_t rd; 1032 do { 1033 do { 1034 rd = read(fd, buf, sizeof(buf)); 1035 } while (rd < 0 && errno == EINTR); 1036 if (rd < 0) break; 1037 if (0 != chunkqueue_append_mem_to_tempfile(cq, buf, (size_t)rd, errh)) 1038 break; 1039 } while ((len -= (unsigned int)rd)); 1040 1041 if (0 == len) 1042 return 0; 1043 else { 1044 const int errnum = errno; 1045 if (cq->last && 0 == chunk_remaining_length(cq->last)) { 1046 /*(remove empty chunk and unlink tempfile)*/ 1047 chunkqueue_remove_empty_chunks(cq); 1048 } 1049 return -errnum; 1050 } 1051 } 1052 1053 ssize_t chunkqueue_append_splice_pipe_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1054 /* check if prior MEM_CHUNK(s) exist and write to tempfile 1055 * (check first chunk only, since if we are using tempfiles, then 1056 * we expect further chunks to be tempfiles after starting tempfiles)*/ 1057 if (cq->first && cq->first->type == MEM_CHUNK) { 1058 int rc = chunkqueue_to_tempfiles(cq, errh); 1059 if (__builtin_expect( (0 != rc), 0)) return rc; 1060 } 1061 1062 /*(returns num bytes written, or -errno (negative errno) if error)*/ 1063 ssize_t total = 0; 1064 do { 1065 chunk * const c = chunkqueue_get_append_tempfile(cq, errh); 1066 if (__builtin_expect( (NULL == c), 0)) return -errno; 1067 1068 loff_t off = c->file.length; 1069 ssize_t wr = splice(fd, NULL, c->file.fd, &off, len, 1070 SPLICE_F_MOVE | SPLICE_F_NONBLOCK); 1071 1072 if (__builtin_expect(((size_t)wr == len), 1)) { 1073 c->file.length += len; 1074 cq->bytes_in += len; 1075 return total + len; 1076 } 1077 else if (wr >= 0) { 1078 /*(assume EINTR if partial write and retry; 1079 * retry might fail with ENOSPC if no more space on volume)*/ 1080 cq->bytes_in += wr; 1081 total += wr; 1082 len -= (size_t)wr; 1083 c->file.length += (size_t)wr; 1084 /* continue; retry */ 1085 } 1086 else { 1087 const int errnum = errno; 1088 switch (errnum) { 1089 case EAGAIN: 1090 #ifdef EWOULDBLOCK 1091 #if EWOULDBLOCK != EAGAIN 1092 case EWOULDBLOCK: 1093 #endif 1094 #endif 1095 if (0 == chunk_remaining_length(c)) { 1096 /*(remove empty chunk and unlink tempfile)*/ 1097 chunkqueue_remove_empty_chunks(cq); 1098 } 1099 return total; 1100 case EINVAL: /*(assume total == 0 if EINVAL)*/ 1101 wr = chunkqueue_append_drain_pipe_tempfile(cq, fd, len, errh); 1102 return (0 == wr) ? total + (ssize_t)len : wr; 1103 default: 1104 if (!chunkqueue_append_tempfile_err(cq, errh, c)) 1105 return -errnum; 1106 break; /* else continue; retry */ 1107 } 1108 } 1109 } while (len); 1110 return -EIO; /*(not reached)*/ 1111 } 1112 1113 static int cqpipes[2] = { -1, -1 }; 1114 1115 __attribute_cold__ 1116 __attribute_noinline__ 1117 void chunkqueue_internal_pipes(int init) { 1118 /*(intended for internal use within a single lighttpd process; 1119 * must be initialized after fork() and graceful-restart to avoid 1120 * sharing pipes between processes)*/ 1121 if (-1 != cqpipes[0]) { close(cqpipes[0]); cqpipes[0] = -1; } 1122 if (-1 != cqpipes[1]) { close(cqpipes[1]); cqpipes[1] = -1; } 1123 if (init) 1124 if (0 != fdevent_pipe_cloexec(cqpipes, 262144)) { } /*(ignore error)*/ 1125 } 1126 1127 __attribute_cold__ 1128 __attribute_noinline__ 1129 static void chunkqueue_pipe_read_discard (void) { 1130 char buf[16384]; 1131 ssize_t rd; 1132 do { 1133 rd = read(cqpipes[0], buf, sizeof(buf)); 1134 } while (rd > 0 || (rd < 0 && errno == EINTR)); 1135 if (rd < 0 1136 #ifdef EWOULDBLOCK 1137 #if EWOULDBLOCK != EAGAIN 1138 && errno != EWOULDBLOCK 1139 #endif 1140 #endif 1141 && errno != EAGAIN) { 1142 chunkqueue_internal_pipes(1); /*(close() and re-initialize)*/ 1143 } 1144 } 1145 1146 ssize_t chunkqueue_append_splice_sock_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1147 /*(returns num bytes written, or -errno (negative errno) if error)*/ 1148 int * const pipes = cqpipes; 1149 if (-1 == pipes[1]) 1150 return -EINVAL; /*(not configured; not handled here)*/ 1151 1152 /* splice() socket data to intermediate pipe */ 1153 ssize_t wr = splice(fd, NULL, pipes[1], NULL, len, 1154 SPLICE_F_MOVE | SPLICE_F_NONBLOCK); 1155 if (__builtin_expect( (wr <= 0), 0)) 1156 return -EINVAL; /*(reuse to indicate not handled here)*/ 1157 len = (unsigned int)wr; 1158 1159 /* splice() data from intermediate pipe to tempfile */ 1160 wr = chunkqueue_append_splice_pipe_tempfile(cq, pipes[0], len, errh); 1161 if (wr < 0) /* expect (wr == (ssize_t)len) or (wr == -1) */ 1162 chunkqueue_pipe_read_discard();/* discard data from intermediate pipe */ 1163 return wr; 1164 } 1165 1166 #endif /* HAVE_SPLICE */ 1167 1168 int chunkqueue_steal_with_tempfiles(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len, log_error_st * const restrict errh) { 1169 /*(0-length first chunk (unexpected) is removed from src even if len == 0; 1170 * progress is made when caller loops on this func)*/ 1171 off_t clen; 1172 do { 1173 chunk * const c = src->first; 1174 if (__builtin_expect( (NULL == c), 0)) break; 1175 1176 #ifdef HAVE_PWRITEV 1177 1178 if (c->type == MEM_CHUNK) { 1179 clen = chunkqueue_append_cqmem_to_tempfile(dest, src, len, errh); 1180 if (__builtin_expect( (clen < 0), 0)) return -1; 1181 chunkqueue_mark_written(src, clen); 1182 } 1183 else { /* (c->type == FILE_CHUNK) */ 1184 clen = chunk_remaining_length(c); 1185 if (len < clen) clen = len; 1186 chunkqueue_steal(dest, src, clen); 1187 } 1188 1189 #else 1190 1191 clen = chunk_remaining_length(c); 1192 if (len < clen) clen = len; 1193 1194 switch (c->type) { 1195 case FILE_CHUNK: 1196 chunkqueue_steal(dest, src, clen); 1197 break; 1198 1199 case MEM_CHUNK: 1200 /* store bytes from memory chunk in tempfile */ 1201 if (clen 1202 && 0 != chunkqueue_append_mem_to_tempfile(dest, 1203 c->mem->ptr+c->offset, 1204 clen, errh)) 1205 return -1; 1206 chunkqueue_mark_written(src, clen); 1207 break; 1208 } 1209 1210 #endif 1211 } while ((len -= clen)); 1212 1213 return 0; 1214 } 1215 1216 void chunkqueue_append_cq_range (chunkqueue * const dst, const chunkqueue * const src, off_t offset, off_t len) { 1217 /* similar to chunkqueue_steal() but copy and append src range to dst cq */ 1218 /* (dst cq and src cq can be the same cq, so neither is marked restrict) */ 1219 1220 /* copy and append range len from src to dst */ 1221 for (const chunk *c = src->first; len > 0 && c != NULL; c = c->next) { 1222 /* scan into src to range offset (also skips empty chunks) */ 1223 off_t clen = chunk_remaining_length(c); 1224 if (offset >= clen) { 1225 offset -= clen; 1226 continue; 1227 } 1228 clen -= offset; 1229 if (len < clen) clen = len; 1230 len -= clen; 1231 1232 if (c->type == FILE_CHUNK) { 1233 chunkqueue_append_file(dst, c->mem, c->offset + offset, clen); 1234 chunkqueue_dup_file_chunk_fd(dst->last, c); 1235 } 1236 else { /*(c->type == MEM_CHUNK)*/ 1237 /*(string refs would reduce copying, 1238 * but this path is not expected to be hot)*/ 1239 chunkqueue_append_mem(dst, c->mem->ptr + c->offset + offset, clen); 1240 } 1241 offset = 0; 1242 } 1243 } 1244 1245 void chunkqueue_mark_written(chunkqueue *cq, off_t len) { 1246 cq->bytes_out += len; 1247 1248 for (chunk *c = cq->first; c; ) { 1249 off_t c_len = chunk_remaining_length(c); 1250 if (len >= c_len) { /* chunk got finished */ 1251 chunk * const x = c; 1252 c = c->next; 1253 len -= c_len; 1254 chunk_release(x); 1255 } 1256 else { /* partial chunk */ 1257 c->offset += len; 1258 cq->first = c; 1259 return; /* chunk not finished */ 1260 } 1261 } 1262 cq->first = cq->last = NULL; 1263 } 1264 1265 void chunkqueue_remove_finished_chunks(chunkqueue *cq) { 1266 for (chunk *c; (c = cq->first) && 0 == chunk_remaining_length(c); ){ 1267 if (NULL == (cq->first = c->next)) cq->last = NULL; 1268 chunk_release(c); 1269 } 1270 } 1271 1272 void chunkqueue_remove_empty_chunks(chunkqueue *cq) { 1273 chunk *c; 1274 chunkqueue_remove_finished_chunks(cq); 1275 1276 for (c = cq->first; c && c->next; c = c->next) { 1277 if (0 == chunk_remaining_length(c->next)) { 1278 chunk *empty = c->next; 1279 c->next = empty->next; 1280 if (empty == cq->last) cq->last = c; 1281 chunk_release(empty); 1282 } 1283 } 1284 } 1285 1286 void chunkqueue_compact_mem_offset(chunkqueue * const cq) { 1287 chunk * const restrict c = cq->first; 1288 if (0 == c->offset) return; 1289 if (c->type != MEM_CHUNK) return; /*(should not happen)*/ 1290 1291 buffer * const restrict b = c->mem; 1292 size_t len = buffer_clen(b) - c->offset; 1293 memmove(b->ptr, b->ptr+c->offset, len); 1294 c->offset = 0; 1295 buffer_truncate(b, len); 1296 } 1297 1298 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen) { 1299 /* caller must guarantee that chunks in chunkqueue are MEM_CHUNK, 1300 * which is currently always true when reading input from client */ 1301 chunk *c = cq->first; 1302 buffer *b = c->mem; 1303 size_t len = buffer_clen(b) - c->offset; 1304 if (len >= clen) return; 1305 if (b->size > clen) { 1306 if (buffer_string_space(b) < clen - len) 1307 chunkqueue_compact_mem_offset(cq); 1308 } 1309 else { 1310 b = chunkqueue_prepend_buffer_open_sz(cq, clen+1); 1311 buffer_append_string_len(b, c->mem->ptr + c->offset, len); 1312 cq->first->next = c->next; 1313 if (NULL == c->next) cq->last = cq->first; 1314 chunk_release(c); 1315 c = cq->first; 1316 } 1317 1318 for (chunk *fc = c; ((clen -= len) && (c = fc->next)); ) { 1319 len = buffer_clen(c->mem) - c->offset; 1320 if (len > clen) { 1321 buffer_append_string_len(b, c->mem->ptr + c->offset, clen); 1322 c->offset += clen; 1323 break; 1324 } 1325 buffer_append_string_len(b, c->mem->ptr + c->offset, len); 1326 fc->next = c->next; 1327 if (NULL == c->next) cq->last = fc; 1328 chunk_release(c); 1329 } 1330 /* chunkqueue_prepend_buffer_commit() is not called here; 1331 * no data added/removed from chunkqueue; consolidated only */ 1332 } 1333 1334 static int chunk_open_file_chunk(chunk * const restrict c, log_error_st * const restrict errh) { 1335 if (-1 == c->file.fd) { 1336 /* (permit symlinks; should already have been checked. However, TOC-TOU remains) */ 1337 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0))) { 1338 log_perror(errh, __FILE__, __LINE__, "open failed: %s",c->mem->ptr); 1339 return -1; 1340 } 1341 } 1342 1343 /*(skip file size checks if file is temp file created by lighttpd)*/ 1344 if (c->file.is_temp) return 0; 1345 1346 struct stat st; 1347 if (-1 == fstat(c->file.fd, &st)) { 1348 log_perror(errh, __FILE__, __LINE__, "fstat failed"); 1349 return -1; 1350 } 1351 1352 const off_t offset = c->offset; 1353 const off_t len = c->file.length - c->offset; 1354 force_assert(offset >= 0 && len >= 0); 1355 if (offset > st.st_size - len) { 1356 log_error(errh, __FILE__, __LINE__, "file shrunk: %s", c->mem->ptr); 1357 return -1; 1358 } 1359 1360 return 0; 1361 } 1362 1363 int chunkqueue_open_file_chunk(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 1364 return chunk_open_file_chunk(cq->first, errh); 1365 } 1366 1367 1368 static ssize_t 1369 chunkqueue_write_data (const int fd, const void *buf, size_t len) 1370 { 1371 ssize_t wr = 0; 1372 if (len) 1373 do { wr = write(fd, buf, len); } while (-1 == wr && errno == EINTR); 1374 return wr; 1375 } 1376 1377 1378 #ifdef HAVE_MMAP 1379 __attribute_cold__ 1380 #endif 1381 __attribute_noinline__ 1382 static ssize_t 1383 chunkqueue_write_chunk_file_intermed (const int fd, chunk * const restrict c, log_error_st * const errh) 1384 { 1385 char buf[16384]; 1386 char *data = buf; 1387 const off_t len = c->file.length - c->offset; 1388 uint32_t dlen = len < (off_t)sizeof(buf) ? (uint32_t)len : sizeof(buf); 1389 chunkqueue cq = {c,c,0,0,0,0,0}; /*(fake cq for chunkqueue_peek_data())*/ 1390 if (0 != chunkqueue_peek_data(&cq, &data, &dlen, errh) && 0 == dlen) 1391 return -1; 1392 return chunkqueue_write_data(fd, data, dlen); 1393 } 1394 1395 1396 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 1397 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 1398 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 1399 #include <sys/sendfile.h> 1400 #include <stdint.h> 1401 #endif 1402 static ssize_t 1403 chunkqueue_write_chunk_file (const int fd, chunk * const restrict c, log_error_st * const errh) 1404 { 1405 /*(similar to network_write_file_chunk_mmap(), but does not use send() on 1406 * Windows because fd is expected to be file or pipe here, not socket)*/ 1407 1408 if (0 != chunk_open_file_chunk(c, errh)) 1409 return -1; 1410 1411 const off_t len = c->file.length - c->offset; 1412 if (0 == len) return 0; /*(sanity check)*/ 1413 1414 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 1415 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 1416 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 1417 /* Linux kernel >= 2.6.33 supports sendfile() between most fd types */ 1418 off_t offset = c->offset; 1419 const ssize_t wr = 1420 sendfile(fd, c->file.fd, &offset, len < INT32_MAX ? len : INT32_MAX); 1421 if (__builtin_expect( (wr >= 0), 1) || (errno != EINVAL && errno != ENOSYS)) 1422 return wr; 1423 /*(could fallback to mmap, but if sendfile fails on linux, mmap may, too)*/ 1424 #elif defined(HAVE_MMAP) 1425 /*(chunkqueue_write_chunk() caller must protect against SIGBUS, if needed)*/ 1426 const chunk_file_view * const restrict cfv = 1427 chunkqueue_chunk_file_view(c, len, errh); 1428 if (NULL != cfv) { 1429 const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); 1430 return chunkqueue_write_data(fd, chunk_file_view_dptr(cfv, c->offset), 1431 len <= mmap_avail ? len : mmap_avail); 1432 } 1433 #endif 1434 1435 return chunkqueue_write_chunk_file_intermed(fd, c, errh); 1436 } 1437 1438 1439 static ssize_t 1440 chunkqueue_write_chunk_mem (const int fd, const chunk * const restrict c) 1441 { 1442 const void * const buf = c->mem->ptr + c->offset; 1443 const size_t len = buffer_clen(c->mem) - (size_t)c->offset; 1444 return chunkqueue_write_data(fd, buf, len); 1445 } 1446 1447 1448 ssize_t 1449 chunkqueue_write_chunk (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh) 1450 { 1451 /*(note: expects non-empty cq->first)*/ 1452 chunk * const c = cq->first; 1453 switch (c->type) { 1454 case MEM_CHUNK: 1455 return chunkqueue_write_chunk_mem(fd, c); 1456 case FILE_CHUNK: 1457 return chunkqueue_write_chunk_file(fd, c, errh); 1458 default: 1459 errno = EINVAL; 1460 return -1; 1461 } 1462 } 1463 1464 1465 ssize_t 1466 chunkqueue_write_chunk_to_pipe (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh) 1467 { 1468 #ifdef HAVE_SPLICE /* splice() temp files to pipe on Linux */ 1469 chunk * const c = cq->first; 1470 if (c->type == FILE_CHUNK) { 1471 const size_t len = (size_t)(c->file.length - c->offset); 1472 loff_t abs_offset = c->offset; 1473 if (__builtin_expect( (0 == len), 0)) return 0; 1474 return (0 == chunk_open_file_chunk(c, errh)) 1475 ? splice(c->file.fd, &abs_offset, fd, NULL, len, SPLICE_F_NONBLOCK) 1476 : -1; 1477 } 1478 #endif 1479 return chunkqueue_write_chunk(fd, cq, errh); 1480 } 1481 1482 1483 void 1484 chunkqueue_small_resp_optim (chunkqueue * const restrict cq) 1485 { 1486 /*(caller must verify response is small (and non-empty) before calling)*/ 1487 /*(caller must verify first chunk is MEM_CHUNK, i.e. response headers)*/ 1488 /*(caller must verify response is non-zero length)*/ 1489 1490 /*(optimization to use fewer syscalls to send a small response by reading 1491 * small files into memory, thereby avoiding use of sendfile() and multiple 1492 * calls to writev() (benefit for cleartext (non-TLS) and <= HTTP/1.1)) 1493 *(If TLS, then will shortly need to be in memory for encryption anyway)*/ 1494 1495 /*assert(cq->first);*/ 1496 /*assert(cq->first->type == MEM_CHUNK);*/ 1497 /*assert(cq->first->next);*/ 1498 chunk * restrict c = cq->first; 1499 chunk * const restrict filec = c->next; /*(require file already be open)*/ 1500 if (filec != cq->last || filec->type != FILE_CHUNK || filec->file.fd < 0) 1501 return; 1502 1503 /* Note: there should be no size change in chunkqueue, 1504 * so cq->bytes_in and cq->bytes_out should not be modified */ 1505 1506 off_t len = filec->file.length - filec->offset; 1507 if ((size_t)len > buffer_string_space(c->mem)) { 1508 c->next = chunk_acquire((size_t)len+1); 1509 c = c->next; 1510 /*c->next = filec;*/ 1511 } 1512 /* detach filec from chunkqueue; file expected to be read fully */ 1513 c->next = NULL; 1514 cq->last = c; 1515 1516 ssize_t rd; 1517 off_t offset = 0; 1518 char * const ptr = buffer_extend(c->mem, len); 1519 do { 1520 rd = chunk_file_pread(filec->file.fd, ptr+offset, (size_t)len, 1521 filec->offset+offset); 1522 } while (rd > 0 && (offset += rd, len -= rd)); 1523 /*(contents of chunkqueue kept valid even if error reading from file)*/ 1524 if (__builtin_expect( (0 == len), 1)) 1525 chunk_release(filec); 1526 else { /*(unexpected; error recovery)*/ 1527 buffer_truncate(c->mem, (uint32_t)(ptr + offset - c->mem->ptr)); 1528 cq->last = c->next = filec; 1529 if (offset) 1530 filec->offset += offset; 1531 else if (__builtin_expect( (cq->first != c), 0)) { 1532 cq->first->next = filec; 1533 chunk_release(c); 1534 } 1535 } 1536 } 1537 1538 1539 #if 0 1540 #ifdef HAVE_MMAP 1541 __attribute_noinline__ 1542 static off_t 1543 chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len) 1544 { 1545 /*(on 32-bit systems, caller should assert len <= SIZE_MAX)*/ 1546 memcpy(dst, src, (size_t)len); 1547 return len; 1548 } 1549 #endif 1550 #endif 1551 1552 1553 int 1554 chunkqueue_peek_data (chunkqueue * const cq, 1555 char ** const data, uint32_t * const dlen, 1556 log_error_st * const errh) 1557 { 1558 char * const data_in = *data; 1559 const uint32_t data_insz = *dlen; 1560 *dlen = 0; 1561 1562 for (chunk *c = cq->first; c; ) { 1563 uint32_t space = data_insz - *dlen; 1564 switch (c->type) { 1565 case MEM_CHUNK: 1566 { 1567 uint32_t have = buffer_clen(c->mem) - (uint32_t)c->offset; 1568 if (have > space) 1569 have = space; 1570 if (__builtin_expect( (0 == have), 0)) 1571 break; 1572 if (*dlen) 1573 memcpy(data_in + *dlen, c->mem->ptr + c->offset, have); 1574 else 1575 *data = c->mem->ptr + c->offset; /*(reference; defer copy)*/ 1576 *dlen += have; 1577 break; 1578 } 1579 1580 case FILE_CHUNK: 1581 if (c->file.fd >= 0 || 0 == chunk_open_file_chunk(c, errh)) { 1582 off_t len = c->file.length - c->offset; 1583 if (len > (off_t)space) 1584 len = (off_t)space; 1585 if (__builtin_expect( (0 == len), 0)) 1586 break; 1587 1588 #if 0 /* XXX: might improve performance on some system workloads */ 1589 #ifdef HAVE_MMAP 1590 /* mmap file to access data 1591 * fd need not be kept open for the mmap once 1592 * the mmap has been created, but is currently kept open for 1593 * other pre-existing logic which checks fd and opens file, 1594 * such as the condition for entering this code block above. */ 1595 /* Note: current use is with somewhat large buffers, e.g. 128k. 1596 * If larger buffers are used, then upper limit, e.g. 512k, 1597 * should be set for 32-bit to avoid address space issues) */ 1598 /* Note: under heavy load (or microbenchmark), system-reported 1599 * memory use for RSS can be very, very large, due to presence 1600 * of lots and lots of temp file read-only memory maps. 1601 * pmap -X and exclude lighttpd mmap files to get a better 1602 * view of memory use */ 1603 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 1604 ? chunkqueue_chunk_file_view(c, len, errh) 1605 : NULL; 1606 if (cfv && chunk_file_view_dlen(cfv, c->offset) >= len) { 1607 /*(check (above) that mapped chunk length >= requested len)*/ 1608 char * const mdata = chunk_file_view_dptr(cfv, c->offset); 1609 if (!c->file.is_temp) {/*(might be changed to policy flag)*/ 1610 if (sys_setjmp_eval3(chunk_setjmp_memcpy_cb, 1611 data_in+*dlen, mdata, len) < 0) { 1612 log_error(errh, __FILE__, __LINE__, 1613 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1614 return -1; 1615 } 1616 } 1617 else if (*dlen) 1618 memcpy(data_in+*dlen, mdata, (size_t)len); 1619 else 1620 *data = mdata; 1621 *dlen += (uint32_t)len; 1622 break; 1623 } 1624 #endif 1625 #endif 1626 1627 ssize_t rd = chunk_file_pread(c->file.fd, data_in+*dlen, 1628 (size_t)len, c->offset); 1629 if (rd <= 0) { /* -1 error; 0 EOF (unexpected) */ 1630 log_perror(errh, __FILE__, __LINE__, "read(\"%s\")", 1631 c->mem->ptr); 1632 return -1; 1633 } 1634 1635 *dlen += (uint32_t)rd; 1636 break; 1637 } 1638 return -1; 1639 1640 default: 1641 return -1; 1642 } 1643 1644 if (*dlen == data_insz) 1645 break; 1646 1647 c = c->next; 1648 if (NULL == c) 1649 break; 1650 1651 if (*dlen && *data != data_in) { 1652 memcpy(data_in, *data, *dlen); 1653 *data = data_in; 1654 } 1655 } 1656 1657 return 0; 1658 } 1659 1660 1661 int 1662 chunkqueue_read_data (chunkqueue * const cq, 1663 char * const data, const uint32_t dlen, 1664 log_error_st * const errh) 1665 { 1666 char *ptr = data; 1667 uint32_t len = dlen; 1668 if (chunkqueue_peek_data(cq, &ptr, &len, errh) < 0 || len != dlen) 1669 return -1; 1670 if (data != ptr) memcpy(data, ptr, len); 1671 chunkqueue_mark_written(cq, len); 1672 return 0; 1673 } 1674 1675 1676 buffer * 1677 chunkqueue_read_squash (chunkqueue * const restrict cq, log_error_st * const restrict errh) 1678 { 1679 /* read and replace chunkqueue contents with single MEM_CHUNK. 1680 * cq->bytes_out is not modified */ 1681 1682 off_t cqlen = chunkqueue_length(cq); 1683 if (cqlen >= UINT32_MAX) return NULL; 1684 1685 if (cq->first && NULL == cq->first->next && cq->first->type == MEM_CHUNK) 1686 return cq->first->mem; 1687 1688 chunk * const c = chunk_acquire((uint32_t)cqlen+1); 1689 char *data = c->mem->ptr; 1690 uint32_t dlen = (uint32_t)cqlen; 1691 int rc = chunkqueue_peek_data(cq, &data, &dlen, errh); 1692 if (rc < 0) { 1693 chunk_release(c); 1694 return NULL; 1695 } 1696 buffer_truncate(c->mem, dlen); 1697 1698 chunkqueue_release_chunks(cq); 1699 chunkqueue_append_chunk(cq, c); 1700 return c->mem; 1701 } 1702 1703 1704 #ifdef HAVE_MMAP 1705 1706 const chunk_file_view * 1707 chunkqueue_chunk_file_viewadj (chunk * const c, off_t n, log_error_st * restrict errh) 1708 { 1709 /*assert(c->type == FILE_CHUNK);*/ 1710 chunk_file_view * restrict cfv = c->file.view; 1711 1712 if (NULL == cfv) { 1713 /* XXX: might add global config check to enable/disable mmap use here */ 1714 cfv = c->file.view = chunk_file_view_init(); 1715 } 1716 else if (MAP_FAILED != cfv->mptr) 1717 munmap(cfv->mptr, (size_t)cfv->mlen); 1718 /*cfv->mptr= MAP_FAILED;*//*(assigned below)*/ 1719 1720 if (c->file.fd < 0 && 0 != chunk_open_file_chunk(c, errh)) { 1721 c->file.view = chunk_file_view_failed(cfv); 1722 return NULL; 1723 } 1724 1725 cfv->foff = mmap_align_offset(c->offset); 1726 1727 if (0 != n) { 1728 cfv->mlen = c->offset - cfv->foff + n; 1729 #if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64)) 1730 /*(consider 512k blocks if this func is used more generically)*/ 1731 const off_t mmap_chunk_size = 8 * 1024 * 1024; 1732 if (cfv->mlen > mmap_chunk_size) 1733 cfv->mlen = mmap_chunk_size; 1734 #endif 1735 } 1736 else 1737 cfv->mlen = MMAP_CHUNK_SIZE; 1738 /* XXX: 64-bit might consider larger min block size, or even entire file */ 1739 if (cfv->mlen < MMAP_CHUNK_SIZE) 1740 cfv->mlen = MMAP_CHUNK_SIZE; 1741 if (cfv->mlen > c->file.length - cfv->foff) 1742 cfv->mlen = c->file.length - cfv->foff; 1743 1744 cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, 1745 c->file.is_temp ? MAP_PRIVATE : chunk_mmap_flags, 1746 c->file.fd, cfv->foff); 1747 1748 if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { 1749 if (__builtin_expect( (errno == EINVAL), 0)) { 1750 chunk_mmap_flags &= ~MAP_SHARED; 1751 chunk_mmap_flags |= MAP_PRIVATE; 1752 cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, 1753 MAP_PRIVATE, c->file.fd, cfv->foff); 1754 } 1755 if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { 1756 c->file.view = chunk_file_view_failed(cfv); 1757 return NULL; 1758 } 1759 } 1760 1761 #if 0 /*(review callers before changing; some expect open file)*/ 1762 /* close() fd as soon as fully mmap() rather than when done w/ chunk 1763 * (possibly worthwhile to keep active fd count lower) 1764 * (probably only reasonable if entire file is mapped) */ 1765 if (c->file.is_temp && !c->file.refchg) { 1766 close(c->file.fd); 1767 c->file.fd = -1; 1768 } 1769 #endif 1770 1771 #if 0 1772 /* disable madvise unless we find common cases where there is a benefit 1773 * (??? madvise for full mmap length or only for original requested n ???) 1774 * (??? might additional flags param to func to indicate madvise pref ???) 1775 * (??? might experiment with Linux mmap flags MAP_POPULATE|MAP_PRIVATE) 1776 * (??? might experiment with madvise MADV_POPULATE_READ (since Linux 5.14)) 1777 * note: caller might be in better position to know if starting an mmap 1778 * which will be flushed in entirety, and perform madvise at that point, 1779 * perhaps with MADV_SEQUENTIAL */ 1780 #ifdef HAVE_MADVISE 1781 if (cfv->mlen > 65536) /*(skip syscall if size <= 64KB)*/ 1782 (void)madvise(cfv->mptr, (size_t)cfv->mlen, MADV_WILLNEED); 1783 #endif 1784 #endif 1785 1786 return cfv; 1787 } 1788 1789 #endif /* HAVE_MMAP */ 1790