1 #include "first.h" 2 3 /** 4 * the network chunk-API 5 * 6 * 7 */ 8 9 #include "chunk.h" 10 #include "fdevent.h" 11 #include "log.h" 12 13 #include <sys/types.h> 14 #include <sys/stat.h> 15 #include "sys-mmap.h" 16 #include "sys-setjmp.h" 17 18 #include <stdlib.h> 19 #include <fcntl.h> 20 #include <unistd.h> 21 22 #include <errno.h> 23 #include <string.h> 24 25 26 #ifdef HAVE_MMAP 27 28 #define MMAP_CHUNK_SIZE (512*1024) 29 30 __attribute_cold__ 31 /*__attribute_noinline__*/ 32 static off_t 33 mmap_pagemask (void) 34 { 35 #ifndef _WIN32 36 long pagesize = sysconf(_SC_PAGESIZE); 37 #else 38 long pagesize = -1; /*(not implemented (yet))*/ 39 #endif 40 if (-1 == pagesize) pagesize = 4096; 41 force_assert(pagesize < MMAP_CHUNK_SIZE); 42 return ~((off_t)pagesize - 1); /* pagesize always power-of-2 */ 43 } 44 45 #if 0 46 static off_t 47 mmap_align_offset (off_t start) 48 { 49 static off_t pagemask = 0; 50 if (0 == pagemask) 51 pagemask = mmap_pagemask(); 52 return (start & pagemask); 53 } 54 #endif 55 56 #define mmap_align_offset(offset) ((offset) & chunk_pagemask) 57 static off_t chunk_pagemask = 0; 58 static int chunk_mmap_flags = MAP_SHARED; 59 60 #endif /* HAVE_MMAP */ 61 62 63 /* default 1 MB */ 64 #define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024) 65 66 static size_t chunk_buf_sz = 8192; 67 static chunk *chunks, *chunks_oversized, *chunks_filechunk; 68 static chunk *chunk_buffers; 69 static int chunks_oversized_n; 70 static const array *chunkqueue_default_tempdirs = NULL; 71 static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; 72 73 void chunkqueue_set_chunk_size (size_t sz) 74 { 75 size_t x = 1024; 76 while (x < sz && x < (1u << 30)) x <<= 1; 77 chunk_buf_sz = sz > 0 ? x : 8192; 78 } 79 80 void chunkqueue_set_tempdirs_default_reset (void) 81 { 82 chunk_buf_sz = 8192; 83 chunkqueue_default_tempdirs = NULL; 84 chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE; 85 86 #ifdef HAVE_MMAP /*(abuse this func to initialize statics as startup)*/ 87 if (0 == chunk_pagemask) 88 chunk_pagemask = mmap_pagemask(); 89 chunk_mmap_flags = MAP_SHARED; 90 #endif 91 } 92 93 chunkqueue *chunkqueue_init(chunkqueue *cq) { 94 /* (if caller passes non-NULL cq, it must be 0-init) */ 95 if (NULL == cq) { 96 cq = calloc(1, sizeof(*cq)); 97 force_assert(NULL != cq); 98 } 99 100 cq->first = NULL; 101 cq->last = NULL; 102 103 cq->tempdirs = chunkqueue_default_tempdirs; 104 cq->upload_temp_file_size = chunkqueue_default_tempfile_size; 105 106 return cq; 107 } 108 109 __attribute_returns_nonnull__ 110 static chunk *chunk_init(void) { 111 chunk * const restrict c = calloc(1, sizeof(*c)); 112 force_assert(NULL != c); 113 114 #if 0 /*(zeroed by calloc())*/ 115 c->type = MEM_CHUNK; 116 c->next = NULL; 117 c->offset = 0; 118 c->file.length = 0; 119 c->file.is_temp = 0; 120 c->file.view = NULL; 121 #endif 122 c->file.fd = -1; 123 124 c->mem = buffer_init(); 125 return c; 126 } 127 128 __attribute_noinline__ 129 __attribute_returns_nonnull__ 130 static chunk *chunk_init_sz(size_t sz) { 131 chunk * const restrict c = chunk_init(); 132 buffer_string_prepare_copy(c->mem, sz-1); 133 return c; 134 } 135 136 #ifdef HAVE_MMAP 137 138 __attribute_malloc__ 139 __attribute_returns_nonnull__ 140 static void * chunk_file_view_init (void) { 141 chunk_file_view * const restrict cfv = calloc(1, sizeof(*cfv)); 142 force_assert(NULL != cfv); 143 cfv->mptr = MAP_FAILED; 144 #if 0 /*(zeroed by calloc())*/ 145 cfv->mlen = 0; 146 cfv->foff = 0; 147 #endif 148 cfv->refcnt = 1; 149 return cfv; 150 } 151 152 __attribute_nonnull__() 153 static chunk_file_view * chunk_file_view_release (chunk_file_view *cfv) { 154 if (0 == --cfv->refcnt) { 155 if (MAP_FAILED != cfv->mptr) 156 munmap(cfv->mptr, (size_t)cfv->mlen); 157 free(cfv); 158 } 159 return NULL; 160 } 161 162 __attribute_cold__ 163 __attribute_noinline__ 164 __attribute_nonnull__() 165 static chunk_file_view * chunk_file_view_failed (chunk_file_view *cfv) { 166 return chunk_file_view_release(cfv); 167 } 168 169 #endif /* HAVE_MMAP */ 170 171 ssize_t 172 chunk_file_pread (int fd, void *buf, size_t count, off_t offset) 173 { 174 /*(expects open file for FILE_CHUNK)*/ 175 #ifndef HAVE_PREAD 176 /*(On systems without pread() or equivalent, lseek() is repeated if this 177 * func is called in a loop, but this func is generally used on small files, 178 * or reading a small bit at a time. Even in the case of mod_deflate, files 179 * are not expected to be excessively large.) */ 180 if (-1 == lseek(fd, offset, SEEK_SET)) 181 return -1; 182 #endif 183 184 ssize_t rd; 185 do { 186 #ifdef HAVE_PREAD 187 rd =pread(fd, buf, count, offset); 188 #else 189 rd = read(fd, buf, count); 190 #endif 191 } while (-1 == rd && errno == EINTR); 192 return rd; 193 } 194 195 static void chunk_reset_file_chunk(chunk *c) { 196 if (c->file.is_temp) { 197 c->file.is_temp = 0; 198 if (!buffer_is_blank(c->mem)) 199 unlink(c->mem->ptr); 200 } 201 if (c->file.refchg) { 202 c->file.refchg(c->file.ref, -1); 203 c->file.refchg = 0; /* NULL fn ptr */ 204 c->file.ref = NULL; 205 } 206 else if (c->file.fd != -1) { 207 close(c->file.fd); 208 } 209 #ifdef HAVE_MMAP 210 if (c->file.view) 211 c->file.view = chunk_file_view_release(c->file.view); 212 #endif 213 c->file.fd = -1; 214 c->file.length = 0; 215 c->type = MEM_CHUNK; 216 } 217 218 static void chunk_reset(chunk *c) { 219 if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c); 220 221 buffer_clear(c->mem); 222 c->offset = 0; 223 } 224 225 static void chunk_free(chunk *c) { 226 if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c); 227 buffer_free(c->mem); 228 free(c); 229 } 230 231 static chunk * chunk_pop_oversized(size_t sz) { 232 /* future: might have buckets of certain sizes, up to socket buf sizes */ 233 if (chunks_oversized && chunks_oversized->mem->size >= sz) { 234 --chunks_oversized_n; 235 chunk *c = chunks_oversized; 236 chunks_oversized = c->next; 237 return c; 238 } 239 return NULL; 240 } 241 242 static void chunk_push_oversized(chunk * const c, const size_t sz) { 243 /* XXX: chunk_buffer_yield() may have removed need for list size limit */ 244 if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) { 245 ++chunks_oversized_n; 246 chunk **co = &chunks_oversized; 247 while (*co && sz < (*co)->mem->size) co = &(*co)->next; 248 c->next = *co; 249 *co = c; 250 } 251 else { 252 buffer * const tb = chunks_oversized ? chunks_oversized->mem : NULL; 253 if (tb && tb->size < sz) { 254 /* swap larger mem block onto head of list; free smaller mem */ 255 chunks_oversized->mem = c->mem; 256 c->mem = tb; 257 } 258 chunk_free(c); 259 } 260 } 261 262 __attribute_noinline__ 263 __attribute_returns_nonnull__ 264 static buffer * chunk_buffer_acquire_sz(const size_t sz) { 265 chunk *c; 266 buffer *b; 267 if (sz <= (chunk_buf_sz|1)) { 268 if (chunks) { 269 c = chunks; 270 chunks = c->next; 271 } 272 else 273 c = chunk_init_sz(chunk_buf_sz); 274 } 275 else { 276 c = chunk_pop_oversized(sz); 277 if (NULL == c) { 278 /*(round up to nearest chunk_buf_sz)*/ 279 /* NB: round down power-2 + 1 to avoid excess allocation 280 * (sz & ~1uL) relies on buffer_realloc() adding +1 *and* on callers 281 * of this func never passing power-2 + 1 sz unless direct caller 282 * adds +1 for '\0', as is done in chunk_buffer_prepare_append() */ 283 c = chunk_init_sz(((sz&~1uL)+(chunk_buf_sz-1)) & ~(chunk_buf_sz-1)); 284 } 285 } 286 c->next = chunk_buffers; 287 chunk_buffers = c; 288 b = c->mem; 289 c->mem = NULL; 290 return b; 291 } 292 293 buffer * chunk_buffer_acquire(void) { 294 return chunk_buffer_acquire_sz(chunk_buf_sz); 295 } 296 297 void chunk_buffer_release(buffer *b) { 298 if (NULL == b) return; 299 if (chunk_buffers) { 300 chunk *c = chunk_buffers; 301 chunk_buffers = c->next; 302 c->mem = b; 303 buffer_clear(b); 304 if (b->size == (chunk_buf_sz|1)) { 305 c->next = chunks; 306 chunks = c; 307 } 308 else if (b->size > chunk_buf_sz) 309 chunk_push_oversized(c, b->size); 310 else 311 chunk_free(c); 312 } 313 else { 314 buffer_free(b); 315 } 316 } 317 318 void chunk_buffer_yield(buffer *b) { 319 if (b->size == (chunk_buf_sz|1)) return; 320 321 buffer * const cb = chunk_buffer_acquire_sz(chunk_buf_sz); 322 buffer tb = *b; 323 *b = *cb; 324 *cb = tb; 325 chunk_buffer_release(cb); 326 } 327 328 size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) { 329 if (sz > buffer_string_space(b)) { 330 sz += b->used ? b->used : 1; 331 buffer * const cb = chunk_buffer_acquire_sz(sz); 332 /* swap buffer contents and copy original b->ptr into larger b->ptr */ 333 /*(this does more than buffer_move())*/ 334 buffer tb = *b; 335 *b = *cb; 336 *cb = tb; 337 if ((b->used = tb.used)) 338 memcpy(b->ptr, tb.ptr, tb.used); 339 chunk_buffer_release(cb); 340 } 341 return buffer_string_space(b); 342 } 343 344 __attribute_noinline__ 345 __attribute_returns_nonnull__ 346 static chunk * chunk_acquire(size_t sz) { 347 if (sz <= (chunk_buf_sz|1)) { 348 if (chunks) { 349 chunk *c = chunks; 350 chunks = c->next; 351 return c; 352 } 353 sz = chunk_buf_sz; 354 } 355 else { 356 /*(round up to nearest chunk_buf_sz)*/ 357 sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1); 358 chunk *c = chunk_pop_oversized(sz); 359 if (c) return c; 360 } 361 362 return chunk_init_sz(sz); 363 } 364 365 static void chunk_release(chunk *c) { 366 const size_t sz = c->mem->size; 367 if (sz == (chunk_buf_sz|1)) { 368 chunk_reset(c); 369 c->next = chunks; 370 chunks = c; 371 } 372 else if (sz > chunk_buf_sz) { 373 chunk_reset(c); 374 chunk_push_oversized(c, sz); 375 } 376 else if (c->type == FILE_CHUNK) { 377 chunk_reset(c); 378 c->next = chunks_filechunk; 379 chunks_filechunk = c; 380 } 381 else { 382 chunk_free(c); 383 } 384 } 385 386 __attribute_returns_nonnull__ 387 static chunk * chunk_acquire_filechunk(void) { 388 if (chunks_filechunk) { 389 chunk *c = chunks_filechunk; 390 chunks_filechunk = c->next; 391 return c; 392 } 393 return chunk_init(); 394 } 395 396 void chunkqueue_chunk_pool_clear(void) 397 { 398 for (chunk *next, *c = chunks; c; c = next) { 399 next = c->next; 400 chunk_free(c); 401 } 402 chunks = NULL; 403 for (chunk *next, *c = chunks_oversized; c; c = next) { 404 next = c->next; 405 chunk_free(c); 406 } 407 chunks_oversized = NULL; 408 chunks_oversized_n = 0; 409 for (chunk *next, *c = chunks_filechunk; c; c = next) { 410 next = c->next; 411 chunk_free(c); 412 } 413 chunks_filechunk = NULL; 414 } 415 416 void chunkqueue_chunk_pool_free(void) 417 { 418 chunkqueue_chunk_pool_clear(); 419 for (chunk *next, *c = chunk_buffers; c; c = next) { 420 next = c->next; 421 #if 1 /*(chunk_buffers contains MEM_CHUNK with (c->mem == NULL))*/ 422 free(c); 423 #else /*(c->mem = buffer_init() is no longer necessary below)*/ 424 c->mem = buffer_init(); /*(chunk_reset() expects c->mem != NULL)*/ 425 chunk_free(c); 426 #endif 427 } 428 chunk_buffers = NULL; 429 } 430 431 __attribute_pure__ 432 static off_t chunk_remaining_length(const chunk *c) { 433 /* MEM_CHUNK or FILE_CHUNK */ 434 return (c->type == MEM_CHUNK 435 ? (off_t)buffer_clen(c->mem) 436 : c->file.length) 437 - c->offset; 438 } 439 440 static void chunkqueue_release_chunks(chunkqueue *cq) { 441 cq->last = NULL; 442 for (chunk *c; (c = cq->first); ) { 443 cq->first = c->next; 444 chunk_release(c); 445 } 446 } 447 448 void chunkqueue_free(chunkqueue *cq) { 449 if (NULL == cq) return; 450 chunkqueue_release_chunks(cq); 451 free(cq); 452 } 453 454 static void chunkqueue_prepend_chunk(chunkqueue * const restrict cq, chunk * const restrict c) { 455 if (NULL == (c->next = cq->first)) cq->last = c; 456 cq->first = c; 457 } 458 459 static void chunkqueue_append_chunk(chunkqueue * const restrict cq, chunk * const restrict c) { 460 c->next = NULL; 461 *(cq->last ? &cq->last->next : &cq->first) = c; 462 cq->last = c; 463 } 464 465 __attribute_returns_nonnull__ 466 static chunk * chunkqueue_prepend_mem_chunk(chunkqueue *cq, size_t sz) { 467 chunk *c = chunk_acquire(sz); 468 chunkqueue_prepend_chunk(cq, c); 469 return c; 470 } 471 472 __attribute_returns_nonnull__ 473 static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq, size_t sz) { 474 chunk *c = chunk_acquire(sz); 475 chunkqueue_append_chunk(cq, c); 476 return c; 477 } 478 479 __attribute_nonnull__() 480 __attribute_returns_nonnull__ 481 static chunk * chunkqueue_append_file_chunk(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) { 482 chunk * const c = chunk_acquire_filechunk(); 483 chunkqueue_append_chunk(cq, c); 484 c->type = FILE_CHUNK; 485 c->offset = offset; 486 c->file.length = offset + len; 487 cq->bytes_in += len; 488 buffer_copy_buffer(c->mem, fn); 489 return c; 490 } 491 492 void chunkqueue_reset(chunkqueue *cq) { 493 chunkqueue_release_chunks(cq); 494 cq->bytes_in = 0; 495 cq->bytes_out = 0; 496 cq->tempdir_idx = 0; 497 } 498 499 void chunkqueue_append_file_fd(chunkqueue * const restrict cq, const buffer * const restrict fn, int fd, off_t offset, off_t len) { 500 if (len > 0) { 501 (chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd; 502 } 503 else { 504 close(fd); 505 } 506 } 507 508 void chunkqueue_append_file(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) { 509 if (len > 0) { 510 chunkqueue_append_file_chunk(cq, fn, offset, len); 511 } 512 } 513 514 515 static int chunkqueue_append_mem_extend_chunk(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 516 chunk *c = cq->last; 517 if (0 == len) return 1; 518 if (c != NULL && c->type == MEM_CHUNK 519 && buffer_string_space(c->mem) >= len) { 520 buffer_append_string_len(c->mem, mem, len); 521 cq->bytes_in += len; 522 return 1; 523 } 524 return 0; 525 } 526 527 528 void chunkqueue_append_buffer(chunkqueue * const restrict cq, buffer * const restrict mem) { 529 chunk *c; 530 const size_t len = buffer_clen(mem); 531 if (len < 1024 && chunkqueue_append_mem_extend_chunk(cq, mem->ptr, len)) { 532 buffer_clear(mem); 533 return; 534 } 535 536 c = chunkqueue_append_mem_chunk(cq, chunk_buf_sz); 537 cq->bytes_in += len; 538 buffer_move(c->mem, mem); 539 } 540 541 542 void chunkqueue_append_mem(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 543 chunk *c; 544 if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len)) 545 return; 546 547 c = chunkqueue_append_mem_chunk(cq, len+1); 548 cq->bytes_in += len; 549 buffer_copy_string_len(c->mem, mem, len); 550 } 551 552 553 void chunkqueue_append_mem_min(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) { 554 chunk *c; 555 if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len)) 556 return; 557 558 c = chunk_init_sz(len+1); 559 chunkqueue_append_chunk(cq, c); 560 cq->bytes_in += len; 561 buffer_copy_string_len(c->mem, mem, len); 562 } 563 564 565 void chunkqueue_append_chunkqueue(chunkqueue * const restrict cq, chunkqueue * const restrict src) { 566 if (NULL == src->first) return; 567 568 if (NULL == cq->first) { 569 cq->first = src->first; 570 } else { 571 cq->last->next = src->first; 572 } 573 cq->last = src->last; 574 cq->bytes_in += chunkqueue_length(src); 575 576 src->first = NULL; 577 src->last = NULL; 578 src->bytes_out = src->bytes_in; 579 } 580 581 582 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz) { 583 chunk * const c = chunkqueue_prepend_mem_chunk(cq, sz); 584 return c->mem; 585 } 586 587 588 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq) { 589 return chunkqueue_prepend_buffer_open_sz(cq, chunk_buf_sz); 590 } 591 592 593 void chunkqueue_prepend_buffer_commit(chunkqueue *cq) { 594 cq->bytes_in += buffer_clen(cq->first->mem); 595 } 596 597 598 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz) { 599 chunk * const c = chunkqueue_append_mem_chunk(cq, sz); 600 return c->mem; 601 } 602 603 604 buffer * chunkqueue_append_buffer_open(chunkqueue *cq) { 605 return chunkqueue_append_buffer_open_sz(cq, chunk_buf_sz); 606 } 607 608 609 void chunkqueue_append_buffer_commit(chunkqueue *cq) { 610 cq->bytes_in += buffer_clen(cq->last->mem); 611 } 612 613 614 char * chunkqueue_get_memory(chunkqueue * const restrict cq, size_t * const restrict len) { 615 size_t sz = *len ? *len : (chunk_buf_sz >> 1); 616 buffer *b; 617 chunk *c = cq->last; 618 if (NULL != c && MEM_CHUNK == c->type) { 619 /* return pointer into existing buffer if large enough */ 620 size_t avail = buffer_string_space(c->mem); 621 if (avail >= sz) { 622 *len = avail; 623 b = c->mem; 624 return b->ptr + buffer_clen(b); 625 } 626 } 627 628 /* allocate new chunk */ 629 b = chunkqueue_append_buffer_open_sz(cq, sz); 630 *len = buffer_string_space(b); 631 return b->ptr; 632 } 633 634 void chunkqueue_use_memory(chunkqueue * const restrict cq, chunk *ckpt, size_t len) { 635 buffer *b = cq->last->mem; 636 637 if (len > 0) { 638 buffer_commit(b, len); 639 cq->bytes_in += len; 640 if (cq->last == ckpt || NULL == ckpt || MEM_CHUNK != ckpt->type 641 || len > buffer_string_space(ckpt->mem)) return; 642 643 buffer_append_string_buffer(ckpt->mem, b); 644 } 645 else if (!buffer_is_blank(b)) { /*(cq->last == ckpt)*/ 646 return; /* last chunk is not empty */ 647 } 648 649 /* remove empty last chunk */ 650 chunk_release(cq->last); 651 cq->last = ckpt; 652 *(ckpt ? &ckpt->next : &cq->first) = NULL; 653 } 654 655 void chunkqueue_update_file(chunkqueue * const restrict cq, chunk *c, off_t len) { 656 /*assert(c->type == FILE_CHUNK);*/ 657 c->file.length += len; 658 cq->bytes_in += len; 659 if (0 == chunk_remaining_length(c)) 660 chunkqueue_remove_empty_chunks(cq); 661 } 662 663 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size) { 664 if (upload_temp_file_size == 0) 665 upload_temp_file_size = DEFAULT_TEMPFILE_SIZE; 666 chunkqueue_default_tempdirs = tempdirs; 667 chunkqueue_default_tempfile_size = upload_temp_file_size; 668 } 669 670 void chunkqueue_set_tempdirs(chunkqueue * const restrict cq, const array * const restrict tempdirs, off_t upload_temp_file_size) { 671 if (upload_temp_file_size == 0) 672 upload_temp_file_size = chunkqueue_default_tempfile_size; 673 cq->tempdirs = tempdirs; 674 cq->upload_temp_file_size = upload_temp_file_size; 675 cq->tempdir_idx = 0; 676 } 677 678 __attribute_noinline__ 679 static void chunkqueue_dup_file_chunk_fd (chunk * const restrict d, const chunk * const restrict c) { 680 /*assert(d != c);*/ 681 /*assert(d->type == FILE_CHUNK);*/ 682 /*assert(c->type == FILE_CHUNK);*/ 683 if (c->file.fd >= 0) { 684 if (c->file.refchg) { 685 d->file.fd = c->file.fd; 686 d->file.ref = c->file.ref; 687 d->file.refchg = c->file.refchg; 688 d->file.refchg(d->file.ref, 1); 689 } 690 else 691 d->file.fd = fdevent_dup_cloexec(c->file.fd); 692 #ifdef HAVE_MMAP 693 if ((d->file.view = c->file.view)) 694 ++d->file.view->refcnt; 695 #endif 696 } 697 } 698 699 __attribute_noinline__ 700 static void chunkqueue_steal_partial_file_chunk(chunkqueue * const restrict dest, const chunk * const restrict c, const off_t len) { 701 chunkqueue_append_file(dest, c->mem, c->offset, len); 702 chunkqueue_dup_file_chunk_fd(dest->last, c); 703 } 704 705 void chunkqueue_steal(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len) { 706 for (off_t clen; len > 0; len -= clen) { 707 chunk * const c = src->first; 708 if (__builtin_expect( (NULL == c), 0)) break; 709 710 clen = chunk_remaining_length(c); 711 712 if (len >= clen) { 713 /* move complete chunk */ 714 src->first = c->next; 715 if (c == src->last) src->last = NULL; 716 717 if (__builtin_expect( (0 != clen), 1)) { 718 chunkqueue_append_chunk(dest, c); 719 dest->bytes_in += clen; 720 } 721 else /* drop empty chunk */ 722 chunk_release(c); 723 } else { 724 /* copy partial chunk */ 725 726 switch (c->type) { 727 case MEM_CHUNK: 728 chunkqueue_append_mem(dest, c->mem->ptr + c->offset, len); 729 break; 730 case FILE_CHUNK: 731 /* tempfile flag is in "last" chunk after the split */ 732 chunkqueue_steal_partial_file_chunk(dest, c, len); 733 break; 734 } 735 736 c->offset += len; 737 clen = len; 738 } 739 740 src->bytes_out += clen; 741 } 742 } 743 744 static int chunkqueue_get_append_mkstemp(buffer * const b, const char *path, const uint32_t len) { 745 buffer_copy_path_len2(b,path,len,CONST_STR_LEN("lighttpd-upload-XXXXXX")); 746 #if defined(HAVE_SPLICE) && defined(HAVE_PWRITE) 747 /*(splice() rejects O_APPEND target; omit flag if also using pwrite())*/ 748 return fdevent_mkostemp(b->ptr, 0); 749 #else 750 return fdevent_mkostemp(b->ptr, O_APPEND); 751 #endif 752 } 753 754 static chunk *chunkqueue_get_append_newtempfile(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 755 static const buffer emptyb = { "", 0, 0 }; 756 chunk * const restrict last = cq->last; 757 chunk * const restrict c = chunkqueue_append_file_chunk(cq, &emptyb, 0, 0); 758 buffer * const restrict template = c->mem; 759 c->file.is_temp = 1; 760 761 if (cq->tempdirs && cq->tempdirs->used) { 762 /* we have several tempdirs, only if all of them fail we jump out */ 763 for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) { 764 data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx]; 765 c->file.fd = 766 chunkqueue_get_append_mkstemp(template, BUF_PTR_LEN(&ds->value)); 767 if (-1 != c->file.fd) return c; 768 } 769 } 770 else { 771 c->file.fd = 772 chunkqueue_get_append_mkstemp(template, CONST_STR_LEN("/var/tmp")); 773 if (-1 != c->file.fd) return c; 774 } 775 776 /* (report only last error to mkstemp() even if multiple temp dirs tried) */ 777 log_perror(errh, __FILE__, __LINE__, 778 "opening temp-file failed: %s", template->ptr); 779 /* remove (failed) final chunk */ 780 c->file.is_temp = 0; 781 if ((cq->last = last)) 782 last->next = NULL; 783 else 784 cq->first = NULL; 785 chunk_release(c); 786 return NULL; 787 } 788 789 static chunk *chunkqueue_get_append_tempfile(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 790 /* 791 * if the last chunk is 792 * - smaller than cq->upload_temp_file_size 793 * -> append to it (and it then might exceed cq->upload_temp_file_size) 794 * otherwise 795 * -> create a new chunk 796 */ 797 798 chunk * const c = cq->last; 799 if (NULL != c && c->file.is_temp && c->file.fd >= 0) { 800 801 if (c->file.length < (off_t)cq->upload_temp_file_size) 802 return c; /* ok, take the last chunk for our job */ 803 804 /* the chunk is too large now, close it */ 805 force_assert(0 == c->file.refchg); /*(else should not happen)*/ 806 int rc = close(c->file.fd); 807 c->file.fd = -1; 808 if (0 != rc) { 809 log_perror(errh, __FILE__, __LINE__, 810 "close() temp-file %s failed", c->mem->ptr); 811 return NULL; 812 } 813 } 814 return chunkqueue_get_append_newtempfile(cq, errh); 815 } 816 817 __attribute_cold__ 818 static int chunkqueue_append_tempfile_err(chunkqueue * const cq, log_error_st * const restrict errh, chunk * const c) { 819 const int errnum = errno; 820 if (errnum == EINTR) return 1; /* retry */ 821 822 int retry = (errnum == ENOSPC && cq->tempdirs 823 && ++cq->tempdir_idx < cq->tempdirs->used); 824 if (!retry) 825 log_perror(errh, __FILE__, __LINE__, 826 "write() temp-file %s failed", c->mem->ptr); 827 828 if (0 == chunk_remaining_length(c)) { 829 /*(remove empty chunk and unlink tempfile)*/ 830 chunkqueue_remove_empty_chunks(cq); 831 } 832 else {/*(close tempfile; avoid later attempts to append)*/ 833 force_assert(0 == c->file.refchg); /*(else should not happen)*/ 834 int rc = close(c->file.fd); 835 c->file.fd = -1; 836 if (0 != rc) { 837 log_perror(errh, __FILE__, __LINE__, 838 "close() temp-file %s failed", c->mem->ptr); 839 retry = 0; 840 } 841 } 842 return retry; 843 } 844 845 __attribute_cold__ 846 __attribute_noinline__ 847 static int chunkqueue_to_tempfiles(chunkqueue * const restrict dest, log_error_st * const restrict errh) { 848 /* transfer chunks from dest to src, adjust dest->bytes_in, and then call 849 * chunkqueue_steal_with_tempfiles() to write chunks from src back into 850 * dest, but into tempfiles. chunkqueue_steal_with_tempfiles() calls back 851 * into chunkqueue_append_mem_to_tempfile(), but will not re-enter this func 852 * since chunks moved to src, and dest made empty before recursive call */ 853 const off_t cqlen = chunkqueue_length(dest); 854 chunkqueue src = *dest; /*(copy struct)*/ 855 dest->first = dest->last = NULL; 856 dest->bytes_in -= cqlen; 857 if (0 == chunkqueue_steal_with_tempfiles(dest, &src, cqlen, errh)) 858 return 0; 859 else { 860 const int errnum = errno; 861 chunkqueue_release_chunks(&src); 862 return -errnum; 863 } 864 } 865 866 int chunkqueue_append_mem_to_tempfile(chunkqueue * const restrict dest, const char * restrict mem, size_t len, log_error_st * const restrict errh) { 867 chunk *dst_c = dest->first; 868 869 /* check if prior MEM_CHUNK(s) exist and write to tempfile 870 * (check first chunk only, since if we are using tempfiles, then 871 * we expect further chunks to be tempfiles after starting tempfiles)*/ 872 if (dst_c && dst_c->type == MEM_CHUNK 873 && 0 != chunkqueue_to_tempfiles(dest, errh)) { 874 return -1; 875 } 876 877 do { 878 /*(aside: arg len is permitted to be 0 and creates tempfile as a 879 * side effect. This is used by mod_ssi for ssi exec, as the func 880 * chunkqueue_get_append_tempfile() is not public. The result is 881 * an empty chunk at the end of the chunkqueue, which typically 882 * should be avoided)*/ 883 dst_c = chunkqueue_get_append_tempfile(dest, errh); 884 if (NULL == dst_c) 885 return -1; 886 #ifdef __COVERITY__ 887 if (dst_c->file.fd < 0) return -1; 888 #endif 889 #ifdef HAVE_PWRITE 890 /* coverity[negative_returns : FALSE] */ 891 const ssize_t written =pwrite(dst_c->file.fd, mem, len, dst_c->file.length); 892 #else 893 /* coverity[negative_returns : FALSE] */ 894 const ssize_t written = write(dst_c->file.fd, mem, len); 895 #endif 896 897 if ((size_t) written == len) { 898 dst_c->file.length += len; 899 dest->bytes_in += len; 900 return 0; 901 } else if (written >= 0) { 902 /*(assume EINTR if partial write and retry write(); 903 * retry write() might fail with ENOSPC if no more space on volume)*/ 904 dest->bytes_in += written; 905 mem += written; 906 len -= (size_t)written; 907 dst_c->file.length += (size_t)written; 908 /* continue; retry */ 909 } else if (!chunkqueue_append_tempfile_err(dest, errh, dst_c)) { 910 break; /* return -1; */ 911 } /* else continue; retry */ 912 } while (len); 913 914 return -1; 915 } 916 917 #ifdef HAVE_PWRITEV 918 919 #ifdef HAVE_SYS_UIO_H 920 #include <sys/uio.h> 921 #endif 922 923 __attribute_cold__ 924 __attribute_noinline__ 925 static ssize_t chunkqueue_append_cqmem_to_tempfile_partial(chunkqueue * const dest, chunk * const c, ssize_t wr, log_error_st * const restrict errh) { 926 /* recover from partial write of existing dest MEM_CHUNK to tempfile */ 927 chunk *ckpt = dest->first; 928 while (ckpt->next != c) ckpt = ckpt->next; 929 ckpt->next = NULL; 930 dest->last = ckpt; 931 dest->bytes_in -= wr; /*(avoid double count in dest cq)*/ 932 dest->bytes_out -= wr; 933 chunkqueue_mark_written(dest, wr);/*(remove MEM_CHUNK written to tempfile)*/ 934 935 c->next = dest->first; /*(place tempfile at beginning of dest cq)*/ 936 dest->first = c; 937 return (0 == chunkqueue_to_tempfiles(dest, errh)) ? 0 : -1; 938 } 939 940 static ssize_t chunkqueue_append_cqmem_to_tempfile(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len, log_error_st * const restrict errh) { 941 /* write multiple MEM_CHUNKs to tempfile in single pwritev() syscall */ 942 /*(could lseek() and writev() if pwritev() is not available, 943 * but if writev() is available, pwritev() is likely available, 944 * e.g. any modern Linux or *BSD, and possibly anything not Windows)*/ 945 unsigned int iovcnt = 0; 946 struct iovec iov[16]; 947 948 off_t dlen = 0; 949 chunk *c; 950 for (c = dest->first; c && c->type == MEM_CHUNK; c = c->next) { 951 const off_t clen = chunk_remaining_length(c); 952 iov[iovcnt].iov_base = c->mem->ptr + c->offset; 953 iov[iovcnt].iov_len = (size_t)clen; 954 dlen += clen; 955 ++iovcnt; 956 if (__builtin_expect( (iovcnt == sizeof(iov)/sizeof(*iov)), 0)) 957 break; /*(not expecting large number of MEM_CHUNK)*/ 958 } 959 if (__builtin_expect( (c != NULL), 0) && dest->first->type == MEM_CHUNK) { 960 /*(expecting only MEM_CHUNK if dest cq starts w/ MEM_CHUNK)*/ 961 /*(use less efficient fallback if that assumption does not hold true)*/ 962 if (0 != chunkqueue_to_tempfiles(dest, errh)) 963 return -1; 964 dlen = 0; 965 iovcnt = 0; 966 } 967 968 if (__builtin_expect( (iovcnt < sizeof(iov)/sizeof(*iov)), 1)) { 969 for (c = src->first; c && c->type == MEM_CHUNK; c = c->next) { 970 off_t clen = chunk_remaining_length(c); 971 if (clen > len) clen = len; 972 iov[iovcnt].iov_base = c->mem->ptr + c->offset; 973 iov[iovcnt].iov_len = (size_t)clen; 974 len -= clen; 975 ++iovcnt; 976 if (0 == len) break; 977 if (__builtin_expect( (iovcnt == sizeof(iov)/sizeof(*iov)), 0)) 978 break; /*(not expecting large number of MEM_CHUNK)*/ 979 } 980 } 981 982 if (__builtin_expect( (0 == iovcnt), 0)) return 0; /*(should not happen)*/ 983 984 c = chunkqueue_get_append_tempfile(dest, errh); 985 if (NULL == c) 986 return -1; 987 #ifdef __COVERITY__ 988 if (c->file.fd < 0) return -1; 989 #endif 990 /* coverity[negative_returns : FALSE] */ 991 ssize_t wr = pwritev(c->file.fd, iov, (int)iovcnt, c->file.length); 992 993 /*(memory use in chunkqueues is expected to be limited before spilling 994 * to tempfiles, so common case will write entire iovec to tempfile, 995 * and we return amount written *from src cq*, even if partial write; 996 * (not looping here to retry writing more, but caller might loop))*/ 997 998 if (wr >= 0) { 999 c->file.length += wr; 1000 dest->bytes_in += wr; 1001 if (dlen) { 1002 if (__builtin_expect( (wr < dlen), 0)) 1003 return 1004 chunkqueue_append_cqmem_to_tempfile_partial(dest,c,wr,errh); 1005 wr -= (ssize_t)dlen; 1006 dest->bytes_in -= dlen; /*(avoid double count in dest cq)*/ 1007 dest->bytes_out -= dlen; 1008 chunkqueue_mark_written(dest, dlen); 1009 } 1010 } 1011 1012 return wr; 1013 } 1014 1015 #endif /* HAVE_PWRITEV */ 1016 1017 #ifdef HAVE_SPLICE 1018 1019 __attribute_cold__ 1020 __attribute_noinline__ 1021 static ssize_t chunkqueue_append_drain_pipe_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1022 /* attempt to drain full 'len' from pipe 1023 * (even if len not reduced to opts->max_per_read limit) 1024 * since data may have already been moved from socket to pipe 1025 *(returns 0 on success, or -errno (negative errno) if error, 1026 * even if partial write occurred)*/ 1027 char buf[16384]; 1028 ssize_t rd; 1029 do { 1030 do { 1031 rd = read(fd, buf, sizeof(buf)); 1032 } while (rd < 0 && errno == EINTR); 1033 if (rd < 0) break; 1034 if (0 != chunkqueue_append_mem_to_tempfile(cq, buf, (size_t)rd, errh)) 1035 break; 1036 } while ((len -= (unsigned int)rd)); 1037 1038 if (0 == len) 1039 return 0; 1040 else { 1041 const int errnum = errno; 1042 if (cq->last && 0 == chunk_remaining_length(cq->last)) { 1043 /*(remove empty chunk and unlink tempfile)*/ 1044 chunkqueue_remove_empty_chunks(cq); 1045 } 1046 return -errnum; 1047 } 1048 } 1049 1050 ssize_t chunkqueue_append_splice_pipe_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1051 /* check if prior MEM_CHUNK(s) exist and write to tempfile 1052 * (check first chunk only, since if we are using tempfiles, then 1053 * we expect further chunks to be tempfiles after starting tempfiles)*/ 1054 if (cq->first && cq->first->type == MEM_CHUNK) { 1055 int rc = chunkqueue_to_tempfiles(cq, errh); 1056 if (__builtin_expect( (0 != rc), 0)) return rc; 1057 } 1058 1059 /*(returns num bytes written, or -errno (negative errno) if error)*/ 1060 ssize_t total = 0; 1061 do { 1062 chunk * const c = chunkqueue_get_append_tempfile(cq, errh); 1063 if (__builtin_expect( (NULL == c), 0)) return -errno; 1064 1065 loff_t off = c->file.length; 1066 ssize_t wr = splice(fd, NULL, c->file.fd, &off, len, 1067 SPLICE_F_MOVE | SPLICE_F_NONBLOCK); 1068 1069 if (__builtin_expect(((size_t)wr == len), 1)) { 1070 c->file.length += len; 1071 cq->bytes_in += len; 1072 return total + len; 1073 } 1074 else if (wr >= 0) { 1075 /*(assume EINTR if partial write and retry; 1076 * retry might fail with ENOSPC if no more space on volume)*/ 1077 cq->bytes_in += wr; 1078 total += wr; 1079 len -= (size_t)wr; 1080 c->file.length += (size_t)wr; 1081 /* continue; retry */ 1082 } 1083 else { 1084 const int errnum = errno; 1085 switch (errnum) { 1086 case EAGAIN: 1087 #ifdef EWOULDBLOCK 1088 #if EWOULDBLOCK != EAGAIN 1089 case EWOULDBLOCK: 1090 #endif 1091 #endif 1092 if (0 == chunk_remaining_length(c)) { 1093 /*(remove empty chunk and unlink tempfile)*/ 1094 chunkqueue_remove_empty_chunks(cq); 1095 } 1096 return total; 1097 case EINVAL: /*(assume total == 0 if EINVAL)*/ 1098 wr = chunkqueue_append_drain_pipe_tempfile(cq, fd, len, errh); 1099 return (0 == wr) ? total + (ssize_t)len : wr; 1100 default: 1101 if (!chunkqueue_append_tempfile_err(cq, errh, c)) 1102 return -errnum; 1103 break; /* else continue; retry */ 1104 } 1105 } 1106 } while (len); 1107 return -EIO; /*(not reached)*/ 1108 } 1109 1110 static int cqpipes[2] = { -1, -1 }; 1111 1112 __attribute_cold__ 1113 __attribute_noinline__ 1114 void chunkqueue_internal_pipes(int init) { 1115 /*(intended for internal use within a single lighttpd process; 1116 * must be initialized after fork() and graceful-restart to avoid 1117 * sharing pipes between processes)*/ 1118 if (-1 != cqpipes[0]) { close(cqpipes[0]); cqpipes[0] = -1; } 1119 if (-1 != cqpipes[1]) { close(cqpipes[1]); cqpipes[1] = -1; } 1120 if (init) 1121 if (0 != fdevent_pipe_cloexec(cqpipes, 262144)) { } /*(ignore error)*/ 1122 } 1123 1124 __attribute_cold__ 1125 __attribute_noinline__ 1126 static void chunkqueue_pipe_read_discard (void) { 1127 char buf[16384]; 1128 ssize_t rd; 1129 do { 1130 rd = read(cqpipes[0], buf, sizeof(buf)); 1131 } while (rd > 0 || (rd < 0 && errno == EINTR)); 1132 if (rd < 0 1133 #ifdef EWOULDBLOCK 1134 #if EWOULDBLOCK != EAGAIN 1135 && errno != EWOULDBLOCK 1136 #endif 1137 #endif 1138 && errno != EAGAIN) { 1139 chunkqueue_internal_pipes(1); /*(close() and re-initialize)*/ 1140 } 1141 } 1142 1143 ssize_t chunkqueue_append_splice_sock_tempfile(chunkqueue * const restrict cq, const int fd, unsigned int len, log_error_st * const restrict errh) { 1144 /*(returns num bytes written, or -errno (negative errno) if error)*/ 1145 int * const pipes = cqpipes; 1146 if (-1 == pipes[1]) 1147 return -EINVAL; /*(not configured; not handled here)*/ 1148 1149 /* splice() socket data to intermediate pipe */ 1150 ssize_t wr = splice(fd, NULL, pipes[1], NULL, len, 1151 SPLICE_F_MOVE | SPLICE_F_NONBLOCK); 1152 if (__builtin_expect( (wr <= 0), 0)) 1153 return -EINVAL; /*(reuse to indicate not handled here)*/ 1154 len = (unsigned int)wr; 1155 1156 /* splice() data from intermediate pipe to tempfile */ 1157 wr = chunkqueue_append_splice_pipe_tempfile(cq, pipes[0], len, errh); 1158 if (wr < 0) /* expect (wr == (ssize_t)len) or (wr == -1) */ 1159 chunkqueue_pipe_read_discard();/* discard data from intermediate pipe */ 1160 return wr; 1161 } 1162 1163 #endif /* HAVE_SPLICE */ 1164 1165 int chunkqueue_steal_with_tempfiles(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len, log_error_st * const restrict errh) { 1166 for (off_t clen; len > 0; len -= clen) { 1167 chunk * const c = src->first; 1168 if (__builtin_expect( (NULL == c), 0)) break; 1169 1170 #ifdef HAVE_PWRITEV 1171 1172 if (c->type == MEM_CHUNK) { 1173 clen = chunkqueue_append_cqmem_to_tempfile(dest, src, len, errh); 1174 if (__builtin_expect( (clen < 0), 0)) return -1; 1175 chunkqueue_mark_written(src, clen); /*(updates src->bytes_out)*/ 1176 } 1177 else { /* (c->type == FILE_CHUNK) */ 1178 clen = chunk_remaining_length(c); 1179 if (len < clen) clen = len; 1180 chunkqueue_steal(dest, src, clen);/*(same as below for FILE_CHUNK)*/ 1181 } 1182 1183 #else 1184 1185 clen = chunk_remaining_length(c); 1186 if (__builtin_expect( (0 == clen), 0)) { 1187 /* drop empty chunk */ 1188 src->first = c->next; 1189 if (c == src->last) src->last = NULL; 1190 chunk_release(c); 1191 continue; 1192 } 1193 1194 switch (c->type) { 1195 case FILE_CHUNK: 1196 if (len >= clen) { 1197 /* move complete chunk */ 1198 src->first = c->next; 1199 if (c == src->last) src->last = NULL; 1200 chunkqueue_append_chunk(dest, c); 1201 dest->bytes_in += clen; 1202 } else { 1203 /* copy partial chunk */ 1204 /* tempfile flag is in "last" chunk after the split */ 1205 chunkqueue_steal_partial_file_chunk(dest, c, len); 1206 c->offset += len; 1207 clen = len; 1208 } 1209 break; 1210 1211 case MEM_CHUNK: 1212 /* store bytes from memory chunk in tempfile */ 1213 if (0 != chunkqueue_append_mem_to_tempfile(dest, c->mem->ptr + c->offset, 1214 (len >= clen) ? clen : len, errh)) { 1215 return -1; 1216 } 1217 1218 if (len >= clen) { 1219 /* finished chunk */ 1220 src->first = c->next; 1221 if (c == src->last) src->last = NULL; 1222 chunk_release(c); 1223 } else { 1224 /* partial chunk */ 1225 c->offset += len; 1226 clen = len; 1227 } 1228 break; 1229 } 1230 1231 src->bytes_out += clen; 1232 1233 #endif 1234 } 1235 1236 return 0; 1237 } 1238 1239 void chunkqueue_append_cq_range (chunkqueue * const dst, const chunkqueue * const src, off_t offset, off_t len) { 1240 /* similar to chunkqueue_steal() but copy and append src range to dst cq */ 1241 /* (dst cq and src cq can be the same cq, so neither is marked restrict) */ 1242 1243 /* copy and append range len from src to dst */ 1244 for (const chunk *c = src->first; len > 0 && c != NULL; c = c->next) { 1245 /* scan into src to range offset (also skips empty chunks) */ 1246 off_t clen = chunk_remaining_length(c); 1247 if (offset >= clen) { 1248 offset -= clen; 1249 continue; 1250 } 1251 clen -= offset; 1252 if (len < clen) clen = len; 1253 len -= clen; 1254 1255 if (c->type == FILE_CHUNK) { 1256 chunkqueue_append_file(dst, c->mem, c->offset + offset, clen); 1257 chunkqueue_dup_file_chunk_fd(dst->last, c); 1258 } 1259 else { /*(c->type == MEM_CHUNK)*/ 1260 /*(string refs would reduce copying, 1261 * but this path is not expected to be hot)*/ 1262 chunkqueue_append_mem(dst, c->mem->ptr + c->offset + offset, clen); 1263 } 1264 offset = 0; 1265 } 1266 } 1267 1268 void chunkqueue_mark_written(chunkqueue *cq, off_t len) { 1269 cq->bytes_out += len; 1270 1271 for (chunk *c = cq->first; c; ) { 1272 off_t c_len = chunk_remaining_length(c); 1273 if (len >= c_len) { /* chunk got finished */ 1274 chunk * const x = c; 1275 c = c->next; 1276 len -= c_len; 1277 chunk_release(x); 1278 } 1279 else { /* partial chunk */ 1280 c->offset += len; 1281 cq->first = c; 1282 return; /* chunk not finished */ 1283 } 1284 } 1285 cq->first = cq->last = NULL; 1286 } 1287 1288 void chunkqueue_remove_finished_chunks(chunkqueue *cq) { 1289 for (chunk *c; (c = cq->first) && 0 == chunk_remaining_length(c); ){ 1290 if (NULL == (cq->first = c->next)) cq->last = NULL; 1291 chunk_release(c); 1292 } 1293 } 1294 1295 void chunkqueue_remove_empty_chunks(chunkqueue *cq) { 1296 chunk *c; 1297 chunkqueue_remove_finished_chunks(cq); 1298 1299 for (c = cq->first; c && c->next; c = c->next) { 1300 if (0 == chunk_remaining_length(c->next)) { 1301 chunk *empty = c->next; 1302 c->next = empty->next; 1303 if (empty == cq->last) cq->last = c; 1304 chunk_release(empty); 1305 } 1306 } 1307 } 1308 1309 void chunkqueue_compact_mem_offset(chunkqueue * const cq) { 1310 chunk * const restrict c = cq->first; 1311 if (0 == c->offset) return; 1312 if (c->type != MEM_CHUNK) return; /*(should not happen)*/ 1313 1314 buffer * const restrict b = c->mem; 1315 size_t len = buffer_clen(b) - c->offset; 1316 memmove(b->ptr, b->ptr+c->offset, len); 1317 c->offset = 0; 1318 buffer_truncate(b, len); 1319 } 1320 1321 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen) { 1322 /* caller must guarantee that chunks in chunkqueue are MEM_CHUNK, 1323 * which is currently always true when reading input from client */ 1324 chunk *c = cq->first; 1325 buffer *b = c->mem; 1326 size_t len = buffer_clen(b) - c->offset; 1327 if (len >= clen) return; 1328 if (b->size > clen) { 1329 if (buffer_string_space(b) < clen - len) 1330 chunkqueue_compact_mem_offset(cq); 1331 } 1332 else { 1333 b = chunkqueue_prepend_buffer_open_sz(cq, clen+1); 1334 buffer_append_string_len(b, c->mem->ptr + c->offset, len); 1335 cq->first->next = c->next; 1336 if (NULL == c->next) cq->last = cq->first; 1337 chunk_release(c); 1338 c = cq->first; 1339 } 1340 1341 for (chunk *fc = c; ((clen -= len) && (c = fc->next)); ) { 1342 len = buffer_clen(c->mem) - c->offset; 1343 if (len > clen) { 1344 buffer_append_string_len(b, c->mem->ptr + c->offset, clen); 1345 c->offset += clen; 1346 break; 1347 } 1348 buffer_append_string_len(b, c->mem->ptr + c->offset, len); 1349 fc->next = c->next; 1350 if (NULL == c->next) cq->last = fc; 1351 chunk_release(c); 1352 } 1353 /* chunkqueue_prepend_buffer_commit() is not called here; 1354 * no data added/removed from chunkqueue; consolidated only */ 1355 } 1356 1357 static int chunk_open_file_chunk(chunk * const restrict c, log_error_st * const restrict errh) { 1358 if (-1 == c->file.fd) { 1359 /* (permit symlinks; should already have been checked. However, TOC-TOU remains) */ 1360 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0))) { 1361 log_perror(errh, __FILE__, __LINE__, "open failed: %s",c->mem->ptr); 1362 return -1; 1363 } 1364 } 1365 1366 /*(skip file size checks if file is temp file created by lighttpd)*/ 1367 if (c->file.is_temp) return 0; 1368 1369 struct stat st; 1370 if (-1 == fstat(c->file.fd, &st)) { 1371 log_perror(errh, __FILE__, __LINE__, "fstat failed"); 1372 return -1; 1373 } 1374 1375 const off_t offset = c->offset; 1376 const off_t len = c->file.length - c->offset; 1377 force_assert(offset >= 0 && len >= 0); 1378 if (offset > st.st_size - len) { 1379 log_error(errh, __FILE__, __LINE__, "file shrunk: %s", c->mem->ptr); 1380 return -1; 1381 } 1382 1383 return 0; 1384 } 1385 1386 int chunkqueue_open_file_chunk(chunkqueue * const restrict cq, log_error_st * const restrict errh) { 1387 return chunk_open_file_chunk(cq->first, errh); 1388 } 1389 1390 1391 static ssize_t 1392 chunkqueue_write_data (const int fd, const void *buf, size_t len) 1393 { 1394 ssize_t wr = 0; 1395 if (len) 1396 do { wr = write(fd, buf, len); } while (-1 == wr && errno == EINTR); 1397 return wr; 1398 } 1399 1400 1401 #ifdef HAVE_MMAP 1402 __attribute_cold__ 1403 #endif 1404 __attribute_noinline__ 1405 static ssize_t 1406 chunkqueue_write_chunk_file_intermed (const int fd, chunk * const restrict c, log_error_st * const errh) 1407 { 1408 char buf[16384]; 1409 char *data = buf; 1410 const off_t len = c->file.length - c->offset; 1411 uint32_t dlen = len < (off_t)sizeof(buf) ? (uint32_t)len : sizeof(buf); 1412 chunkqueue cq = {c,c,0,0,0,0,0}; /*(fake cq for chunkqueue_peek_data())*/ 1413 if (0 != chunkqueue_peek_data(&cq, &data, &dlen, errh) && 0 == dlen) 1414 return -1; 1415 return chunkqueue_write_data(fd, data, dlen); 1416 } 1417 1418 1419 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 1420 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 1421 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 1422 #include <sys/sendfile.h> 1423 #include <stdint.h> 1424 #endif 1425 static ssize_t 1426 chunkqueue_write_chunk_file (const int fd, chunk * const restrict c, log_error_st * const errh) 1427 { 1428 /*(similar to network_write_file_chunk_mmap(), but does not use send() on 1429 * Windows because fd is expected to be file or pipe here, not socket)*/ 1430 1431 if (0 != chunk_open_file_chunk(c, errh)) 1432 return -1; 1433 1434 const off_t len = c->file.length - c->offset; 1435 if (0 == len) return 0; /*(sanity check)*/ 1436 1437 #if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \ 1438 && (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \ 1439 && defined(__linux__) && !defined HAVE_SENDFILE_BROKEN 1440 /* Linux kernel >= 2.6.33 supports sendfile() between most fd types */ 1441 off_t offset = c->offset; 1442 const ssize_t wr = 1443 sendfile(fd, c->file.fd, &offset, len < INT32_MAX ? len : INT32_MAX); 1444 if (__builtin_expect( (wr >= 0), 1) || (errno != EINVAL && errno != ENOSYS)) 1445 return wr; 1446 /*(could fallback to mmap, but if sendfile fails on linux, mmap may, too)*/ 1447 #elif defined(HAVE_MMAP) 1448 /*(chunkqueue_write_chunk() caller must protect against SIGBUS, if needed)*/ 1449 const chunk_file_view * const restrict cfv = 1450 chunkqueue_chunk_file_view(c, len, errh); 1451 if (NULL != cfv) { 1452 const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset); 1453 return chunkqueue_write_data(fd, chunk_file_view_dptr(cfv, c->offset), 1454 len <= mmap_avail ? len : mmap_avail); 1455 } 1456 #endif 1457 1458 return chunkqueue_write_chunk_file_intermed(fd, c, errh); 1459 } 1460 1461 1462 static ssize_t 1463 chunkqueue_write_chunk_mem (const int fd, const chunk * const restrict c) 1464 { 1465 const void * const buf = c->mem->ptr + c->offset; 1466 const size_t len = buffer_clen(c->mem) - (size_t)c->offset; 1467 return chunkqueue_write_data(fd, buf, len); 1468 } 1469 1470 1471 ssize_t 1472 chunkqueue_write_chunk (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh) 1473 { 1474 /*(note: expects non-empty cq->first)*/ 1475 chunk * const c = cq->first; 1476 switch (c->type) { 1477 case MEM_CHUNK: 1478 return chunkqueue_write_chunk_mem(fd, c); 1479 case FILE_CHUNK: 1480 return chunkqueue_write_chunk_file(fd, c, errh); 1481 default: 1482 errno = EINVAL; 1483 return -1; 1484 } 1485 } 1486 1487 1488 ssize_t 1489 chunkqueue_write_chunk_to_pipe (const int fd, chunkqueue * const restrict cq, log_error_st * const restrict errh) 1490 { 1491 #ifdef HAVE_SPLICE /* splice() temp files to pipe on Linux */ 1492 chunk * const c = cq->first; 1493 if (c->type == FILE_CHUNK) { 1494 const size_t len = (size_t)(c->file.length - c->offset); 1495 loff_t abs_offset = c->offset; 1496 if (__builtin_expect( (0 == len), 0)) return 0; 1497 return (0 == chunk_open_file_chunk(c, errh)) 1498 ? splice(c->file.fd, &abs_offset, fd, NULL, len, SPLICE_F_NONBLOCK) 1499 : -1; 1500 } 1501 #endif 1502 return chunkqueue_write_chunk(fd, cq, errh); 1503 } 1504 1505 1506 void 1507 chunkqueue_small_resp_optim (chunkqueue * const restrict cq) 1508 { 1509 /*(caller must verify response is small (and non-empty) before calling)*/ 1510 /*(caller must verify first chunk is MEM_CHUNK, i.e. response headers)*/ 1511 /*(caller must verify response is non-zero length)*/ 1512 1513 /*(optimization to use fewer syscalls to send a small response by reading 1514 * small files into memory, thereby avoiding use of sendfile() and multiple 1515 * calls to writev() (benefit for cleartext (non-TLS) and <= HTTP/1.1)) 1516 *(If TLS, then will shortly need to be in memory for encryption anyway)*/ 1517 1518 /*assert(cq->first);*/ 1519 /*assert(cq->first->type == MEM_CHUNK);*/ 1520 /*assert(cq->first->next);*/ 1521 chunk * restrict c = cq->first; 1522 chunk * const restrict filec = c->next; /*(require file already be open)*/ 1523 if (filec != cq->last || filec->type != FILE_CHUNK || filec->file.fd < 0) 1524 return; 1525 1526 /* Note: there should be no size change in chunkqueue, 1527 * so cq->bytes_in and cq->bytes_out should not be modified */ 1528 1529 off_t len = filec->file.length - filec->offset; 1530 if ((size_t)len > buffer_string_space(c->mem)) { 1531 c->next = chunk_acquire((size_t)len+1); 1532 c = c->next; 1533 /*c->next = filec;*/ 1534 } 1535 /* detach filec from chunkqueue; file expected to be read fully */ 1536 c->next = NULL; 1537 cq->last = c; 1538 1539 ssize_t rd; 1540 off_t offset = 0; 1541 char * const ptr = buffer_extend(c->mem, len); 1542 do { 1543 rd = chunk_file_pread(filec->file.fd, ptr+offset, (size_t)len, 1544 filec->offset+offset); 1545 } while (rd > 0 && (offset += rd, len -= rd)); 1546 /*(contents of chunkqueue kept valid even if error reading from file)*/ 1547 if (__builtin_expect( (0 == len), 1)) 1548 chunk_release(filec); 1549 else { /*(unexpected; error recovery)*/ 1550 buffer_truncate(c->mem, (uint32_t)(ptr + offset - c->mem->ptr)); 1551 cq->last = c->next = filec; 1552 if (offset) 1553 filec->offset += offset; 1554 else if (__builtin_expect( (cq->first != c), 0)) { 1555 cq->first->next = filec; 1556 chunk_release(c); 1557 } 1558 } 1559 } 1560 1561 1562 #if 0 1563 #ifdef HAVE_MMAP 1564 __attribute_noinline__ 1565 static off_t 1566 chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len) 1567 { 1568 /*(on 32-bit systems, caller should assert len <= SIZE_MAX)*/ 1569 memcpy(dst, src, (size_t)len); 1570 return len; 1571 } 1572 #endif 1573 #endif 1574 1575 1576 int 1577 chunkqueue_peek_data (chunkqueue * const cq, 1578 char ** const data, uint32_t * const dlen, 1579 log_error_st * const errh) 1580 { 1581 char * const data_in = *data; 1582 const uint32_t data_insz = *dlen; 1583 *dlen = 0; 1584 1585 for (chunk *c = cq->first; c; ) { 1586 uint32_t space = data_insz - *dlen; 1587 switch (c->type) { 1588 case MEM_CHUNK: 1589 { 1590 uint32_t have = buffer_clen(c->mem) - (uint32_t)c->offset; 1591 if (have > space) 1592 have = space; 1593 if (__builtin_expect( (0 == have), 0)) 1594 break; 1595 if (*dlen) 1596 memcpy(data_in + *dlen, c->mem->ptr + c->offset, have); 1597 else 1598 *data = c->mem->ptr + c->offset; /*(reference; defer copy)*/ 1599 *dlen += have; 1600 break; 1601 } 1602 1603 case FILE_CHUNK: 1604 if (c->file.fd >= 0 || 0 == chunk_open_file_chunk(c, errh)) { 1605 off_t len = c->file.length - c->offset; 1606 if (len > (off_t)space) 1607 len = (off_t)space; 1608 if (__builtin_expect( (0 == len), 0)) 1609 break; 1610 1611 #if 0 /* XXX: might improve performance on some system workloads */ 1612 #ifdef HAVE_MMAP 1613 /* mmap file to access data 1614 * fd need not be kept open for the mmap once 1615 * the mmap has been created, but is currently kept open for 1616 * other pre-existing logic which checks fd and opens file, 1617 * such as the condition for entering this code block above. */ 1618 /* Note: current use is with somewhat large buffers, e.g. 128k. 1619 * If larger buffers are used, then upper limit, e.g. 512k, 1620 * should be set for 32-bit to avoid address space issues) */ 1621 /* Note: under heavy load (or microbenchmark), system-reported 1622 * memory use for RSS can be very, very large, due to presence 1623 * of lots and lots of temp file read-only memory maps. 1624 * pmap -X and exclude lighttpd mmap files to get a better 1625 * view of memory use */ 1626 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 1627 ? chunkqueue_chunk_file_view(c, len, errh) 1628 : NULL; 1629 if (cfv && chunk_file_view_dlen(cfv, c->offset) >= len) { 1630 /*(check (above) that mapped chunk length >= requested len)*/ 1631 char * const mdata = chunk_file_view_dptr(cfv, c->offset); 1632 if (!c->file.is_temp) {/*(might be changed to policy flag)*/ 1633 if (sys_setjmp_eval3(chunk_setjmp_memcpy_cb, 1634 data_in+*dlen, mdata, len) < 0) { 1635 log_error(errh, __FILE__, __LINE__, 1636 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1637 return -1; 1638 } 1639 } 1640 else if (*dlen) 1641 memcpy(data_in+*dlen, mdata, (size_t)len); 1642 else 1643 *data = mdata; 1644 *dlen += (uint32_t)len; 1645 break; 1646 } 1647 #endif 1648 #endif 1649 1650 ssize_t rd = chunk_file_pread(c->file.fd, data_in+*dlen, 1651 (size_t)len, c->offset); 1652 if (rd <= 0) { /* -1 error; 0 EOF (unexpected) */ 1653 log_perror(errh, __FILE__, __LINE__, "read(\"%s\")", 1654 c->mem->ptr); 1655 return -1; 1656 } 1657 1658 *dlen += (uint32_t)rd; 1659 break; 1660 } 1661 return -1; 1662 1663 default: 1664 return -1; 1665 } 1666 1667 if (*dlen == data_insz) 1668 break; 1669 1670 c = c->next; 1671 if (NULL == c) 1672 break; 1673 1674 if (*dlen && *data != data_in) { 1675 memcpy(data_in, *data, *dlen); 1676 *data = data_in; 1677 } 1678 } 1679 1680 return 0; 1681 } 1682 1683 1684 int 1685 chunkqueue_read_data (chunkqueue * const cq, 1686 char * const data, const uint32_t dlen, 1687 log_error_st * const errh) 1688 { 1689 char *ptr = data; 1690 uint32_t len = dlen; 1691 if (chunkqueue_peek_data(cq, &ptr, &len, errh) < 0 || len != dlen) 1692 return -1; 1693 if (data != ptr) memcpy(data, ptr, len); 1694 chunkqueue_mark_written(cq, len); 1695 return 0; 1696 } 1697 1698 1699 buffer * 1700 chunkqueue_read_squash (chunkqueue * const restrict cq, log_error_st * const restrict errh) 1701 { 1702 /* read and replace chunkqueue contents with single MEM_CHUNK. 1703 * cq->bytes_out is not modified */ 1704 1705 off_t cqlen = chunkqueue_length(cq); 1706 if (cqlen >= UINT32_MAX) return NULL; 1707 1708 if (cq->first && NULL == cq->first->next && cq->first->type == MEM_CHUNK) 1709 return cq->first->mem; 1710 1711 chunk * const c = chunk_acquire((uint32_t)cqlen+1); 1712 char *data = c->mem->ptr; 1713 uint32_t dlen = (uint32_t)cqlen; 1714 int rc = chunkqueue_peek_data(cq, &data, &dlen, errh); 1715 if (rc < 0) { 1716 chunk_release(c); 1717 return NULL; 1718 } 1719 buffer_truncate(c->mem, dlen); 1720 1721 chunkqueue_release_chunks(cq); 1722 chunkqueue_append_chunk(cq, c); 1723 return c->mem; 1724 } 1725 1726 1727 #ifdef HAVE_MMAP 1728 1729 const chunk_file_view * 1730 chunkqueue_chunk_file_viewadj (chunk * const c, off_t n, log_error_st * restrict errh) 1731 { 1732 /*assert(c->type == FILE_CHUNK);*/ 1733 chunk_file_view * restrict cfv = c->file.view; 1734 1735 if (NULL == cfv) { 1736 /* XXX: might add global config check to enable/disable mmap use here */ 1737 cfv = c->file.view = chunk_file_view_init(); 1738 } 1739 else if (MAP_FAILED != cfv->mptr) 1740 munmap(cfv->mptr, (size_t)cfv->mlen); 1741 /*cfv->mptr= MAP_FAILED;*//*(assigned below)*/ 1742 1743 if (c->file.fd < 0 && 0 != chunk_open_file_chunk(c, errh)) { 1744 c->file.view = chunk_file_view_failed(cfv); 1745 return NULL; 1746 } 1747 1748 cfv->foff = mmap_align_offset(c->offset); 1749 1750 if (0 != n) { 1751 cfv->mlen = c->offset - cfv->foff + n; 1752 #if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64)) 1753 /*(consider 512k blocks if this func is used more generically)*/ 1754 const off_t mmap_chunk_size = 8 * 1024 * 1024; 1755 if (cfv->mlen > mmap_chunk_size) 1756 cfv->mlen = mmap_chunk_size; 1757 #endif 1758 } 1759 else 1760 cfv->mlen = MMAP_CHUNK_SIZE; 1761 /* XXX: 64-bit might consider larger min block size, or even entire file */ 1762 if (cfv->mlen < MMAP_CHUNK_SIZE) 1763 cfv->mlen = MMAP_CHUNK_SIZE; 1764 if (cfv->mlen > c->file.length - cfv->foff) 1765 cfv->mlen = c->file.length - cfv->foff; 1766 1767 cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, 1768 c->file.is_temp ? MAP_PRIVATE : chunk_mmap_flags, 1769 c->file.fd, cfv->foff); 1770 1771 if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { 1772 if (__builtin_expect( (errno == EINVAL), 0)) { 1773 chunk_mmap_flags &= ~MAP_SHARED; 1774 chunk_mmap_flags |= MAP_PRIVATE; 1775 cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ, 1776 MAP_PRIVATE, c->file.fd, cfv->foff); 1777 } 1778 if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) { 1779 c->file.view = chunk_file_view_failed(cfv); 1780 return NULL; 1781 } 1782 } 1783 1784 #if 0 /*(review callers before changing; some expect open file)*/ 1785 /* close() fd as soon as fully mmap() rather than when done w/ chunk 1786 * (possibly worthwhile to keep active fd count lower) 1787 * (probably only reasonable if entire file is mapped) */ 1788 if (c->file.is_temp && !c->file.refchg) { 1789 close(c->file.fd); 1790 c->file.fd = -1; 1791 } 1792 #endif 1793 1794 #if 0 1795 /* disable madvise unless we find common cases where there is a benefit 1796 * (??? madvise for full mmap length or only for original requested n ???) 1797 * (??? might additional flags param to func to indicate madvise pref ???) 1798 * (??? might experiment with Linux mmap flags MAP_POPULATE|MAP_PRIVATE) 1799 * (??? might experiment with madvise MADV_POPULATE_READ (since Linux 5.14)) 1800 * note: caller might be in better position to know if starting an mmap 1801 * which will be flushed in entirety, and perform madvise at that point, 1802 * perhaps with MADV_SEQUENTIAL */ 1803 #ifdef HAVE_MADVISE 1804 if (cfv->mlen > 65536) /*(skip syscall if size <= 64KB)*/ 1805 (void)madvise(cfv->mptr, (size_t)cfv->mlen, MADV_WILLNEED); 1806 #endif 1807 #endif 1808 1809 return cfv; 1810 } 1811 1812 #endif /* HAVE_MMAP */ 1813