1 /* 2 * http_chunk - append response to chunkqueue, possibly in "chunked" encoding 3 * 4 * Fully-rewritten from original 5 * Copyright(c) 2019 Glenn Strauss gstrauss()gluelogic.com All rights reserved 6 * License: BSD 3-clause (same as lighttpd) 7 */ 8 #include "first.h" 9 10 #include "http_chunk.h" 11 #include "base.h" 12 #include "chunk.h" 13 #include "stat_cache.h" 14 #include "fdevent.h" 15 #include "log.h" 16 17 #include <sys/types.h> 18 #include <sys/stat.h> 19 20 #include <stdlib.h> 21 #include <unistd.h> 22 23 #include <errno.h> 24 #include <string.h> 25 26 static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) { 27 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 28 #if 0 29 buffer b = { buf, 0, sizeof(buf) }; 30 buffer_append_uint_hex(&b, len); 31 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 32 chunkqueue_append_mem(cq, b.ptr, b.used-1); 33 #else 34 int i = (int)(sizeof(buf)); 35 buf[--i] = '\n'; 36 buf[--i] = '\r'; 37 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 38 chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i); 39 #endif 40 } 41 42 static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) { 43 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 44 #if 0 45 buffer b = { buf, 0, sizeof(buf) }; 46 buffer_append_uint_hex(&b, len); 47 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 48 return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh); 49 #else 50 int i = (int)(sizeof(buf)); 51 buf[--i] = '\n'; 52 buf[--i] = '\r'; 53 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 54 return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh); 55 #endif 56 } 57 58 static int http_chunk_append_file_open_fstat(const request_st * const r, const buffer * const fn, struct stat * const st) { 59 return 60 (r->conf.follow_symlink 61 || !stat_cache_path_contains_symlink(fn, r->conf.errh)) 62 ? stat_cache_open_rdonly_fstat(fn, st, r->conf.follow_symlink) 63 : -1; 64 } 65 66 static int http_chunk_append_read_fd_range(request_st * const r, const buffer * const fn, const int fd, off_t offset, off_t len) { 67 /* note: this routine should not be used for range requests 68 * unless the total size of ranges requested is small */ 69 /* note: future: could read into existing MEM_CHUNK in cq->last if 70 * there is sufficient space, but would need to adjust for existing 71 * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */ 72 UNUSED(fn); 73 74 chunkqueue * const cq = &r->write_queue; 75 76 if (r->resp_send_chunked) 77 http_chunk_len_append(cq, (uintmax_t)len); 78 79 if (-1 == lseek(fd, offset, SEEK_SET)) return -1; 80 buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2+1); 81 ssize_t rd; 82 offset = 0; 83 do { 84 rd = read(fd, b->ptr+offset, (size_t)(len-offset)); 85 } while (rd > 0 ? (offset += rd) != len : errno == EINTR); 86 buffer_commit(b, offset); 87 88 if (r->resp_send_chunked) 89 buffer_append_string_len(b, CONST_STR_LEN("\r\n")); 90 91 chunkqueue_append_buffer_commit(cq); 92 return (rd >= 0) ? 0 : -1; 93 } 94 95 void http_chunk_append_file_ref_range(request_st * const r, stat_cache_entry * const sce, const off_t offset, const off_t len) { 96 chunkqueue * const cq = &r->write_queue; 97 98 if (r->resp_send_chunked) 99 http_chunk_len_append(cq, (uintmax_t)len); 100 101 const buffer * const fn = &sce->name; 102 const int fd = sce->fd; 103 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 104 if (fd >= 0) { 105 chunk * const d = cq->last; 106 d->file.ref = sce; 107 d->file.refchg = stat_cache_entry_refchg; 108 stat_cache_entry_refchg(sce, 1); 109 } 110 111 if (r->resp_send_chunked) 112 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 113 } 114 115 void http_chunk_append_file_fd_range(request_st * const r, const buffer * const fn, const int fd, const off_t offset, const off_t len) { 116 chunkqueue * const cq = &r->write_queue; 117 118 if (r->resp_send_chunked) 119 http_chunk_len_append(cq, (uintmax_t)len); 120 121 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 122 123 if (r->resp_send_chunked) 124 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 125 } 126 127 int http_chunk_append_file_range(request_st * const r, const buffer * const fn, const off_t offset, off_t len) { 128 struct stat st; 129 const int fd = http_chunk_append_file_open_fstat(r, fn, &st); 130 if (fd < 0) return -1; 131 132 if (-1 == len) { 133 if (offset >= st.st_size) { 134 close(fd); 135 return (offset == st.st_size) ? 0 : -1; 136 } 137 len = st.st_size - offset; 138 } 139 else if (st.st_size - offset < len) { 140 close(fd); 141 return -1; 142 } 143 144 http_chunk_append_file_fd_range(r, fn, fd, offset, len); 145 return 0; 146 } 147 148 int http_chunk_append_file(request_st * const r, const buffer * const fn) { 149 struct stat st; 150 const int fd = http_chunk_append_file_open_fstat(r, fn, &st); 151 if (fd < 0) return -1; 152 http_chunk_append_file_fd(r, fn, fd, st.st_size); 153 return 0; 154 } 155 156 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) { 157 if (sz > 32768 || !r->resp_send_chunked) { 158 http_chunk_append_file_fd_range(r, fn, fd, 0, sz); 159 return 0; 160 } 161 162 /*(read small files into memory)*/ 163 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 164 close(fd); 165 return rc; 166 } 167 168 int http_chunk_append_file_ref(request_st * const r, stat_cache_entry * const sce) { 169 const off_t sz = sce->st.st_size; 170 if (sz > 32768 || !r->resp_send_chunked) { 171 http_chunk_append_file_ref_range(r, sce, 0, sz); 172 return 0; 173 } 174 175 /*(read small files into memory)*/ 176 const buffer * const fn = &sce->name; 177 const int fd = sce->fd; 178 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 179 return rc; 180 } 181 182 static int http_chunk_append_to_tempfile(request_st * const r, const char * const mem, const size_t len) { 183 chunkqueue * const cq = &r->write_queue; 184 log_error_st * const errh = r->conf.errh; 185 186 if (r->resp_send_chunked 187 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 188 return -1; 189 190 if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) 191 return -1; 192 193 if (r->resp_send_chunked 194 && 0 != 195 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 196 return -1; 197 198 return 0; 199 } 200 201 static int http_chunk_append_cq_to_tempfile(request_st * const r, chunkqueue * const src, const size_t len) { 202 chunkqueue * const cq = &r->write_queue; 203 log_error_st * const errh = r->conf.errh; 204 205 if (r->resp_send_chunked 206 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 207 return -1; 208 209 if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) 210 return -1; 211 212 if (r->resp_send_chunked 213 && 0 != 214 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 215 return -1; 216 217 return 0; 218 } 219 220 __attribute_pure__ 221 static int http_chunk_uses_tempfile(const request_st * const r, const chunkqueue * const cq, const size_t len) { 222 223 /* current usage does not append_mem or append_buffer after appending 224 * file, so not checking if users of this interface have appended large 225 * (references to) files to chunkqueue, which would not be in memory 226 * (but included in calculation for whether or not to use temp file) */ 227 228 /*(allow slightly larger mem use if FDEVENT_STREAM_RESPONSE_BUFMIN 229 * to reduce creation of temp files when backend producer will be 230 * blocked until more data is sent to network to client)*/ 231 232 const chunk * const c = cq->last; 233 return 234 ((c && c->type == FILE_CHUNK && c->file.is_temp) 235 || chunkqueue_length(cq) + len 236 > ((r->conf.stream_response_body & FDEVENT_STREAM_RESPONSE_BUFMIN) 237 ? 128*1024 238 : 64*1024)); 239 } 240 241 int http_chunk_append_buffer(request_st * const r, buffer * const mem) { 242 size_t len = buffer_string_length(mem); 243 if (0 == len) return 0; 244 245 chunkqueue * const cq = &r->write_queue; 246 247 if (http_chunk_uses_tempfile(r, cq, len)) 248 return http_chunk_append_to_tempfile(r, mem->ptr, len); 249 250 if (r->resp_send_chunked) 251 http_chunk_len_append(cq, len); 252 253 /*(chunkqueue_append_buffer() might steal buffer contents)*/ 254 chunkqueue_append_buffer(cq, mem); 255 256 if (r->resp_send_chunked) 257 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 258 259 return 0; 260 } 261 262 int http_chunk_append_mem(request_st * const r, const char * const mem, const size_t len) { 263 if (0 == len) return 0; 264 force_assert(NULL != mem); 265 266 chunkqueue * const cq = &r->write_queue; 267 268 if (http_chunk_uses_tempfile(r, cq, len)) 269 return http_chunk_append_to_tempfile(r, mem, len); 270 271 if (r->resp_send_chunked) 272 http_chunk_len_append(cq, len); 273 274 chunkqueue_append_mem(cq, mem, len); 275 276 if (r->resp_send_chunked) 277 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 278 279 return 0; 280 } 281 282 int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, const size_t len) { 283 if (0 == len) return 0; 284 285 chunkqueue * const cq = &r->write_queue; 286 287 if (http_chunk_uses_tempfile(r, cq, len)) 288 return http_chunk_append_cq_to_tempfile(r, src, len); 289 290 if (r->resp_send_chunked) 291 http_chunk_len_append(cq, len); 292 293 chunkqueue_steal(cq, src, len); 294 295 if (r->resp_send_chunked) 296 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 297 298 return 0; 299 } 300 301 void http_chunk_close(request_st * const r) { 302 if (!r->resp_send_chunked) return; 303 304 if (r->gw_dechunk) { 305 if (!r->gw_dechunk->done) 306 r->keep_alive = 0; 307 } 308 else 309 chunkqueue_append_mem(&r->write_queue, CONST_STR_LEN("0\r\n\r\n")); 310 } 311 312 static int 313 http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len) 314 { 315 if (r->gw_dechunk->done) return -1; /*(excess data)*/ 316 317 buffer * const h = &r->gw_dechunk->b; 318 off_t te_chunked = r->gw_dechunk->gw_chunked; 319 while (len) { 320 if (0 == te_chunked) { 321 const char *p; 322 unsigned char *s = (unsigned char *)mem; 323 off_t hsz; 324 if (buffer_string_is_empty(h)) { 325 /*(short-circuit common case: complete chunked header line)*/ 326 p = memchr(mem, '\n', (size_t)len); 327 if (p) 328 hsz = (off_t)(++p - mem); 329 else { 330 if (len >= 1024) { 331 log_error(r->conf.errh, __FILE__, __LINE__, 332 "chunked header line too long"); 333 return -1; 334 } 335 buffer_append_string_len(h, mem, (uint32_t)len); 336 break; /* incomplete HTTP chunked header line */ 337 } 338 } 339 else { 340 uint32_t hlen = buffer_string_length(h); 341 p = strchr(h->ptr, '\n'); 342 if (p) 343 hsz = (off_t)(++p - h->ptr); 344 else { 345 p = memchr(mem, '\n', (size_t)len); 346 hsz = (p ? (off_t)(++p - mem) : len); 347 if ((off_t)(1024 - hlen) < hsz) { 348 log_error(r->conf.errh, __FILE__, __LINE__, 349 "chunked header line too long"); 350 return -1; 351 } 352 buffer_append_string_len(h, mem, hsz); 353 if (NULL == p) break;/*incomplete HTTP chunked header line*/ 354 mem += hsz; 355 len -= hsz; 356 hsz = 0; 357 } 358 s = (unsigned char *)h->ptr;/*(note: read h->ptr after append)*/ 359 } 360 361 for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) { 362 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) { 363 log_error(r->conf.errh, __FILE__, __LINE__, 364 "chunked data size too large"); 365 return -1; 366 } 367 te_chunked <<= 4; 368 te_chunked |= u; 369 } 370 if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/ 371 while (*s == ' ' || *s == '\t') ++s; 372 if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/ 373 log_error(r->conf.errh, __FILE__, __LINE__, 374 "chunked header invalid chars"); 375 return -1; 376 } 377 378 if (0 == te_chunked) { 379 /* do not consume final chunked header until 380 * (optional) trailers received along with 381 * request-ending blank line "\r\n" */ 382 if (len - hsz >= 2 && p[0] == '\r' && p[1] == '\n') { 383 if (len - hsz > 2) return -1; /*(excess data)*/ 384 /* common case with no trailers; final \r\n received */ 385 #if 0 /*(avoid allocation for common case; users must check)*/ 386 if (buffer_is_empty(h)) 387 buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n")); 388 #else 389 buffer_clear(h); 390 #endif 391 r->gw_dechunk->done = r->http_status; 392 break; 393 } 394 395 /* accumulate trailers and check for end of trailers */ 396 /* XXX: reuse r->conf.max_request_field_size 397 * or have separate limit? */ 398 uint32_t mlen = buffer_string_length(h); 399 mlen = (r->conf.max_request_field_size > mlen) 400 ? r->conf.max_request_field_size - mlen 401 : 0; 402 if ((off_t)mlen < len) { 403 /* truncate excessively long trailers */ 404 /* (not truncated; passed as-is if r->resp_send_chunked) */ 405 if (r->resp_send_chunked) r->keep_alive = 0; 406 r->gw_dechunk->done = r->http_status; 407 buffer_append_string_len(h, mem, mlen); 408 p = strrchr(h->ptr, '\n'); 409 if (NULL != p) { 410 buffer_string_set_length(h, p + 1 - h->ptr); 411 if (p[-1] != '\r') 412 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 413 } 414 else { /*(should not happen)*/ 415 buffer_clear(h); 416 buffer_append_string_len(h, CONST_STR_LEN("0\r\n")); 417 } 418 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 419 break; 420 } 421 buffer_append_string_len(h, mem, (uint32_t)len); 422 if ((p = strstr(h->ptr, "\r\n\r\n"))) { 423 r->gw_dechunk->done = r->http_status; 424 if (p[4] != '\0') return -1; /*(excess data)*/ 425 /*buffer_string_set_length(h, (uint32_t)(p+4-h->ptr));*/ 426 } 427 break; 428 } 429 430 mem += hsz; 431 len -= hsz; 432 433 te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/ 434 buffer_clear(h); 435 if (0 == len) break; 436 } 437 438 if (te_chunked >= 2) { 439 off_t clen = te_chunked - 2; 440 if (clen > len) clen = len; 441 if (!r->resp_send_chunked 442 && 0 != http_chunk_append_mem(r, mem, clen)) 443 return -1; 444 mem += clen; 445 len -= clen; 446 te_chunked -= clen; 447 if (te_chunked == 2) { 448 if (len >= 2) { 449 if (mem[0] != '\r' || mem[1] != '\n') return -1; 450 mem += 2; 451 len -= 2; 452 te_chunked = 0; 453 } 454 else if (len == 1) { 455 if (mem[0] != '\r') return -1; 456 /*++mem;*/ 457 /*--len;*/ 458 te_chunked = 1; 459 break; 460 } 461 } 462 } 463 else if (1 == te_chunked) { 464 /* finish reading chunk block "\r\n" */ 465 if (mem[0] != '\n') return -1; 466 ++mem; 467 --len; 468 te_chunked = 0; 469 } 470 } 471 if (r->gw_dechunk->done) 472 r->resp_body_finished = 1; 473 r->gw_dechunk->gw_chunked = te_chunked; 474 return 0; 475 } 476 477 int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem) 478 { 479 /* Note: this routine is separate from http_chunk_decode_append_mem() to 480 * potentially avoid copying in http_chunk_append_buffer(). Otherwise this 481 * would be: return http_chunk_decode_append_mem(r, CONST_BUF_LEN(mem)); */ 482 483 /*(called by funcs receiving data from backends, which might be chunked)*/ 484 /*(separate from http_chunk_append_buffer() called by numerous others)*/ 485 if (!r->resp_decode_chunked) { 486 if (r->resp_body_scratchpad > 0) { 487 off_t len = (off_t)buffer_string_length(mem); 488 r->resp_body_scratchpad -= len; 489 if (r->resp_body_scratchpad <= 0) { 490 r->resp_body_finished = 1; 491 if (r->resp_body_scratchpad < 0) { 492 /*(silently truncate if data exceeds Content-Length)*/ 493 len += r->resp_body_scratchpad; 494 r->resp_body_scratchpad = 0; 495 buffer_string_set_length(mem, (uint32_t)len); 496 } 497 } 498 } 499 else if (0 == r->resp_body_scratchpad) { 500 /*(silently truncate if data exceeds Content-Length)*/ 501 return 0; 502 } 503 return http_chunk_append_buffer(r, mem); 504 } 505 506 /* might avoid copy by transferring buffer if buffer is all data that is 507 * part of large chunked block, but choosing to *not* expand that out here*/ 508 if (0 != http_chunk_decode_append_data(r, CONST_BUF_LEN(mem))) 509 return -1; 510 511 /* no need to decode chunked to immediately re-encode chunked; 512 * pass through chunked encoding as provided by backend, 513 * though it is still parsed (above) to maintain state. 514 * XXX: consider having callers use chunk buffers for hctx->b 515 * for more efficient data copy avoidance and buffer reuse 516 * note: r->resp_send_chunked = 0 until response headers sent, 517 * which is when Transfer-Encoding: chunked might be chosen */ 518 if (r->resp_send_chunked) { 519 r->resp_send_chunked = 0; 520 int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */ 521 r->resp_send_chunked = 1; 522 return rc; 523 } 524 525 return 0; 526 } 527 528 int http_chunk_decode_append_mem(request_st * const r, const char * const mem, size_t len) 529 { 530 /*(called by funcs receiving data from backends, which might be chunked)*/ 531 /*(separate from http_chunk_append_mem() called by numerous others)*/ 532 if (!r->resp_decode_chunked) { 533 if (r->resp_body_scratchpad > 0) { 534 r->resp_body_scratchpad -= (off_t)len; 535 if (r->resp_body_scratchpad <= 0) { 536 r->resp_body_finished = 1; 537 if (r->resp_body_scratchpad < 0) { 538 /*(silently truncate if data exceeds Content-Length)*/ 539 len = (size_t)(r->resp_body_scratchpad + (off_t)len); 540 r->resp_body_scratchpad = 0; 541 } 542 } 543 } 544 else if (0 == r->resp_body_scratchpad) { 545 /*(silently truncate if data exceeds Content-Length)*/ 546 return 0; 547 } 548 return http_chunk_append_mem(r, mem, len); 549 } 550 551 if (0 != http_chunk_decode_append_data(r, mem, (off_t)len)) 552 return -1; 553 554 /* no need to decode chunked to immediately re-encode chunked; 555 * pass through chunked encoding as provided by backend, 556 * though it is still parsed (above) to maintain state. 557 * note: r->resp_send_chunked = 0 until response headers sent, 558 * which is when Transfer-Encoding: chunked might be chosen */ 559 if (r->resp_send_chunked) { 560 r->resp_send_chunked = 0; 561 int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/ 562 r->resp_send_chunked = 1; 563 return rc; 564 } 565 566 return 0; 567 } 568