1 /* 2 * http_chunk - append response to chunkqueue, possibly in "chunked" encoding 3 * 4 * Fully-rewritten from original 5 * Copyright(c) 2019 Glenn Strauss gstrauss()gluelogic.com All rights reserved 6 * License: BSD 3-clause (same as lighttpd) 7 */ 8 #include "first.h" 9 10 #include "http_chunk.h" 11 #include "chunk.h" 12 #include "stat_cache.h" 13 #include "fdevent.h" 14 #include "log.h" 15 #include "request.h" 16 17 #include <sys/types.h> 18 #include <sys/stat.h> 19 20 #include <stdlib.h> 21 #include <unistd.h> 22 23 #include <errno.h> 24 #include <string.h> 25 26 __attribute_noinline__ 27 static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) { 28 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 29 #if 0 30 buffer b = { buf, 0, sizeof(buf) }; 31 buffer_append_uint_hex(&b, len); 32 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 33 chunkqueue_append_mem(cq, b.ptr, b.used-1); 34 #else 35 int i = (int)(sizeof(buf)); 36 buf[--i] = '\n'; 37 buf[--i] = '\r'; 38 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 39 chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i); 40 #endif 41 } 42 43 __attribute_noinline__ 44 static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) { 45 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 46 #if 0 47 buffer b = { buf, 0, sizeof(buf) }; 48 buffer_append_uint_hex(&b, len); 49 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 50 return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh); 51 #else 52 int i = (int)(sizeof(buf)); 53 buf[--i] = '\n'; 54 buf[--i] = '\r'; 55 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 56 return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh); 57 #endif 58 } 59 60 __attribute_noinline__ 61 static int http_chunk_append_read_fd_range(request_st * const r, const buffer * const fn, const int fd, off_t offset, off_t len) { 62 /* note: this routine should not be used for range requests 63 * unless the total size of ranges requested is small */ 64 /* note: future: could read into existing MEM_CHUNK in cq->last if 65 * there is sufficient space, but would need to adjust for existing 66 * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */ 67 UNUSED(fn); 68 69 chunkqueue * const cq = &r->write_queue; 70 71 if (r->resp_send_chunked) 72 http_chunk_len_append(cq, (uintmax_t)len); 73 74 #ifndef HAVE_PREAD 75 if (-1 == lseek(fd, offset, SEEK_SET)) return -1; 76 #endif 77 buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2+1); 78 ssize_t rd; 79 #ifdef HAVE_PREAD 80 const off_t foff = offset; 81 #endif 82 offset = 0; 83 do { 84 #ifdef HAVE_PREAD 85 rd =pread(fd, b->ptr+offset, (size_t)(len-offset), foff+offset); 86 #else 87 rd = read(fd, b->ptr+offset, (size_t)(len-offset)); 88 #endif 89 } while (rd > 0 ? (offset += rd) != len : errno == EINTR); 90 buffer_commit(b, offset); 91 92 if (r->resp_send_chunked) 93 buffer_append_string_len(b, CONST_STR_LEN("\r\n")); 94 95 chunkqueue_append_buffer_commit(cq); 96 return (rd >= 0) ? 0 : -1; 97 } 98 99 __attribute_noinline__ 100 void http_chunk_append_file_ref_range(request_st * const r, stat_cache_entry * const sce, const off_t offset, off_t len) { 101 chunkqueue * const cq = &r->write_queue; 102 103 if (sce->st.st_size - offset < len) 104 len = sce->st.st_size - offset; 105 if (len <= 0) 106 return; 107 108 if (r->resp_send_chunked) 109 http_chunk_len_append(cq, (uintmax_t)len); 110 111 const buffer * const fn = &sce->name; 112 const int fd = sce->fd; 113 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 114 if (fd >= 0) { 115 chunk * const d = cq->last; 116 d->file.ref = sce; 117 d->file.refchg = stat_cache_entry_refchg; 118 stat_cache_entry_refchg(sce, 1); 119 } 120 121 if (r->resp_send_chunked) 122 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 123 } 124 125 __attribute_noinline__ 126 void http_chunk_append_file_fd_range(request_st * const r, const buffer * const fn, const int fd, const off_t offset, const off_t len) { 127 chunkqueue * const cq = &r->write_queue; 128 129 if (r->resp_send_chunked) 130 http_chunk_len_append(cq, (uintmax_t)len); 131 132 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 133 134 if (r->resp_send_chunked) 135 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 136 } 137 138 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) { 139 if (sz > 32768 || !r->resp_send_chunked) { 140 http_chunk_append_file_fd_range(r, fn, fd, 0, sz); 141 return 0; 142 } 143 144 /*(read small files into memory)*/ 145 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 146 close(fd); 147 return rc; 148 } 149 150 int http_chunk_append_file_ref(request_st * const r, stat_cache_entry * const sce) { 151 const off_t sz = sce->st.st_size; 152 if (sz > 32768 || !r->resp_send_chunked) { 153 http_chunk_append_file_ref_range(r, sce, 0, sz); 154 return 0; 155 } 156 157 /*(read small files into memory)*/ 158 const buffer * const fn = &sce->name; 159 const int fd = sce->fd; 160 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 161 return rc; 162 } 163 164 __attribute_noinline__ 165 static int http_chunk_append_to_tempfile(request_st * const r, const char * const mem, const size_t len) { 166 chunkqueue * const cq = &r->write_queue; 167 log_error_st * const errh = r->conf.errh; 168 169 if (r->resp_send_chunked 170 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 171 return -1; 172 173 if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) 174 return -1; 175 176 if (r->resp_send_chunked 177 && 0 != 178 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 179 return -1; 180 181 return 0; 182 } 183 184 __attribute_noinline__ 185 static int http_chunk_append_cq_to_tempfile(request_st * const r, chunkqueue * const src, const size_t len) { 186 chunkqueue * const cq = &r->write_queue; 187 log_error_st * const errh = r->conf.errh; 188 189 if (r->resp_send_chunked 190 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 191 return -1; 192 193 if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) 194 return -1; 195 196 if (r->resp_send_chunked 197 && 0 != 198 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 199 return -1; 200 201 return 0; 202 } 203 204 /*(inlined by compiler optimizer)*/ 205 __attribute_pure__ 206 static int http_chunk_uses_tempfile(const chunkqueue * const cq, const size_t len) { 207 208 /* current usage does not append_mem or append_buffer after appending 209 * file, so not checking if users of this interface have appended large 210 * (references to) files to chunkqueue, which would not be in memory 211 * (but included in calculation for whether or not to use temp file) */ 212 const chunk * const c = cq->last; 213 return 214 ((c && c->type == FILE_CHUNK && c->file.is_temp) 215 || chunkqueue_length(cq) + len > 65536); 216 } 217 218 __attribute_noinline__ 219 int http_chunk_append_buffer(request_st * const r, buffer * const mem) { 220 size_t len = mem ? buffer_clen(mem) : 0; 221 if (0 == len) return 0; 222 223 chunkqueue * const cq = &r->write_queue; 224 225 if (http_chunk_uses_tempfile(cq, len)) { 226 int rc = http_chunk_append_to_tempfile(r, mem->ptr, len); 227 buffer_clear(mem); 228 return rc; 229 } 230 231 if (r->resp_send_chunked) 232 http_chunk_len_append(cq, len); 233 234 /*(chunkqueue_append_buffer() might steal buffer contents)*/ 235 chunkqueue_append_buffer(cq, mem); 236 237 if (r->resp_send_chunked) 238 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 239 240 return 0; 241 } 242 243 __attribute_noinline__ 244 int http_chunk_append_mem(request_st * const r, const char * const mem, const size_t len) { 245 if (0 == len) return 0; 246 force_assert(NULL != mem); 247 248 chunkqueue * const cq = &r->write_queue; 249 250 if (http_chunk_uses_tempfile(cq, len)) 251 return http_chunk_append_to_tempfile(r, mem, len); 252 253 if (r->resp_send_chunked) 254 http_chunk_len_append(cq, len); 255 256 chunkqueue_append_mem(cq, mem, len); 257 258 if (r->resp_send_chunked) 259 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 260 261 return 0; 262 } 263 264 int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, const size_t len) { 265 if (0 == len) return 0; 266 267 chunkqueue * const cq = &r->write_queue; 268 269 if (http_chunk_uses_tempfile(cq, len)) 270 return http_chunk_append_cq_to_tempfile(r, src, len); 271 272 if (r->resp_send_chunked) 273 http_chunk_len_append(cq, len); 274 275 chunkqueue_steal(cq, src, len); 276 277 if (r->resp_send_chunked) 278 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 279 280 return 0; 281 } 282 283 void http_chunk_close(request_st * const r) { 284 if (!r->resp_send_chunked) return; 285 286 if (r->gw_dechunk) { 287 if (!r->gw_dechunk->done) 288 r->keep_alive = 0; 289 } 290 else 291 chunkqueue_append_mem(&r->write_queue, CONST_STR_LEN("0\r\n\r\n")); 292 } 293 294 static int 295 http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len) 296 { 297 if (r->gw_dechunk->done) return -1; /*(excess data)*/ 298 299 buffer * const h = &r->gw_dechunk->b; 300 off_t te_chunked = r->gw_dechunk->gw_chunked; 301 while (len) { 302 if (0 == te_chunked) { 303 const char *p; 304 unsigned char *s = (unsigned char *)mem; 305 off_t hsz; 306 if (buffer_is_blank(h)) { 307 /*(short-circuit common case: complete chunked header line)*/ 308 p = memchr(mem, '\n', (size_t)len); 309 if (p) 310 hsz = (off_t)(++p - mem); 311 else { 312 if (len >= 1024) { 313 log_error(r->conf.errh, __FILE__, __LINE__, 314 "chunked header line too long"); 315 return -1; 316 } 317 buffer_append_string_len(h, mem, (uint32_t)len); 318 break; /* incomplete HTTP chunked header line */ 319 } 320 } 321 else { 322 uint32_t hlen = buffer_clen(h); 323 p = strchr(h->ptr, '\n'); 324 if (p) 325 hsz = (off_t)(++p - h->ptr); 326 else { 327 p = memchr(mem, '\n', (size_t)len); 328 hsz = (p ? (off_t)(++p - mem) : len); 329 if ((off_t)(1024 - hlen) < hsz) { 330 log_error(r->conf.errh, __FILE__, __LINE__, 331 "chunked header line too long"); 332 return -1; 333 } 334 buffer_append_string_len(h, mem, hsz); 335 if (NULL == p) break;/*incomplete HTTP chunked header line*/ 336 mem += hsz; 337 len -= hsz; 338 hsz = 0; 339 } 340 s = (unsigned char *)h->ptr;/*(note: read h->ptr after append)*/ 341 } 342 343 for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) { 344 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) { 345 log_error(r->conf.errh, __FILE__, __LINE__, 346 "chunked data size too large"); 347 return -1; 348 } 349 te_chunked <<= 4; 350 te_chunked |= u; 351 } 352 if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/ 353 while (*s == ' ' || *s == '\t') ++s; 354 if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/ 355 log_error(r->conf.errh, __FILE__, __LINE__, 356 "chunked header invalid chars"); 357 return -1; 358 } 359 360 if (0 == te_chunked) { 361 /* do not consume final chunked header until 362 * (optional) trailers received along with 363 * request-ending blank line "\r\n" */ 364 if (len - hsz >= 2 && p[0] == '\r' && p[1] == '\n') { 365 if (len - hsz > 2) return -1; /*(excess data)*/ 366 /* common case with no trailers; final \r\n received */ 367 #if 0 /*(avoid allocation for common case; users must check)*/ 368 if (buffer_is_unset(h)) 369 buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n")); 370 #else 371 buffer_clear(h); 372 #endif 373 r->gw_dechunk->done = r->http_status; 374 break; 375 } 376 377 /* accumulate trailers and check for end of trailers */ 378 /* XXX: reuse r->conf.max_request_field_size 379 * or have separate limit? */ 380 uint32_t mlen = buffer_clen(h); 381 mlen = (r->conf.max_request_field_size > mlen) 382 ? r->conf.max_request_field_size - mlen 383 : 0; 384 if ((off_t)mlen < len) { 385 /* truncate excessively long trailers */ 386 /* (not truncated; passed as-is if r->resp_send_chunked) */ 387 if (r->resp_send_chunked) r->keep_alive = 0; 388 r->gw_dechunk->done = r->http_status; 389 buffer_append_string_len(h, mem, mlen); 390 p = strrchr(h->ptr, '\n'); 391 if (NULL != p) { 392 buffer_truncate(h, p + 1 - h->ptr); 393 if (p[-1] != '\r') 394 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 395 } 396 else { /*(should not happen)*/ 397 buffer_clear(h); 398 buffer_append_string_len(h, CONST_STR_LEN("0\r\n")); 399 } 400 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 401 break; 402 } 403 buffer_append_string_len(h, mem, (uint32_t)len); 404 if ((p = strstr(h->ptr, "\r\n\r\n"))) { 405 r->gw_dechunk->done = r->http_status; 406 if (p[4] != '\0') return -1; /*(excess data)*/ 407 /*buffer_truncate(h, (uint32_t)(p+4-h->ptr));*/ 408 } 409 break; 410 } 411 412 mem += hsz; 413 len -= hsz; 414 415 te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/ 416 buffer_clear(h); 417 if (0 == len) break; 418 } 419 420 if (te_chunked >= 2) { 421 off_t clen = te_chunked - 2; 422 if (clen > len) clen = len; 423 if (!r->resp_send_chunked 424 && 0 != http_chunk_append_mem(r, mem, clen)) 425 return -1; 426 mem += clen; 427 len -= clen; 428 te_chunked -= clen; 429 if (te_chunked == 2) { 430 if (len >= 2) { 431 if (mem[0] != '\r' || mem[1] != '\n') return -1; 432 mem += 2; 433 len -= 2; 434 te_chunked = 0; 435 } 436 else if (len == 1) { 437 if (mem[0] != '\r') return -1; 438 /*++mem;*/ 439 /*--len;*/ 440 te_chunked = 1; 441 break; 442 } 443 } 444 } 445 else if (1 == te_chunked) { 446 /* finish reading chunk block "\r\n" */ 447 if (mem[0] != '\n') return -1; 448 ++mem; 449 --len; 450 te_chunked = 0; 451 } 452 } 453 if (r->gw_dechunk->done) 454 r->resp_body_finished = 1; 455 r->gw_dechunk->gw_chunked = te_chunked; 456 return 0; 457 } 458 459 int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem) 460 { 461 /* Note: this routine is separate from http_chunk_decode_append_mem() to 462 * potentially avoid copying in http_chunk_append_buffer(). Otherwise this 463 * would be: return http_chunk_decode_append_mem(r, BUF_PTR_LEN(mem)); */ 464 465 /*(called by funcs receiving chunked data from backends)*/ 466 /*(separate from http_chunk_append_buffer() called by numerous others)*/ 467 468 /* might avoid copy by transferring buffer if buffer is all data that is 469 * part of large chunked block, but choosing to *not* expand that out here*/ 470 if (0 != http_chunk_decode_append_data(r, BUF_PTR_LEN(mem))) 471 return -1; 472 473 /* no need to decode chunked to immediately re-encode chunked; 474 * pass through chunked encoding as provided by backend, 475 * though it is still parsed (above) to maintain state. 476 * XXX: consider having callers use chunk buffers for hctx->b 477 * for more efficient data copy avoidance and buffer reuse 478 * note: r->resp_send_chunked = 0 until response headers sent, 479 * which is when Transfer-Encoding: chunked might be chosen */ 480 if (r->resp_send_chunked) { 481 r->resp_send_chunked = 0; 482 int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */ 483 r->resp_send_chunked = 1; 484 return rc; 485 } 486 else 487 buffer_clear(mem); 488 489 return 0; 490 } 491 492 int http_chunk_decode_append_mem(request_st * const r, const char * const mem, size_t len) 493 { 494 /*(called by funcs receiving chunked data from backends)*/ 495 /*(separate from http_chunk_append_mem() called by numerous others)*/ 496 497 if (0 != http_chunk_decode_append_data(r, mem, (off_t)len)) 498 return -1; 499 500 /* no need to decode chunked to immediately re-encode chunked; 501 * pass through chunked encoding as provided by backend, 502 * though it is still parsed (above) to maintain state. 503 * note: r->resp_send_chunked = 0 until response headers sent, 504 * which is when Transfer-Encoding: chunked might be chosen */ 505 if (r->resp_send_chunked) { 506 r->resp_send_chunked = 0; 507 int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/ 508 r->resp_send_chunked = 1; 509 return rc; 510 } 511 512 return 0; 513 } 514