1 /* 2 * http_chunk - append response to chunkqueue, possibly in "chunked" encoding 3 * 4 * Fully-rewritten from original 5 * Copyright(c) 2019 Glenn Strauss gstrauss()gluelogic.com All rights reserved 6 * License: BSD 3-clause (same as lighttpd) 7 */ 8 #include "first.h" 9 10 #include "http_chunk.h" 11 #include "chunk.h" 12 #include "stat_cache.h" 13 #include "fdevent.h" 14 #include "log.h" 15 #include "request.h" 16 17 #include <sys/types.h> 18 #include <sys/stat.h> 19 20 #include <stdlib.h> 21 #include <unistd.h> 22 23 #include <errno.h> 24 #include <string.h> 25 26 __attribute_noinline__ 27 static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) { 28 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 29 #if 0 30 buffer b = { buf, 0, sizeof(buf) }; 31 buffer_append_uint_hex(&b, len); 32 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 33 chunkqueue_append_mem(cq, b.ptr, b.used-1); 34 #else 35 int i = (int)(sizeof(buf)); 36 buf[--i] = '\n'; 37 buf[--i] = '\r'; 38 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 39 chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i); 40 #endif 41 } 42 43 __attribute_noinline__ 44 static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) { 45 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 46 #if 0 47 buffer b = { buf, 0, sizeof(buf) }; 48 buffer_append_uint_hex(&b, len); 49 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 50 return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh); 51 #else 52 int i = (int)(sizeof(buf)); 53 buf[--i] = '\n'; 54 buf[--i] = '\r'; 55 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 56 return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh); 57 #endif 58 } 59 60 __attribute_noinline__ 61 static int http_chunk_append_read_fd_range(request_st * const r, const buffer * const fn, const int fd, off_t offset, off_t len) { 62 /* note: this routine should not be used for range requests 63 * unless the total size of ranges requested is small */ 64 /* note: future: could read into existing MEM_CHUNK in cq->last if 65 * there is sufficient space, but would need to adjust for existing 66 * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */ 67 UNUSED(fn); 68 69 chunkqueue * const cq = &r->write_queue; 70 71 if (r->resp_send_chunked) 72 http_chunk_len_append(cq, (uintmax_t)len); 73 74 buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2+1); 75 ssize_t rd; 76 const off_t foff = offset; 77 offset = 0; 78 do { 79 rd = chunk_file_pread(fd, b->ptr+offset, (size_t)len, foff+offset); 80 } while (rd > 0 && (offset += rd, len -= rd)); 81 buffer_commit(b, offset); 82 83 if (r->resp_send_chunked) 84 buffer_append_string_len(b, CONST_STR_LEN("\r\n")); 85 86 chunkqueue_append_buffer_commit(cq); 87 return (len == 0) ? 0 : -1; 88 } 89 90 __attribute_noinline__ 91 void http_chunk_append_file_ref_range(request_st * const r, stat_cache_entry * const sce, const off_t offset, off_t len) { 92 chunkqueue * const cq = &r->write_queue; 93 94 if (sce->st.st_size - offset < len) 95 len = sce->st.st_size - offset; 96 if (len <= 0) 97 return; 98 99 if (r->resp_send_chunked) 100 http_chunk_len_append(cq, (uintmax_t)len); 101 102 const buffer * const fn = &sce->name; 103 const int fd = sce->fd; 104 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 105 if (fd >= 0) { 106 chunk * const d = cq->last; 107 d->file.ref = sce; 108 d->file.refchg = stat_cache_entry_refchg; 109 stat_cache_entry_refchg(sce, 1); 110 } 111 112 if (r->resp_send_chunked) 113 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 114 } 115 116 __attribute_noinline__ 117 void http_chunk_append_file_fd_range(request_st * const r, const buffer * const fn, const int fd, const off_t offset, const off_t len) { 118 chunkqueue * const cq = &r->write_queue; 119 120 if (r->resp_send_chunked) 121 http_chunk_len_append(cq, (uintmax_t)len); 122 123 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 124 125 if (r->resp_send_chunked) 126 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 127 } 128 129 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) { 130 if (sz > 32768 || !r->resp_send_chunked) { 131 http_chunk_append_file_fd_range(r, fn, fd, 0, sz); 132 return 0; 133 } 134 135 /*(read small files into memory)*/ 136 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 137 close(fd); 138 return rc; 139 } 140 141 int http_chunk_append_file_ref(request_st * const r, stat_cache_entry * const sce) { 142 const off_t sz = sce->st.st_size; 143 if (sz > 32768 || !r->resp_send_chunked) { 144 http_chunk_append_file_ref_range(r, sce, 0, sz); 145 return 0; 146 } 147 148 /*(read small files into memory)*/ 149 const buffer * const fn = &sce->name; 150 const int fd = sce->fd; 151 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 152 return rc; 153 } 154 155 __attribute_noinline__ 156 static int http_chunk_append_to_tempfile(request_st * const r, const char * const mem, const size_t len) { 157 chunkqueue * const cq = &r->write_queue; 158 log_error_st * const errh = r->conf.errh; 159 160 if (r->resp_send_chunked 161 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 162 return -1; 163 164 if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) 165 return -1; 166 167 if (r->resp_send_chunked 168 && 0 != 169 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 170 return -1; 171 172 return 0; 173 } 174 175 __attribute_noinline__ 176 static int http_chunk_append_cq_to_tempfile(request_st * const r, chunkqueue * const src, const size_t len) { 177 chunkqueue * const cq = &r->write_queue; 178 log_error_st * const errh = r->conf.errh; 179 180 if (r->resp_send_chunked 181 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 182 return -1; 183 184 if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) 185 return -1; 186 187 if (r->resp_send_chunked 188 && 0 != 189 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 190 return -1; 191 192 return 0; 193 } 194 195 /*(inlined by compiler optimizer)*/ 196 __attribute_pure__ 197 static int http_chunk_uses_tempfile(const chunkqueue * const cq, const size_t len) { 198 199 /* current usage does not append_mem or append_buffer after appending 200 * file, so not checking if users of this interface have appended large 201 * (references to) files to chunkqueue, which would not be in memory 202 * (but included in calculation for whether or not to use temp file) */ 203 const chunk * const c = cq->last; 204 return 205 ((c && c->type == FILE_CHUNK && c->file.is_temp) 206 || chunkqueue_length(cq) + len > 65536); 207 } 208 209 __attribute_noinline__ 210 int http_chunk_append_buffer(request_st * const r, buffer * const mem) { 211 size_t len = mem ? buffer_clen(mem) : 0; 212 if (0 == len) return 0; 213 214 chunkqueue * const cq = &r->write_queue; 215 216 if (http_chunk_uses_tempfile(cq, len)) { 217 int rc = http_chunk_append_to_tempfile(r, mem->ptr, len); 218 buffer_clear(mem); 219 return rc; 220 } 221 222 if (r->resp_send_chunked) 223 http_chunk_len_append(cq, len); 224 225 /*(chunkqueue_append_buffer() might steal buffer contents)*/ 226 chunkqueue_append_buffer(cq, mem); 227 228 if (r->resp_send_chunked) 229 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 230 231 return 0; 232 } 233 234 __attribute_noinline__ 235 int http_chunk_append_mem(request_st * const r, const char * const mem, const size_t len) { 236 if (0 == len) return 0; 237 force_assert(NULL != mem); 238 239 chunkqueue * const cq = &r->write_queue; 240 241 if (http_chunk_uses_tempfile(cq, len)) 242 return http_chunk_append_to_tempfile(r, mem, len); 243 244 if (r->resp_send_chunked) 245 http_chunk_len_append(cq, len); 246 247 chunkqueue_append_mem(cq, mem, len); 248 249 if (r->resp_send_chunked) 250 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 251 252 return 0; 253 } 254 255 int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, const size_t len) { 256 if (0 == len) return 0; 257 258 chunkqueue * const cq = &r->write_queue; 259 260 if (http_chunk_uses_tempfile(cq, len)) 261 return http_chunk_append_cq_to_tempfile(r, src, len); 262 263 if (r->resp_send_chunked) 264 http_chunk_len_append(cq, len); 265 266 chunkqueue_steal(cq, src, len); 267 268 if (r->resp_send_chunked) 269 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 270 271 return 0; 272 } 273 274 void http_chunk_close(request_st * const r) { 275 if (!r->resp_send_chunked) return; 276 277 if (r->gw_dechunk) { 278 if (!r->gw_dechunk->done) 279 r->keep_alive = 0; 280 } 281 else 282 chunkqueue_append_mem(&r->write_queue, CONST_STR_LEN("0\r\n\r\n")); 283 } 284 285 static int 286 http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len) 287 { 288 if (r->gw_dechunk->done) return -1; /*(excess data)*/ 289 290 buffer * const h = &r->gw_dechunk->b; 291 off_t te_chunked = r->gw_dechunk->gw_chunked; 292 while (len) { 293 if (0 == te_chunked) { 294 const char *p; 295 unsigned char *s = (unsigned char *)mem; 296 off_t hsz; 297 if (buffer_is_blank(h)) { 298 /*(short-circuit common case: complete chunked header line)*/ 299 p = memchr(mem, '\n', (size_t)len); 300 if (p) 301 hsz = (off_t)(++p - mem); 302 else { 303 if (len >= 1024) { 304 log_error(r->conf.errh, __FILE__, __LINE__, 305 "chunked header line too long"); 306 return -1; 307 } 308 buffer_append_string_len(h, mem, (uint32_t)len); 309 break; /* incomplete HTTP chunked header line */ 310 } 311 } 312 else { 313 uint32_t hlen = buffer_clen(h); 314 p = strchr(h->ptr, '\n'); 315 if (p) 316 hsz = (off_t)(++p - h->ptr); 317 else { 318 p = memchr(mem, '\n', (size_t)len); 319 hsz = (p ? (off_t)(++p - mem) : len); 320 if ((off_t)(1024 - hlen) < hsz) { 321 log_error(r->conf.errh, __FILE__, __LINE__, 322 "chunked header line too long"); 323 return -1; 324 } 325 buffer_append_string_len(h, mem, hsz); 326 if (NULL == p) break;/*incomplete HTTP chunked header line*/ 327 mem += hsz; 328 len -= hsz; 329 hsz = 0; 330 } 331 s = (unsigned char *)h->ptr;/*(note: read h->ptr after append)*/ 332 } 333 334 for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) { 335 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) { 336 log_error(r->conf.errh, __FILE__, __LINE__, 337 "chunked data size too large"); 338 return -1; 339 } 340 te_chunked <<= 4; 341 te_chunked |= u; 342 } 343 if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/ 344 while (*s == ' ' || *s == '\t') ++s; 345 if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/ 346 log_error(r->conf.errh, __FILE__, __LINE__, 347 "chunked header invalid chars"); 348 return -1; 349 } 350 351 if (0 == te_chunked) { 352 /* do not consume final chunked header until 353 * (optional) trailers received along with 354 * request-ending blank line "\r\n" */ 355 if (len - hsz >= 2 && p[0] == '\r' && p[1] == '\n') { 356 if (len - hsz > 2) return -1; /*(excess data)*/ 357 /* common case with no trailers; final \r\n received */ 358 #if 0 /*(avoid allocation for common case; users must check)*/ 359 if (buffer_is_unset(h)) 360 buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n")); 361 #else 362 buffer_clear(h); 363 #endif 364 r->gw_dechunk->done = r->http_status; 365 break; 366 } 367 368 /* accumulate trailers and check for end of trailers */ 369 /* XXX: reuse r->conf.max_request_field_size 370 * or have separate limit? */ 371 uint32_t mlen = buffer_clen(h); 372 mlen = (r->conf.max_request_field_size > mlen) 373 ? r->conf.max_request_field_size - mlen 374 : 0; 375 if ((off_t)mlen < len) { 376 /* truncate excessively long trailers */ 377 /* (not truncated; passed as-is if r->resp_send_chunked) */ 378 if (r->resp_send_chunked) r->keep_alive = 0; 379 r->gw_dechunk->done = r->http_status; 380 buffer_append_string_len(h, mem, mlen); 381 p = strrchr(h->ptr, '\n'); 382 if (NULL != p) { 383 buffer_truncate(h, p + 1 - h->ptr); 384 if (p[-1] != '\r') 385 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 386 } 387 else { /*(should not happen)*/ 388 buffer_clear(h); 389 buffer_append_string_len(h, CONST_STR_LEN("0\r\n")); 390 } 391 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 392 break; 393 } 394 buffer_append_string_len(h, mem, (uint32_t)len); 395 if ((p = strstr(h->ptr, "\r\n\r\n"))) { 396 r->gw_dechunk->done = r->http_status; 397 if (p[4] != '\0') return -1; /*(excess data)*/ 398 /*buffer_truncate(h, (uint32_t)(p+4-h->ptr));*/ 399 } 400 break; 401 } 402 403 mem += hsz; 404 len -= hsz; 405 406 te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/ 407 buffer_clear(h); 408 if (0 == len) break; 409 } 410 411 if (te_chunked >= 2) { 412 off_t clen = te_chunked - 2; 413 if (clen > len) clen = len; 414 if (!r->resp_send_chunked 415 && 0 != http_chunk_append_mem(r, mem, clen)) 416 return -1; 417 mem += clen; 418 len -= clen; 419 te_chunked -= clen; 420 if (te_chunked == 2) { 421 if (len >= 2) { 422 if (mem[0] != '\r' || mem[1] != '\n') return -1; 423 mem += 2; 424 len -= 2; 425 te_chunked = 0; 426 } 427 else if (len == 1) { 428 if (mem[0] != '\r') return -1; 429 /*++mem;*/ 430 /*--len;*/ 431 te_chunked = 1; 432 break; 433 } 434 } 435 } 436 else if (1 == te_chunked) { 437 /* finish reading chunk block "\r\n" */ 438 if (mem[0] != '\n') return -1; 439 ++mem; 440 --len; 441 te_chunked = 0; 442 } 443 } 444 if (r->gw_dechunk->done) 445 r->resp_body_finished = 1; 446 r->gw_dechunk->gw_chunked = te_chunked; 447 return 0; 448 } 449 450 int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem) 451 { 452 /* Note: this routine is separate from http_chunk_decode_append_mem() to 453 * potentially avoid copying in http_chunk_append_buffer(). Otherwise this 454 * would be: return http_chunk_decode_append_mem(r, BUF_PTR_LEN(mem)); */ 455 456 /*(called by funcs receiving chunked data from backends)*/ 457 /*(separate from http_chunk_append_buffer() called by numerous others)*/ 458 459 /* might avoid copy by transferring buffer if buffer is all data that is 460 * part of large chunked block, but choosing to *not* expand that out here*/ 461 if (0 != http_chunk_decode_append_data(r, BUF_PTR_LEN(mem))) 462 return -1; 463 464 /* no need to decode chunked to immediately re-encode chunked; 465 * pass through chunked encoding as provided by backend, 466 * though it is still parsed (above) to maintain state. 467 * XXX: consider having callers use chunk buffers for hctx->b 468 * for more efficient data copy avoidance and buffer reuse 469 * note: r->resp_send_chunked = 0 until response headers sent, 470 * which is when Transfer-Encoding: chunked might be chosen */ 471 if (r->resp_send_chunked) { 472 r->resp_send_chunked = 0; 473 int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */ 474 r->resp_send_chunked = 1; 475 return rc; 476 } 477 else 478 buffer_clear(mem); 479 480 return 0; 481 } 482 483 int http_chunk_decode_append_mem(request_st * const r, const char * const mem, size_t len) 484 { 485 /*(called by funcs receiving chunked data from backends)*/ 486 /*(separate from http_chunk_append_mem() called by numerous others)*/ 487 488 if (0 != http_chunk_decode_append_data(r, mem, (off_t)len)) 489 return -1; 490 491 /* no need to decode chunked to immediately re-encode chunked; 492 * pass through chunked encoding as provided by backend, 493 * though it is still parsed (above) to maintain state. 494 * note: r->resp_send_chunked = 0 until response headers sent, 495 * which is when Transfer-Encoding: chunked might be chosen */ 496 if (r->resp_send_chunked) { 497 r->resp_send_chunked = 0; 498 int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/ 499 r->resp_send_chunked = 1; 500 return rc; 501 } 502 503 return 0; 504 } 505