1 /* 2 * http_chunk - append response to chunkqueue, possibly in "chunked" encoding 3 * 4 * Fully-rewritten from original 5 * Copyright(c) 2019 Glenn Strauss gstrauss()gluelogic.com All rights reserved 6 * License: BSD 3-clause (same as lighttpd) 7 */ 8 #include "first.h" 9 10 #include "http_chunk.h" 11 #include "base.h" 12 #include "chunk.h" 13 #include "stat_cache.h" 14 #include "fdevent.h" 15 #include "log.h" 16 17 #include <sys/types.h> 18 #include <sys/stat.h> 19 20 #include <stdlib.h> 21 #include <unistd.h> 22 23 #include <errno.h> 24 #include <string.h> 25 26 static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) { 27 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 28 #if 0 29 buffer b = { buf, 0, sizeof(buf) }; 30 buffer_append_uint_hex(&b, len); 31 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 32 chunkqueue_append_mem(cq, b.ptr, b.used-1); 33 #else 34 int i = (int)(sizeof(buf)); 35 buf[--i] = '\n'; 36 buf[--i] = '\r'; 37 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 38 chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i); 39 #endif 40 } 41 42 static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) { 43 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ 44 #if 0 45 buffer b = { buf, 0, sizeof(buf) }; 46 buffer_append_uint_hex(&b, len); 47 buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); 48 return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh); 49 #else 50 int i = (int)(sizeof(buf)); 51 buf[--i] = '\n'; 52 buf[--i] = '\r'; 53 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); 54 return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh); 55 #endif 56 } 57 58 static int http_chunk_append_file_open_fstat(const request_st * const r, const buffer * const fn, struct stat * const st) { 59 return 60 (r->conf.follow_symlink 61 || !stat_cache_path_contains_symlink(fn, r->conf.errh)) 62 ? stat_cache_open_rdonly_fstat(fn, st, r->conf.follow_symlink) 63 : -1; 64 } 65 66 static int http_chunk_append_read_fd_range(request_st * const r, const buffer * const fn, const int fd, off_t offset, off_t len) { 67 /* note: this routine should not be used for range requests 68 * unless the total size of ranges requested is small */ 69 /* note: future: could read into existing MEM_CHUNK in cq->last if 70 * there is sufficient space, but would need to adjust for existing 71 * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */ 72 UNUSED(fn); 73 74 chunkqueue * const cq = r->write_queue; 75 76 if (r->resp_send_chunked) 77 http_chunk_len_append(cq, (uintmax_t)len); 78 79 if (0 != offset && -1 == lseek(fd, offset, SEEK_SET)) return -1; 80 buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2); 81 ssize_t rd; 82 offset = 0; 83 do { 84 rd = read(fd, b->ptr+offset, len-offset); 85 } while (rd > 0 ? (offset += rd, len -= rd) : errno == EINTR); 86 buffer_commit(b, offset); 87 88 if (r->resp_send_chunked) 89 buffer_append_string_len(b, CONST_STR_LEN("\r\n")); 90 91 chunkqueue_append_buffer_commit(cq); 92 return (rd >= 0) ? 0 : -1; 93 } 94 95 static void http_chunk_append_file_fd_range(request_st * const r, const buffer * const fn, const int fd, const off_t offset, const off_t len) { 96 chunkqueue * const cq = r->write_queue; 97 98 if (r->resp_send_chunked) 99 http_chunk_len_append(cq, (uintmax_t)len); 100 101 chunkqueue_append_file_fd(cq, fn, fd, offset, len); 102 103 if (r->resp_send_chunked) 104 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 105 } 106 107 int http_chunk_append_file_range(request_st * const r, const buffer * const fn, const off_t offset, off_t len) { 108 struct stat st; 109 const int fd = http_chunk_append_file_open_fstat(r, fn, &st); 110 if (fd < 0) return -1; 111 112 if (-1 == len) { 113 if (offset >= st.st_size) { 114 close(fd); 115 return (offset == st.st_size) ? 0 : -1; 116 } 117 len = st.st_size - offset; 118 } 119 else if (st.st_size - offset < len) { 120 close(fd); 121 return -1; 122 } 123 124 http_chunk_append_file_fd_range(r, fn, fd, offset, len); 125 return 0; 126 } 127 128 int http_chunk_append_file(request_st * const r, const buffer * const fn) { 129 struct stat st; 130 const int fd = http_chunk_append_file_open_fstat(r, fn, &st); 131 if (fd < 0) return -1; 132 http_chunk_append_file_fd(r, fn, fd, st.st_size); 133 return 0; 134 } 135 136 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) { 137 if (sz > 32768 || !r->resp_send_chunked) { 138 http_chunk_append_file_fd_range(r, fn, fd, 0, sz); 139 return 0; 140 } 141 142 /*(read small files into memory)*/ 143 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0; 144 close(fd); 145 return rc; 146 } 147 148 static int http_chunk_append_to_tempfile(request_st * const r, const char * const mem, const size_t len) { 149 chunkqueue * const cq = r->write_queue; 150 log_error_st * const errh = r->conf.errh; 151 152 if (r->resp_send_chunked 153 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 154 return -1; 155 156 if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) 157 return -1; 158 159 if (r->resp_send_chunked 160 && 0 != 161 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 162 return -1; 163 164 return 0; 165 } 166 167 static int http_chunk_append_cq_to_tempfile(request_st * const r, chunkqueue * const src, const size_t len) { 168 chunkqueue * const cq = r->write_queue; 169 log_error_st * const errh = r->conf.errh; 170 171 if (r->resp_send_chunked 172 && 0 != http_chunk_len_append_tempfile(cq, len, errh)) 173 return -1; 174 175 if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) 176 return -1; 177 178 if (r->resp_send_chunked 179 && 0 != 180 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) 181 return -1; 182 183 return 0; 184 } 185 186 __attribute_pure__ 187 static int http_chunk_uses_tempfile(const request_st * const r, const chunkqueue * const cq, const size_t len) { 188 189 /* current usage does not append_mem or append_buffer after appending 190 * file, so not checking if users of this interface have appended large 191 * (references to) files to chunkqueue, which would not be in memory 192 * (but included in calculation for whether or not to use temp file) */ 193 194 /*(allow slightly larger mem use if FDEVENT_STREAM_RESPONSE_BUFMIN 195 * to reduce creation of temp files when backend producer will be 196 * blocked until more data is sent to network to client)*/ 197 198 const chunk * const c = cq->last; 199 return 200 ((c && c->type == FILE_CHUNK && c->file.is_temp) 201 || cq->bytes_in - cq->bytes_out + len 202 > ((r->conf.stream_response_body & FDEVENT_STREAM_RESPONSE_BUFMIN) 203 ? 128*1024 204 : 64*1024)); 205 } 206 207 int http_chunk_append_buffer(request_st * const r, buffer * const mem) { 208 size_t len = buffer_string_length(mem); 209 if (0 == len) return 0; 210 211 chunkqueue * const cq = r->write_queue; 212 213 if (http_chunk_uses_tempfile(r, cq, len)) 214 return http_chunk_append_to_tempfile(r, mem->ptr, len); 215 216 if (r->resp_send_chunked) 217 http_chunk_len_append(cq, len); 218 219 /*(chunkqueue_append_buffer() might steal buffer contents)*/ 220 chunkqueue_append_buffer(cq, mem); 221 222 if (r->resp_send_chunked) 223 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 224 225 return 0; 226 } 227 228 int http_chunk_append_mem(request_st * const r, const char * const mem, const size_t len) { 229 if (0 == len) return 0; 230 force_assert(NULL != mem); 231 232 chunkqueue * const cq = r->write_queue; 233 234 if (http_chunk_uses_tempfile(r, cq, len)) 235 return http_chunk_append_to_tempfile(r, mem, len); 236 237 if (r->resp_send_chunked) 238 http_chunk_len_append(cq, len); 239 240 chunkqueue_append_mem(cq, mem, len); 241 242 if (r->resp_send_chunked) 243 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 244 245 return 0; 246 } 247 248 int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, const size_t len) { 249 if (0 == len) return 0; 250 251 chunkqueue * const cq = r->write_queue; 252 253 if (http_chunk_uses_tempfile(r, cq, len)) 254 return http_chunk_append_cq_to_tempfile(r, src, len); 255 256 if (r->resp_send_chunked) 257 http_chunk_len_append(cq, len); 258 259 chunkqueue_steal(cq, src, len); 260 261 if (r->resp_send_chunked) 262 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); 263 264 return 0; 265 } 266 267 void http_chunk_close(request_st * const r) { 268 if (!r->resp_send_chunked) return; 269 270 if (r->gw_dechunk && !buffer_string_is_empty(&r->gw_dechunk->b)) { 271 /* XXX: trailers passed through; no sanity check currently done */ 272 chunkqueue_append_buffer(r->write_queue, &r->gw_dechunk->b); 273 if (!r->gw_dechunk->done) 274 r->keep_alive = 0; 275 } 276 else 277 chunkqueue_append_mem(r->write_queue, CONST_STR_LEN("0\r\n\r\n")); 278 } 279 280 static int 281 http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len) 282 { 283 /*(silently discard data, if any, after final \r\n)*/ 284 if (r->gw_dechunk->done) return 0; 285 286 buffer * const h = &r->gw_dechunk->b; 287 off_t te_chunked = r->gw_dechunk->gw_chunked; 288 while (len) { 289 if (0 == te_chunked) { 290 const char *p = strchr(mem, '\n'); 291 /*(likely better ways to handle chunked header crossing chunkqueue 292 * chunks, but this situation is not expected to occur frequently)*/ 293 if (NULL == p) { /* incomplete HTTP chunked header line */ 294 uint32_t hlen = buffer_string_length(h); 295 if ((off_t)(1024 - hlen) < len) { 296 log_error(r->conf.errh, __FILE__, __LINE__, 297 "chunked header line too long"); 298 return -1; 299 } 300 buffer_append_string_len(h, mem, len); 301 break; 302 } 303 304 off_t hsz = ++p - mem; 305 unsigned char *s = (unsigned char *)mem; 306 if (!buffer_string_is_empty(h)) { 307 uint32_t hlen = buffer_string_length(h); 308 if (NULL == memchr(h->ptr, '\n', hlen)) { 309 if ((off_t)(1024 - hlen) < hsz) { 310 log_error(r->conf.errh, __FILE__, __LINE__, 311 "chunked header line too long"); 312 return -1; 313 } 314 buffer_append_string_len(h, mem, hsz); 315 } 316 s = (unsigned char *)h->ptr; 317 } 318 for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) { 319 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1) { 320 log_error(r->conf.errh, __FILE__, __LINE__, 321 "chunked data size too large"); 322 return -1; 323 } 324 te_chunked <<= 4; 325 te_chunked |= u; 326 } 327 if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/ 328 while (*s == ' ' || *s == '\t') ++s; 329 if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/ 330 log_error(r->conf.errh, __FILE__, __LINE__, 331 "chunked header invalid chars"); 332 return -1; 333 } 334 335 if (0 == te_chunked) { 336 /* do not consume final chunked header until 337 * (optional) trailers received along with 338 * request-ending blank line "\r\n" */ 339 if (len - hsz == 2 && p[0] == '\r' && p[1] == '\n') { 340 /* common case with no trailers; final \r\n received */ 341 /*(silently discard data, if any, after final \r\n)*/ 342 #if 0 /*(avoid allocation for common case; users must check)*/ 343 if (buffer_is_empty(h)) 344 buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n")); 345 #else 346 buffer_clear(h); 347 #endif 348 r->gw_dechunk->done = r->http_status; 349 break; 350 } 351 352 /* accumulate trailers and check for end of trailers */ 353 /* XXX: reuse r->conf.max_request_field_size 354 * or have separate limit? */ 355 uint32_t hlen = buffer_string_length(h); 356 if ((off_t)(r->conf.max_request_field_size - hlen) < hsz) { 357 /* truncate excessively long trailers */ 358 r->gw_dechunk->done = r->http_status; 359 hsz = (off_t)(r->conf.max_request_field_size - hlen); 360 buffer_append_string_len(h, mem, hsz); 361 p = strrchr(h->ptr, '\n'); 362 if (NULL != p) 363 buffer_string_set_length(h, p + 1 - h->ptr); 364 else { /*(should not happen)*/ 365 buffer_clear(h); 366 buffer_append_string_len(h, CONST_STR_LEN("0\r\n")); 367 } 368 buffer_append_string_len(h, CONST_STR_LEN("\r\n")); 369 break; 370 } 371 buffer_append_string_len(h, mem, hsz); 372 hlen += (uint32_t)hsz; /* uint32_t fits in (buffer *) */ 373 if (hlen < 4) break; 374 p = h->ptr + hlen - 4; 375 if (p[0]=='\r'&&p[1]=='\n'&&p[2]=='\r'&&p[3]=='\n') 376 r->gw_dechunk->done = r->http_status; 377 else if ((p = strstr(h->ptr, "\r\n\r\n"))) { 378 r->gw_dechunk->done = r->http_status; 379 /*(silently discard data, if any, after final \r\n)*/ 380 buffer_string_set_length(h, (uint32_t)(p+4-h->ptr)); 381 } 382 break; 383 } 384 385 mem += hsz; 386 len -= hsz; 387 388 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) { 389 log_error(r->conf.errh, __FILE__, __LINE__, 390 "chunked data size too large"); 391 return -1; 392 } 393 te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/ 394 } 395 396 if (te_chunked >= 2) { 397 off_t clen = te_chunked - 2; 398 if (clen > len) clen = len; 399 if (0 != http_chunk_append_mem(r, mem, clen)) 400 return -1; 401 mem += clen; 402 len -= clen; 403 te_chunked -= clen; 404 if (te_chunked == 2) { 405 if (len >= 2) { 406 if (mem[0] != '\r' || mem[1] != '\n') return -1; 407 mem += 2; 408 len -= 2; 409 te_chunked = 0; 410 } 411 else if (len == 1 && mem[0] != '\r') return -1; 412 } 413 } 414 else if (1 == te_chunked) { 415 /* finish reading chunk block "\r\n" */ 416 if (mem[0] != '\n') return -1; 417 ++mem; 418 --len; 419 te_chunked = 0; 420 } 421 } 422 r->gw_dechunk->gw_chunked = te_chunked; 423 return 0; 424 } 425 426 int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem) 427 { 428 /*(called by funcs receiving data from backends, which might be chunked)*/ 429 /*(separate from http_chunk_append_buffer() called by numerous others)*/ 430 if (!r->resp_decode_chunked) 431 return http_chunk_append_buffer(r, mem); 432 433 /* no need to decode chunked to immediately re-encode chunked, 434 * though would be more robust to still validate chunk lengths sent 435 * (or else we might wait for keep-alive while client waits for final chunk) 436 * Before finishing response/stream, we *are not* checking if we got final 437 * chunk of chunked encoding from backend. If we were, we could consider 438 * closing HTTP/1.0 and HTTP/1.1 connections (no keep-alive), and in HTTP/2 439 * we could consider sending RST_STREAM error. http_chunk_close() would 440 * only handle case of streaming chunked to client */ 441 if (r->resp_send_chunked) { 442 r->resp_send_chunked = 0; 443 int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */ 444 r->resp_send_chunked = 1; 445 return rc; 446 } 447 448 /* might avoid copy by transferring buffer if buffer is all data that is 449 * part of large chunked block, but choosing to *not* expand that out here*/ 450 return http_chunk_decode_append_data(r, CONST_BUF_LEN(mem)); 451 } 452 453 int http_chunk_decode_append_mem(request_st * const r, const char * const mem, const size_t len) 454 { 455 /*(called by funcs receiving data from backends, which might be chunked)*/ 456 /*(separate from http_chunk_append_mem() called by numerous others)*/ 457 if (!r->resp_decode_chunked) 458 return http_chunk_append_mem(r, mem, len); 459 460 /* no need to decode chunked to immediately re-encode chunked, 461 * though would be more robust to still validate chunk lengths sent 462 * (or else we might wait for keep-alive while client waits for final chunk) 463 * Before finishing response/stream, we *are not* checking if we got final 464 * chunk of chunked encoding from backend. If we were, we could consider 465 * closing HTTP/1.0 and HTTP/1.1 connections (no keep-alive), and in HTTP/2 466 * we could consider sending RST_STREAM error. http_chunk_close() would 467 * only handle case of streaming chunked to client */ 468 if (r->resp_send_chunked) { 469 r->resp_send_chunked = 0; 470 int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/ 471 r->resp_send_chunked = 1; 472 return rc; 473 } 474 475 return http_chunk_decode_append_data(r, mem, (off_t)len); 476 } 477