1 /*
2 * http_chunk - append response to chunkqueue, possibly in "chunked" encoding
3 *
4 * Fully-rewritten from original
5 * Copyright(c) 2019 Glenn Strauss gstrauss()gluelogic.com All rights reserved
6 * License: BSD 3-clause (same as lighttpd)
7 */
8 #include "first.h"
9
10 #include "http_chunk.h"
11 #include "chunk.h"
12 #include "stat_cache.h"
13 #include "fdevent.h"
14 #include "log.h"
15 #include "request.h"
16
17 #include <sys/types.h>
18 #include <sys/stat.h>
19
20 #include <stdlib.h>
21 #include <unistd.h>
22
23 #include <errno.h>
24 #include <string.h>
25
26 __attribute_noinline__
http_chunk_len_append(chunkqueue * const cq,uintmax_t len)27 static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) {
28 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */
29 #if 0
30 buffer b = { buf, 0, sizeof(buf) };
31 buffer_append_uint_hex(&b, len);
32 buffer_append_string_len(&b, CONST_STR_LEN("\r\n"));
33 chunkqueue_append_mem(cq, b.ptr, b.used-1);
34 #else
35 int i = (int)(sizeof(buf));
36 buf[--i] = '\n';
37 buf[--i] = '\r';
38 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4);
39 chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i);
40 #endif
41 }
42
43 __attribute_noinline__
http_chunk_len_append_tempfile(chunkqueue * const cq,uintmax_t len,log_error_st * const errh)44 static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) {
45 char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */
46 #if 0
47 buffer b = { buf, 0, sizeof(buf) };
48 buffer_append_uint_hex(&b, len);
49 buffer_append_string_len(&b, CONST_STR_LEN("\r\n"));
50 return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh);
51 #else
52 int i = (int)(sizeof(buf));
53 buf[--i] = '\n';
54 buf[--i] = '\r';
55 do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4);
56 return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh);
57 #endif
58 }
59
60 __attribute_noinline__
http_chunk_append_read_fd_range(request_st * const r,const buffer * const fn,const int fd,off_t offset,off_t len)61 static int http_chunk_append_read_fd_range(request_st * const r, const buffer * const fn, const int fd, off_t offset, off_t len) {
62 /* note: this routine should not be used for range requests
63 * unless the total size of ranges requested is small */
64 /* note: future: could read into existing MEM_CHUNK in cq->last if
65 * there is sufficient space, but would need to adjust for existing
66 * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */
67 UNUSED(fn);
68
69 chunkqueue * const cq = &r->write_queue;
70
71 if (r->resp_send_chunked)
72 http_chunk_len_append(cq, (uintmax_t)len);
73
74 buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2+1);
75 ssize_t rd;
76 const off_t foff = offset;
77 offset = 0;
78 do {
79 rd = chunk_file_pread(fd, b->ptr+offset, (size_t)len, foff+offset);
80 } while (rd > 0 && (offset += rd, len -= rd));
81 buffer_commit(b, offset);
82
83 if (r->resp_send_chunked)
84 buffer_append_string_len(b, CONST_STR_LEN("\r\n"));
85
86 chunkqueue_append_buffer_commit(cq);
87 return (len == 0) ? 0 : -1;
88 }
89
90 __attribute_noinline__
http_chunk_append_file_ref_range(request_st * const r,stat_cache_entry * const sce,const off_t offset,off_t len)91 void http_chunk_append_file_ref_range(request_st * const r, stat_cache_entry * const sce, const off_t offset, off_t len) {
92 chunkqueue * const cq = &r->write_queue;
93
94 if (sce->st.st_size - offset < len)
95 len = sce->st.st_size - offset;
96 if (len <= 0)
97 return;
98
99 if (r->resp_send_chunked)
100 http_chunk_len_append(cq, (uintmax_t)len);
101
102 const buffer * const fn = &sce->name;
103 const int fd = sce->fd;
104 chunkqueue_append_file_fd(cq, fn, fd, offset, len);
105 if (fd >= 0) {
106 chunk * const d = cq->last;
107 d->file.ref = sce;
108 d->file.refchg = stat_cache_entry_refchg;
109 stat_cache_entry_refchg(sce, 1);
110 }
111
112 if (r->resp_send_chunked)
113 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
114 }
115
116 __attribute_noinline__
http_chunk_append_file_fd_range(request_st * const r,const buffer * const fn,const int fd,const off_t offset,const off_t len)117 void http_chunk_append_file_fd_range(request_st * const r, const buffer * const fn, const int fd, const off_t offset, const off_t len) {
118 chunkqueue * const cq = &r->write_queue;
119
120 if (r->resp_send_chunked)
121 http_chunk_len_append(cq, (uintmax_t)len);
122
123 chunkqueue_append_file_fd(cq, fn, fd, offset, len);
124
125 if (r->resp_send_chunked)
126 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
127 }
128
http_chunk_append_file_fd(request_st * const r,const buffer * const fn,const int fd,const off_t sz)129 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) {
130 if (sz > 32768 || !r->resp_send_chunked) {
131 http_chunk_append_file_fd_range(r, fn, fd, 0, sz);
132 return 0;
133 }
134
135 /*(read small files into memory)*/
136 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0;
137 close(fd);
138 return rc;
139 }
140
http_chunk_append_file_ref(request_st * const r,stat_cache_entry * const sce)141 int http_chunk_append_file_ref(request_st * const r, stat_cache_entry * const sce) {
142 const off_t sz = sce->st.st_size;
143 if (sz > 32768 || !r->resp_send_chunked) {
144 http_chunk_append_file_ref_range(r, sce, 0, sz);
145 return 0;
146 }
147
148 /*(read small files into memory)*/
149 const buffer * const fn = &sce->name;
150 const int fd = sce->fd;
151 int rc = (0 != sz) ? http_chunk_append_read_fd_range(r,fn,fd,0,sz) : 0;
152 return rc;
153 }
154
155 __attribute_noinline__
http_chunk_append_to_tempfile(request_st * const r,const char * const mem,const size_t len)156 static int http_chunk_append_to_tempfile(request_st * const r, const char * const mem, const size_t len) {
157 chunkqueue * const cq = &r->write_queue;
158 log_error_st * const errh = r->conf.errh;
159
160 if (r->resp_send_chunked
161 && 0 != http_chunk_len_append_tempfile(cq, len, errh))
162 return -1;
163
164 if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh))
165 return -1;
166
167 if (r->resp_send_chunked
168 && 0 !=
169 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh))
170 return -1;
171
172 return 0;
173 }
174
175 __attribute_noinline__
http_chunk_append_cq_to_tempfile(request_st * const r,chunkqueue * const src,const size_t len)176 static int http_chunk_append_cq_to_tempfile(request_st * const r, chunkqueue * const src, const size_t len) {
177 chunkqueue * const cq = &r->write_queue;
178 log_error_st * const errh = r->conf.errh;
179
180 if (r->resp_send_chunked
181 && 0 != http_chunk_len_append_tempfile(cq, len, errh))
182 return -1;
183
184 if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh))
185 return -1;
186
187 if (r->resp_send_chunked
188 && 0 !=
189 chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh))
190 return -1;
191
192 return 0;
193 }
194
195 /*(inlined by compiler optimizer)*/
196 __attribute_pure__
http_chunk_uses_tempfile(const chunkqueue * const cq,const size_t len)197 static int http_chunk_uses_tempfile(const chunkqueue * const cq, const size_t len) {
198
199 /* current usage does not append_mem or append_buffer after appending
200 * file, so not checking if users of this interface have appended large
201 * (references to) files to chunkqueue, which would not be in memory
202 * (but included in calculation for whether or not to use temp file) */
203 const chunk * const c = cq->last;
204 return
205 ((c && c->type == FILE_CHUNK && c->file.is_temp)
206 || chunkqueue_length(cq) + len > 65536);
207 }
208
209 __attribute_noinline__
http_chunk_append_buffer(request_st * const r,buffer * const mem)210 int http_chunk_append_buffer(request_st * const r, buffer * const mem) {
211 size_t len = mem ? buffer_clen(mem) : 0;
212 if (0 == len) return 0;
213
214 chunkqueue * const cq = &r->write_queue;
215
216 if (http_chunk_uses_tempfile(cq, len)) {
217 int rc = http_chunk_append_to_tempfile(r, mem->ptr, len);
218 buffer_clear(mem);
219 return rc;
220 }
221
222 if (r->resp_send_chunked)
223 http_chunk_len_append(cq, len);
224
225 /*(chunkqueue_append_buffer() might steal buffer contents)*/
226 chunkqueue_append_buffer(cq, mem);
227
228 if (r->resp_send_chunked)
229 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
230
231 return 0;
232 }
233
234 __attribute_noinline__
http_chunk_append_mem(request_st * const r,const char * const mem,const size_t len)235 int http_chunk_append_mem(request_st * const r, const char * const mem, const size_t len) {
236 if (0 == len) return 0;
237 force_assert(NULL != mem);
238
239 chunkqueue * const cq = &r->write_queue;
240
241 if (http_chunk_uses_tempfile(cq, len))
242 return http_chunk_append_to_tempfile(r, mem, len);
243
244 if (r->resp_send_chunked)
245 http_chunk_len_append(cq, len);
246
247 chunkqueue_append_mem(cq, mem, len);
248
249 if (r->resp_send_chunked)
250 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
251
252 return 0;
253 }
254
http_chunk_transfer_cqlen(request_st * const r,chunkqueue * const src,const size_t len)255 int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, const size_t len) {
256 if (0 == len) return 0;
257
258 chunkqueue * const cq = &r->write_queue;
259
260 if (http_chunk_uses_tempfile(cq, len))
261 return http_chunk_append_cq_to_tempfile(r, src, len);
262
263 if (r->resp_send_chunked)
264 http_chunk_len_append(cq, len);
265
266 chunkqueue_steal(cq, src, len);
267
268 if (r->resp_send_chunked)
269 chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
270
271 return 0;
272 }
273
http_chunk_close(request_st * const r)274 void http_chunk_close(request_st * const r) {
275 if (!r->resp_send_chunked) return;
276
277 if (r->gw_dechunk) {
278 if (!r->gw_dechunk->done)
279 r->keep_alive = 0;
280 }
281 else
282 chunkqueue_append_mem(&r->write_queue, CONST_STR_LEN("0\r\n\r\n"));
283 }
284
285 static int
http_chunk_decode_append_data(request_st * const r,const char * mem,off_t len)286 http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len)
287 {
288 if (r->gw_dechunk->done) return -1; /*(excess data)*/
289
290 buffer * const h = &r->gw_dechunk->b;
291 off_t te_chunked = r->gw_dechunk->gw_chunked;
292 while (len) {
293 if (0 == te_chunked) {
294 const char *p;
295 unsigned char *s = (unsigned char *)mem;
296 off_t hsz;
297 if (buffer_is_blank(h)) {
298 /*(short-circuit common case: complete chunked header line)*/
299 p = memchr(mem, '\n', (size_t)len);
300 if (p)
301 hsz = (off_t)(++p - mem);
302 else {
303 if (len >= 1024) {
304 log_error(r->conf.errh, __FILE__, __LINE__,
305 "chunked header line too long");
306 return -1;
307 }
308 buffer_append_string_len(h, mem, (uint32_t)len);
309 break; /* incomplete HTTP chunked header line */
310 }
311 }
312 else {
313 uint32_t hlen = buffer_clen(h);
314 p = strchr(h->ptr, '\n');
315 if (p)
316 hsz = (off_t)(++p - h->ptr);
317 else {
318 p = memchr(mem, '\n', (size_t)len);
319 hsz = (p ? (off_t)(++p - mem) : len);
320 if ((off_t)(1024 - hlen) < hsz) {
321 log_error(r->conf.errh, __FILE__, __LINE__,
322 "chunked header line too long");
323 return -1;
324 }
325 buffer_append_string_len(h, mem, hsz);
326 if (NULL == p) break;/*incomplete HTTP chunked header line*/
327 mem += hsz;
328 len -= hsz;
329 hsz = 0;
330 }
331 s = (unsigned char *)h->ptr;/*(note: read h->ptr after append)*/
332 }
333
334 for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) {
335 if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) {
336 log_error(r->conf.errh, __FILE__, __LINE__,
337 "chunked data size too large");
338 return -1;
339 }
340 te_chunked <<= 4;
341 te_chunked |= u;
342 }
343 if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/
344 while (*s == ' ' || *s == '\t') ++s;
345 if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/
346 log_error(r->conf.errh, __FILE__, __LINE__,
347 "chunked header invalid chars");
348 return -1;
349 }
350
351 if (0 == te_chunked) {
352 /* do not consume final chunked header until
353 * (optional) trailers received along with
354 * request-ending blank line "\r\n" */
355 if (len - hsz >= 2 && p[0] == '\r' && p[1] == '\n') {
356 if (len - hsz > 2) return -1; /*(excess data)*/
357 /* common case with no trailers; final \r\n received */
358 #if 0 /*(avoid allocation for common case; users must check)*/
359 if (buffer_is_unset(h))
360 buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n"));
361 #else
362 buffer_clear(h);
363 #endif
364 r->gw_dechunk->done = r->http_status;
365 break;
366 }
367
368 /* accumulate trailers and check for end of trailers */
369 /* XXX: reuse r->conf.max_request_field_size
370 * or have separate limit? */
371 uint32_t mlen = buffer_clen(h);
372 mlen = (r->conf.max_request_field_size > mlen)
373 ? r->conf.max_request_field_size - mlen
374 : 0;
375 if ((off_t)mlen < len) {
376 /* truncate excessively long trailers */
377 /* (not truncated; passed as-is if r->resp_send_chunked) */
378 if (r->resp_send_chunked) r->keep_alive = 0;
379 r->gw_dechunk->done = r->http_status;
380 buffer_append_string_len(h, mem, mlen);
381 p = strrchr(h->ptr, '\n');
382 if (NULL != p) {
383 buffer_truncate(h, p + 1 - h->ptr);
384 if (p[-1] != '\r')
385 buffer_append_string_len(h, CONST_STR_LEN("\r\n"));
386 }
387 else { /*(should not happen)*/
388 buffer_clear(h);
389 buffer_append_string_len(h, CONST_STR_LEN("0\r\n"));
390 }
391 buffer_append_string_len(h, CONST_STR_LEN("\r\n"));
392 break;
393 }
394 buffer_append_string_len(h, mem, (uint32_t)len);
395 if ((p = strstr(h->ptr, "\r\n\r\n"))) {
396 r->gw_dechunk->done = r->http_status;
397 if (p[4] != '\0') return -1; /*(excess data)*/
398 /*buffer_truncate(h, (uint32_t)(p+4-h->ptr));*/
399 }
400 break;
401 }
402
403 mem += hsz;
404 len -= hsz;
405
406 te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/
407 buffer_clear(h);
408 if (0 == len) break;
409 }
410
411 if (te_chunked >= 2) {
412 off_t clen = te_chunked - 2;
413 if (clen > len) clen = len;
414 if (!r->resp_send_chunked
415 && 0 != http_chunk_append_mem(r, mem, clen))
416 return -1;
417 mem += clen;
418 len -= clen;
419 te_chunked -= clen;
420 if (te_chunked == 2) {
421 if (len >= 2) {
422 if (mem[0] != '\r' || mem[1] != '\n') return -1;
423 mem += 2;
424 len -= 2;
425 te_chunked = 0;
426 }
427 else if (len == 1) {
428 if (mem[0] != '\r') return -1;
429 /*++mem;*/
430 /*--len;*/
431 te_chunked = 1;
432 break;
433 }
434 }
435 }
436 else if (1 == te_chunked) {
437 /* finish reading chunk block "\r\n" */
438 if (mem[0] != '\n') return -1;
439 ++mem;
440 --len;
441 te_chunked = 0;
442 }
443 }
444 if (r->gw_dechunk->done)
445 r->resp_body_finished = 1;
446 r->gw_dechunk->gw_chunked = te_chunked;
447 return 0;
448 }
449
http_chunk_decode_append_buffer(request_st * const r,buffer * const mem)450 int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem)
451 {
452 /* Note: this routine is separate from http_chunk_decode_append_mem() to
453 * potentially avoid copying in http_chunk_append_buffer(). Otherwise this
454 * would be: return http_chunk_decode_append_mem(r, BUF_PTR_LEN(mem)); */
455
456 /*(called by funcs receiving chunked data from backends)*/
457 /*(separate from http_chunk_append_buffer() called by numerous others)*/
458
459 /* might avoid copy by transferring buffer if buffer is all data that is
460 * part of large chunked block, but choosing to *not* expand that out here*/
461 if (0 != http_chunk_decode_append_data(r, BUF_PTR_LEN(mem)))
462 return -1;
463
464 /* no need to decode chunked to immediately re-encode chunked;
465 * pass through chunked encoding as provided by backend,
466 * though it is still parsed (above) to maintain state.
467 * XXX: consider having callers use chunk buffers for hctx->b
468 * for more efficient data copy avoidance and buffer reuse
469 * note: r->resp_send_chunked = 0 until response headers sent,
470 * which is when Transfer-Encoding: chunked might be chosen */
471 if (r->resp_send_chunked) {
472 r->resp_send_chunked = 0;
473 int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */
474 r->resp_send_chunked = 1;
475 return rc;
476 }
477 else
478 buffer_clear(mem);
479
480 return 0;
481 }
482
http_chunk_decode_append_mem(request_st * const r,const char * const mem,size_t len)483 int http_chunk_decode_append_mem(request_st * const r, const char * const mem, size_t len)
484 {
485 /*(called by funcs receiving chunked data from backends)*/
486 /*(separate from http_chunk_append_mem() called by numerous others)*/
487
488 if (0 != http_chunk_decode_append_data(r, mem, (off_t)len))
489 return -1;
490
491 /* no need to decode chunked to immediately re-encode chunked;
492 * pass through chunked encoding as provided by backend,
493 * though it is still parsed (above) to maintain state.
494 * note: r->resp_send_chunked = 0 until response headers sent,
495 * which is when Transfer-Encoding: chunked might be chosen */
496 if (r->resp_send_chunked) {
497 r->resp_send_chunked = 0;
498 int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/
499 r->resp_send_chunked = 1;
500 return rc;
501 }
502
503 return 0;
504 }
505