1 /* mod_deflate 2 * 3 * 4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch 5 * 6 * Bug fix and new features: 7 * 1) fix loop bug when content-length is bigger than work-block-size*k 8 * 9 * ------- 10 * 11 * lighttpd-1.4.26.mod_deflate.patch from 12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate 13 * 14 * ------- 15 * 16 * Patch further modified in this incarnation. 17 * 18 * Note: this patch only handles completed responses 19 * (r->resp_body_finished) 20 * this patch does not currently handle streaming dynamic responses, 21 * and therefore also does not worry about Transfer-Encoding: chunked 22 * (or having separate con->output_queue for chunked-encoded output) 23 * (or using separate buffers per connection instead of p->tmp_buf) 24 * (or handling interactions with block buffering and write timeouts) 25 * 26 * Bug fix: 27 * - fixed major bug with compressing chunks with offset > 0 28 * x-ref: 29 * "Response breaking in mod_deflate" 30 * https://redmine.lighttpd.net/issues/986 31 * - fix broken (in some cases) chunk accounting in deflate_compress_response() 32 * - fix broken bzip2 33 * x-ref: 34 * "mod_deflate's bzip2 broken by default" 35 * https://redmine.lighttpd.net/issues/2035 36 * - fix mismatch with current chunk interfaces 37 * x-ref: 38 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)" 39 * https://redmine.lighttpd.net/issues/1510 40 * 41 * Behavior changes from prior patch: 42 * - deflate.mimetypes must now be configured to enable compression 43 * deflate.mimetypes = ( ) # compress nothing (disabled; default) 44 * deflate.mimetypes = ( "" ) # compress all mimetypes 45 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes 46 * x-ref: 47 * "mod_deflate enabled by default" 48 * https://redmine.lighttpd.net/issues/1394 49 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes) 50 * - deflate.debug removed (was developer debug trace, not end-user debug) 51 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress) 52 * x-ref: 53 * "mod_deflate should allow limiting of compression algorithm from the configuration file" 54 * https://redmine.lighttpd.net/issues/996 55 * "mod_compress disabling methods" 56 * https://redmine.lighttpd.net/issues/1773 57 * - deflate.nocompress-url removed since disabling compression for a URL 58 * can now easily be done by setting to a blank list either directive 59 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional 60 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) } 61 * - deflate.sync-flush removed; controlled by r->conf.stream_response_body 62 * (though streaming compression not currently implemented in mod_deflate) 63 * - inactive directives in this patch 64 * (since r->resp_body_finished required) 65 * deflate.work-block-size 66 * deflate.output-buffer-size 67 * - remove weak file size check; SIGBUS is trapped, file that shrink will error 68 * x-ref: 69 * "mod_deflate: filesize check is too weak" 70 * https://redmine.lighttpd.net/issues/1512 71 * - change default deflate.min-compress-size from 0 to now be 256 72 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits 73 * Apache 2.4 mod_deflate minimum is 68 bytes 74 * Akamai recommends minimum 860 bytes 75 * Google recommends minimum be somewhere in range between 150 and 1024 bytes 76 * - deflate.max-compress-size new directive (in kb like compress.max_filesize) 77 * - deflate.mem-level removed (too many knobs for little benefit) 78 * - deflate.window-size removed (too many knobs for little benefit) 79 * 80 * Future: 81 * - config directives may be changed, renamed, or removed 82 * e.g. A set of reasonable defaults might be chosen 83 * instead of making them configurable. 84 * deflate.min-compress-size 85 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes 86 * to avoid compressing, even if a broader deflate.mimetypes matched, 87 * e.g. to compress all "text/" except "text/special". 88 * 89 * Implementation notes: 90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer() 91 * so that p->tmp_buf can be large and re-used. This results in an extra copy 92 * of compressed data before data is sent to network, though if the compressed 93 * size is larger than 64k, it ends up being sent to a temporary file on 94 * disk without suffering an extra copy in memory, and without extra chunk 95 * create and destroy. If this is ever changed to give away buffers, then use 96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across 97 * multiple requests being handled in parallel. 98 */ 99 #include "first.h" 100 101 #include <sys/types.h> 102 #include <sys/stat.h> 103 #include "sys-mmap.h" 104 105 #include <fcntl.h> 106 #include <stdlib.h> 107 #include <string.h> 108 #include <errno.h> 109 #include <time.h> 110 #include <unistd.h> /* getpid() read() unlink() write() */ 111 112 #include "base.h" 113 #include "fdevent.h" 114 #include "log.h" 115 #include "buffer.h" 116 #include "etag.h" 117 #include "http_chunk.h" 118 #include "http_header.h" 119 #include "response.h" 120 #include "stat_cache.h" 121 122 #include "plugin.h" 123 124 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ 125 # define USE_ZLIB 126 # include <zlib.h> 127 #endif 128 #ifndef Z_DEFAULT_COMPRESSION 129 #define Z_DEFAULT_COMPRESSION -1 130 #endif 131 #ifndef MAX_WBITS 132 #define MAX_WBITS 15 133 #endif 134 135 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 136 # define USE_BZ2LIB 137 /* we don't need stdio interface */ 138 # define BZ_NO_STDIO 139 # include <bzlib.h> 140 #endif 141 142 #if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI 143 # define USE_BROTLI 144 # include <brotli/encode.h> 145 #endif 146 147 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP 148 #define USE_MMAP 149 150 #include "sys-mmap.h" 151 #include <setjmp.h> 152 #include <signal.h> 153 154 static volatile int sigbus_jmp_valid; 155 static sigjmp_buf sigbus_jmp; 156 157 static void sigbus_handler(int sig) { 158 UNUSED(sig); 159 if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1); 160 log_failed_assert(__FILE__, __LINE__, "SIGBUS"); 161 } 162 #endif 163 164 /* request: accept-encoding */ 165 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) 166 #define HTTP_ACCEPT_ENCODING_GZIP BV(1) 167 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) 168 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) 169 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) 170 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) 171 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) 172 #define HTTP_ACCEPT_ENCODING_BR BV(7) 173 174 #define KByte * 1024 175 #define MByte * 1024 KByte 176 #define GByte * 1024 MByte 177 178 typedef struct { 179 const array *mimetypes; 180 const buffer *cache_dir; 181 unsigned int max_compress_size; 182 unsigned short min_compress_size; 183 unsigned short output_buffer_size; 184 unsigned short work_block_size; 185 unsigned short sync_flush; 186 short compression_level; 187 short allowed_encodings; 188 double max_loadavg; 189 } plugin_config; 190 191 typedef struct { 192 PLUGIN_DATA; 193 plugin_config defaults; 194 plugin_config conf; 195 196 buffer tmp_buf; 197 } plugin_data; 198 199 typedef struct { 200 union { 201 #ifdef USE_ZLIB 202 z_stream z; 203 #endif 204 #ifdef USE_BZ2LIB 205 bz_stream bz; 206 #endif 207 #ifdef USE_BROTLI 208 BrotliEncoderState *br; 209 #endif 210 int dummy; 211 } u; 212 off_t bytes_in; 213 off_t bytes_out; 214 chunkqueue *in_queue; 215 buffer *output; 216 plugin_data *plugin_data; 217 request_st *r; 218 int compression_type; 219 int cache_fd; 220 char *cache_fn; 221 } handler_ctx; 222 223 static handler_ctx *handler_ctx_init() { 224 handler_ctx *hctx; 225 226 hctx = calloc(1, sizeof(*hctx)); 227 hctx->in_queue = chunkqueue_init(); 228 hctx->cache_fd = -1; 229 230 return hctx; 231 } 232 233 static void handler_ctx_free(handler_ctx *hctx) { 234 if (hctx->cache_fn) { 235 unlink(hctx->cache_fn); 236 free(hctx->cache_fn); 237 } 238 if (-1 != hctx->cache_fd) 239 close(hctx->cache_fd); 240 #if 0 241 if (hctx->output != &p->tmp_buf) { 242 buffer_free(hctx->output); 243 } 244 #endif 245 chunkqueue_free(hctx->in_queue); 246 free(hctx); 247 } 248 249 INIT_FUNC(mod_deflate_init) { 250 plugin_data * const p = calloc(1, sizeof(plugin_data)); 251 buffer_string_prepare_copy(&p->tmp_buf, 64 KByte); 252 return p; 253 } 254 255 FREE_FUNC(mod_deflate_free) { 256 plugin_data *p = p_d; 257 free(p->tmp_buf.ptr); 258 } 259 260 #if defined(_WIN32) && !defined(__CYGWIN__) 261 #define mkdir(x,y) mkdir(x) 262 #endif 263 264 static int mkdir_for_file (char *fn) { 265 for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) { 266 if (p[1] == '\0') return 0; /* ignore trailing slash */ 267 *p = '\0'; 268 int rc = mkdir(fn, 0700); 269 *p = '/'; 270 if (0 != rc && errno != EEXIST) return -1; 271 } 272 return 0; 273 } 274 275 static int mkdir_recursive (char *dir) { 276 return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST) 277 ? 0 278 : -1; 279 } 280 281 static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) { 282 /* XXX: future: for shorter paths into the cache, we could checksum path, 283 * and then shard it to avoid a huge single directory. 284 * Alternatively, could use &r->uri.path, minus any 285 * (matching) &r->pathinfo suffix, with result url-encoded */ 286 buffer * const tb = r->tmp_buf; 287 buffer_copy_buffer(tb, cache_dir); 288 buffer_append_string_len(tb, CONST_BUF_LEN(&r->physical.path)); 289 buffer_append_string_len(tb, CONST_STR_LEN("-")); 290 buffer_append_string_len(tb, etag->ptr+1, buffer_string_length(etag)-2); 291 return tb; 292 } 293 294 static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) { 295 /* race exists whereby up to # workers might attempt to compress same 296 * file at same time if requested at same time, but this is unlikely 297 * and resolves itself by atomic rename into place when done */ 298 const uint32_t fnlen = buffer_string_length(fn); 299 hctx->cache_fn = malloc(fnlen+1+LI_ITOSTRING_LENGTH+1); 300 force_assert(hctx->cache_fn); 301 memcpy(hctx->cache_fn, fn->ptr, fnlen); 302 hctx->cache_fn[fnlen] = '.'; 303 const size_t ilen = 304 li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid()); 305 hctx->cache_fn[fnlen+1+ilen] = '\0'; 306 hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600); 307 if (-1 == hctx->cache_fd) { 308 free(hctx->cache_fn); 309 hctx->cache_fn = NULL; 310 } 311 } 312 313 static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) { 314 if (0 != fdevent_rename(hctx->cache_fn, fn->ptr)) 315 return -1; 316 free(hctx->cache_fn); 317 hctx->cache_fn = NULL; 318 chunkqueue_reset(r->write_queue); 319 int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out); 320 hctx->cache_fd = -1; 321 return rc; 322 } 323 324 static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) { 325 switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */ 326 case 0: /* deflate.mimetypes */ 327 pconf->mimetypes = cpv->v.a; 328 break; 329 case 1: /* deflate.allowed-encodings */ 330 pconf->allowed_encodings = (short)cpv->v.shrt; 331 break; 332 case 2: /* deflate.max-compress-size */ 333 pconf->max_compress_size = cpv->v.u; 334 break; 335 case 3: /* deflate.min-compress-size */ 336 pconf->min_compress_size = cpv->v.shrt; 337 break; 338 case 4: /* deflate.compression-level */ 339 pconf->compression_level = (short)cpv->v.shrt; 340 break; 341 case 5: /* deflate.output-buffer-size */ 342 pconf->output_buffer_size = cpv->v.shrt; 343 break; 344 case 6: /* deflate.work-block-size */ 345 pconf->work_block_size = cpv->v.shrt; 346 break; 347 case 7: /* deflate.max-loadavg */ 348 pconf->max_loadavg = cpv->v.d; 349 break; 350 case 8: /* deflate.cache-dir */ 351 pconf->cache_dir = cpv->v.b; 352 break; 353 case 9: /* compress.filetype */ 354 pconf->mimetypes = cpv->v.a; 355 break; 356 case 10:/* compress.allowed-encodings */ 357 pconf->allowed_encodings = (short)cpv->v.shrt; 358 break; 359 case 11:/* compress.cache-dir */ 360 pconf->cache_dir = cpv->v.b; 361 break; 362 case 12:/* compress.max-filesize */ 363 pconf->max_compress_size = cpv->v.u; 364 break; 365 case 13:/* compress.max-loadavg */ 366 pconf->max_loadavg = cpv->v.d; 367 break; 368 default:/* should not happen */ 369 return; 370 } 371 } 372 373 static void mod_deflate_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) { 374 do { 375 mod_deflate_merge_config_cpv(pconf, cpv); 376 } while ((++cpv)->k_id != -1); 377 } 378 379 static void mod_deflate_patch_config(request_st * const r, plugin_data * const p) { 380 memcpy(&p->conf, &p->defaults, sizeof(plugin_config)); 381 for (int i = 1, used = p->nconfig; i < used; ++i) { 382 if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id)) 383 mod_deflate_merge_config(&p->conf, p->cvlist + p->cvlist[i].v.u2[0]); 384 } 385 } 386 387 static short mod_deflate_encodings_to_flags(const array *encodings) { 388 short allowed_encodings = 0; 389 if (encodings->used) { 390 for (uint32_t j = 0; j < encodings->used; ++j) { 391 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) 392 data_string *ds = (data_string *)encodings->data[j]; 393 #endif 394 #ifdef USE_ZLIB 395 if (NULL != strstr(ds->value.ptr, "gzip")) 396 allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP 397 | HTTP_ACCEPT_ENCODING_X_GZIP; 398 if (NULL != strstr(ds->value.ptr, "x-gzip")) 399 allowed_encodings |= HTTP_ACCEPT_ENCODING_X_GZIP; 400 if (NULL != strstr(ds->value.ptr, "deflate")) 401 allowed_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE; 402 /* 403 if (NULL != strstr(ds->value.ptr, "compress")) 404 allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS; 405 */ 406 #endif 407 #ifdef USE_BZ2LIB 408 if (NULL != strstr(ds->value.ptr, "bzip2")) 409 allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 410 | HTTP_ACCEPT_ENCODING_X_BZIP2; 411 if (NULL != strstr(ds->value.ptr, "x-bzip2")) 412 allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2; 413 #endif 414 #ifdef USE_BROTLI 415 if (NULL != strstr(ds->value.ptr, "br")) 416 allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; 417 #endif 418 } 419 } 420 else { 421 /* default encodings */ 422 #ifdef USE_ZLIB 423 allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP 424 | HTTP_ACCEPT_ENCODING_X_GZIP 425 | HTTP_ACCEPT_ENCODING_DEFLATE; 426 #endif 427 #ifdef USE_BZ2LIB 428 allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 429 | HTTP_ACCEPT_ENCODING_X_BZIP2; 430 #endif 431 #ifdef USE_BROTLI 432 allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; 433 #endif 434 } 435 return allowed_encodings; 436 } 437 438 SETDEFAULTS_FUNC(mod_deflate_set_defaults) { 439 static const config_plugin_keys_t cpk[] = { 440 { CONST_STR_LEN("deflate.mimetypes"), 441 T_CONFIG_ARRAY_VLIST, 442 T_CONFIG_SCOPE_CONNECTION } 443 ,{ CONST_STR_LEN("deflate.allowed-encodings"), 444 T_CONFIG_ARRAY_VLIST, 445 T_CONFIG_SCOPE_CONNECTION } 446 ,{ CONST_STR_LEN("deflate.max-compress-size"), 447 T_CONFIG_INT, 448 T_CONFIG_SCOPE_CONNECTION } 449 ,{ CONST_STR_LEN("deflate.min-compress-size"), 450 T_CONFIG_SHORT, 451 T_CONFIG_SCOPE_CONNECTION } 452 ,{ CONST_STR_LEN("deflate.compression-level"), 453 T_CONFIG_SHORT, 454 T_CONFIG_SCOPE_CONNECTION } 455 ,{ CONST_STR_LEN("deflate.output-buffer-size"), 456 T_CONFIG_SHORT, 457 T_CONFIG_SCOPE_CONNECTION } 458 ,{ CONST_STR_LEN("deflate.work-block-size"), 459 T_CONFIG_SHORT, 460 T_CONFIG_SCOPE_CONNECTION } 461 ,{ CONST_STR_LEN("deflate.max-loadavg"), 462 T_CONFIG_STRING, 463 T_CONFIG_SCOPE_CONNECTION } 464 ,{ CONST_STR_LEN("deflate.cache-dir"), 465 T_CONFIG_STRING, 466 T_CONFIG_SCOPE_CONNECTION } 467 ,{ CONST_STR_LEN("compress.filetype"), 468 T_CONFIG_ARRAY_VLIST, 469 T_CONFIG_SCOPE_CONNECTION } 470 ,{ CONST_STR_LEN("compress.allowed-encodings"), 471 T_CONFIG_ARRAY_VLIST, 472 T_CONFIG_SCOPE_CONNECTION } 473 ,{ CONST_STR_LEN("compress.cache-dir"), 474 T_CONFIG_STRING, 475 T_CONFIG_SCOPE_CONNECTION } 476 ,{ CONST_STR_LEN("compress.max-filesize"), 477 T_CONFIG_INT, 478 T_CONFIG_SCOPE_CONNECTION } 479 ,{ CONST_STR_LEN("compress.max-loadavg"), 480 T_CONFIG_STRING, 481 T_CONFIG_SCOPE_CONNECTION } 482 ,{ NULL, 0, 483 T_CONFIG_UNSET, 484 T_CONFIG_SCOPE_UNSET } 485 }; 486 487 plugin_data * const p = p_d; 488 if (!config_plugin_values_init(srv, p, cpk, "mod_deflate")) 489 return HANDLER_ERROR; 490 491 /* process and validate config directives 492 * (init i to 0 if global context; to 1 to skip empty global context) */ 493 for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) { 494 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 495 for (; -1 != cpv->k_id; ++cpv) { 496 switch (cpv->k_id) { 497 case 0: /* deflate.mimetypes */ 498 /* mod_deflate matches mimetype as prefix of Content-Type 499 * so ignore '*' at end of mimetype for end-user flexibility 500 * in specifying trailing wildcard to grouping of mimetypes */ 501 for (uint32_t m = 0; m < cpv->v.a->used; ++m) { 502 buffer *mimetype=&((data_string *)cpv->v.a->data[m])->value; 503 size_t len = buffer_string_length(mimetype); 504 if (len > 2 && mimetype->ptr[len-1] == '*') 505 buffer_string_set_length(mimetype, len-1); 506 } 507 if (0 == cpv->v.a->used) cpv->v.a = NULL; 508 break; 509 case 1: /* deflate.allowed-encodings */ 510 cpv->v.shrt = (unsigned short) 511 mod_deflate_encodings_to_flags(cpv->v.a); 512 cpv->vtype = T_CONFIG_SHORT; 513 break; 514 case 2: /* deflate.max-compress-size */ 515 case 3: /* deflate.min-compress-size */ 516 break; 517 case 4: /* deflate.compression-level */ 518 if ((cpv->v.shrt < 1 || cpv->v.shrt > 9) 519 && *(short *)&cpv->v.shrt != -1) { 520 log_error(srv->errh, __FILE__, __LINE__, 521 "compression-level must be between 1 and 9: %hu", 522 cpv->v.shrt); 523 return HANDLER_ERROR; 524 } 525 break; 526 case 5: /* deflate.output-buffer-size */ 527 case 6: /* deflate.work-block-size */ 528 break; 529 case 7: /* deflate.max-loadavg */ 530 cpv->v.d = (!buffer_string_is_empty(cpv->v.b)) 531 ? strtod(cpv->v.b->ptr, NULL) 532 : 0.0; 533 break; 534 case 8: /* deflate.cache-dir */ 535 if (!buffer_string_is_empty(cpv->v.b)) { 536 buffer *b; 537 *(const buffer **)&b = cpv->v.b; 538 const uint32_t len = buffer_string_length(b); 539 if (len > 0 && '/' == b->ptr[len-1]) 540 buffer_string_set_length(b, len-1); /*remove end slash*/ 541 struct stat st; 542 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 543 log_perror(srv->errh, __FILE__, __LINE__, 544 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 545 return HANDLER_ERROR; 546 } 547 } 548 break; 549 case 9: /* compress.filetype */ 550 log_error(srv->errh, __FILE__, __LINE__, 551 "DEPRECATED: %s replaced with deflate.mimetypes", 552 cpk[cpv->k_id].k); 553 break; 554 case 10:/* compress.allowed-encodings */ 555 log_error(srv->errh, __FILE__, __LINE__, 556 "DEPRECATED: %s replaced with deflate.allowed-encodings", 557 cpk[cpv->k_id].k); 558 cpv->v.shrt = (unsigned short) 559 mod_deflate_encodings_to_flags(cpv->v.a); 560 cpv->vtype = T_CONFIG_SHORT; 561 break; 562 case 11:/* compress.cache-dir */ 563 log_error(srv->errh, __FILE__, __LINE__, 564 "DEPRECATED: %s replaced with deflate.cache-dir", 565 cpk[cpv->k_id].k); 566 if (!buffer_string_is_empty(cpv->v.b)) { 567 buffer *b; 568 *(const buffer **)&b = cpv->v.b; 569 const uint32_t len = buffer_string_length(b); 570 if (len > 0 && '/' == b->ptr[len-1]) 571 buffer_string_set_length(b, len-1); /*remove end slash*/ 572 struct stat st; 573 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 574 log_perror(srv->errh, __FILE__, __LINE__, 575 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 576 return HANDLER_ERROR; 577 } 578 } 579 break; 580 case 12:/* compress.max-filesize */ 581 log_error(srv->errh, __FILE__, __LINE__, 582 "DEPRECATED: %s replaced with deflate.max-compress-size", 583 cpk[cpv->k_id].k); 584 break; 585 case 13:/* compress.max-loadavg */ 586 log_error(srv->errh, __FILE__, __LINE__, 587 "DEPRECATED: %s replaced with deflate.max-loadavg", 588 cpk[cpv->k_id].k); 589 cpv->v.d = (!buffer_string_is_empty(cpv->v.b)) 590 ? strtod(cpv->v.b->ptr, NULL) 591 : 0.0; 592 break; 593 default:/* should not happen */ 594 break; 595 } 596 } 597 } 598 599 p->defaults.allowed_encodings = 0; 600 p->defaults.max_compress_size = 128*1024; /*(128 MB measured as num KB)*/ 601 p->defaults.min_compress_size = 256; 602 p->defaults.compression_level = -1; 603 p->defaults.output_buffer_size = 0; 604 p->defaults.work_block_size = 2048; 605 p->defaults.max_loadavg = 0.0; 606 p->defaults.sync_flush = 0; 607 608 /* initialize p->defaults from global config context */ 609 if (p->nconfig > 0 && p->cvlist->v.u2[1]) { 610 const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0]; 611 if (-1 != cpv->k_id) 612 mod_deflate_merge_config(&p->defaults, cpv); 613 } 614 615 return HANDLER_GO_ON; 616 } 617 618 619 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) 620 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { 621 ssize_t wr; 622 do { 623 wr = write(hctx->cache_fd, out, len); 624 } while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR); 625 return (0 == len) ? 0 : -1; 626 } 627 628 static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) { 629 if (0 == len) return 0; 630 return (-1 == hctx->cache_fd) 631 ? http_chunk_append_mem(hctx->r, out, len) 632 : mod_deflate_cache_file_append(hctx, out, len); 633 } 634 #endif 635 636 637 #ifdef USE_ZLIB 638 639 static int stream_deflate_init(handler_ctx *hctx) { 640 z_stream * const z = &hctx->u.z; 641 const plugin_data * const p = hctx->plugin_data; 642 z->zalloc = Z_NULL; 643 z->zfree = Z_NULL; 644 z->opaque = Z_NULL; 645 z->total_in = 0; 646 z->total_out = 0; 647 z->next_out = (unsigned char *)hctx->output->ptr; 648 z->avail_out = hctx->output->size; 649 650 if (Z_OK != deflateInit2(z, 651 p->conf.compression_level > 0 652 ? p->conf.compression_level 653 : Z_DEFAULT_COMPRESSION, 654 Z_DEFLATED, 655 (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 656 ? (MAX_WBITS | 16) /*(0x10 flags gzip header, trailer)*/ 657 : -MAX_WBITS, /*(negate to suppress zlib header)*/ 658 8, /* default memLevel */ 659 Z_DEFAULT_STRATEGY)) { 660 return -1; 661 } 662 663 return 0; 664 } 665 666 static int stream_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 667 z_stream * const z = &(hctx->u.z); 668 size_t len; 669 670 z->next_in = start; 671 z->avail_in = st_size; 672 hctx->bytes_in += st_size; 673 674 /* compress data */ 675 do { 676 if (Z_OK != deflate(z, Z_NO_FLUSH)) return -1; 677 678 if (z->avail_out == 0 || z->avail_in > 0) { 679 len = hctx->output->size - z->avail_out; 680 hctx->bytes_out += len; 681 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 682 return -1; 683 z->next_out = (unsigned char *)hctx->output->ptr; 684 z->avail_out = hctx->output->size; 685 } 686 } while (z->avail_in > 0); 687 688 return 0; 689 } 690 691 static int stream_deflate_flush(handler_ctx * const hctx, int end) { 692 z_stream * const z = &(hctx->u.z); 693 const plugin_data *p = hctx->plugin_data; 694 size_t len; 695 int rc = 0; 696 int done; 697 698 /* compress data */ 699 do { 700 done = 1; 701 if (end) { 702 rc = deflate(z, Z_FINISH); 703 if (rc == Z_OK) { 704 done = 0; 705 } else if (rc != Z_STREAM_END) { 706 return -1; 707 } 708 } else { 709 if (p->conf.sync_flush) { 710 rc = deflate(z, Z_SYNC_FLUSH); 711 if (rc != Z_OK) return -1; 712 } else if (z->avail_in > 0) { 713 rc = deflate(z, Z_NO_FLUSH); 714 if (rc != Z_OK) return -1; 715 } 716 } 717 718 len = hctx->output->size - z->avail_out; 719 if (z->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 720 hctx->bytes_out += len; 721 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 722 return -1; 723 z->next_out = (unsigned char *)hctx->output->ptr; 724 z->avail_out = hctx->output->size; 725 } 726 } while (z->avail_in != 0 || !done); 727 728 return 0; 729 } 730 731 static int stream_deflate_end(handler_ctx *hctx) { 732 z_stream * const z = &(hctx->u.z); 733 int rc = deflateEnd(z); 734 if (Z_OK == rc || Z_DATA_ERROR == rc) return 0; 735 736 if (z->msg != NULL) { 737 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 738 "deflateEnd error ret=%d, msg=%s", rc, z->msg); 739 } else { 740 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 741 "deflateEnd error ret=%d", rc); 742 } 743 return -1; 744 } 745 746 #endif 747 748 749 #ifdef USE_BZ2LIB 750 751 static int stream_bzip2_init(handler_ctx *hctx) { 752 bz_stream * const bz = &hctx->u.bz; 753 const plugin_data * const p = hctx->plugin_data; 754 bz->bzalloc = NULL; 755 bz->bzfree = NULL; 756 bz->opaque = NULL; 757 bz->total_in_lo32 = 0; 758 bz->total_in_hi32 = 0; 759 bz->total_out_lo32 = 0; 760 bz->total_out_hi32 = 0; 761 bz->next_out = hctx->output->ptr; 762 bz->avail_out = hctx->output->size; 763 764 if (BZ_OK != BZ2_bzCompressInit(bz, 765 p->conf.compression_level > 0 766 ? p->conf.compression_level 767 : 9, /* blocksize = 900k */ 768 0, /* verbosity */ 769 0)) { /* workFactor: default */ 770 return -1; 771 } 772 773 return 0; 774 } 775 776 static int stream_bzip2_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 777 bz_stream * const bz = &(hctx->u.bz); 778 size_t len; 779 780 bz->next_in = (char *)start; 781 bz->avail_in = st_size; 782 hctx->bytes_in += st_size; 783 784 /* compress data */ 785 do { 786 if (BZ_RUN_OK != BZ2_bzCompress(bz, BZ_RUN)) return -1; 787 788 if (bz->avail_out == 0 || bz->avail_in > 0) { 789 len = hctx->output->size - bz->avail_out; 790 hctx->bytes_out += len; 791 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 792 return -1; 793 bz->next_out = hctx->output->ptr; 794 bz->avail_out = hctx->output->size; 795 } 796 } while (bz->avail_in > 0); 797 798 return 0; 799 } 800 801 static int stream_bzip2_flush(handler_ctx * const hctx, int end) { 802 bz_stream * const bz = &(hctx->u.bz); 803 const plugin_data *p = hctx->plugin_data; 804 size_t len; 805 int rc; 806 int done; 807 808 /* compress data */ 809 do { 810 done = 1; 811 if (end) { 812 rc = BZ2_bzCompress(bz, BZ_FINISH); 813 if (rc == BZ_FINISH_OK) { 814 done = 0; 815 } else if (rc != BZ_STREAM_END) { 816 return -1; 817 } 818 } else if (bz->avail_in > 0) { 819 /* p->conf.sync_flush not implemented here, 820 * which would loop on BZ_FLUSH while BZ_FLUSH_OK 821 * until BZ_RUN_OK returned */ 822 rc = BZ2_bzCompress(bz, BZ_RUN); 823 if (rc != BZ_RUN_OK) { 824 return -1; 825 } 826 } 827 828 len = hctx->output->size - bz->avail_out; 829 if (bz->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 830 hctx->bytes_out += len; 831 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 832 return -1; 833 bz->next_out = hctx->output->ptr; 834 bz->avail_out = hctx->output->size; 835 } 836 } while (bz->avail_in != 0 || !done); 837 838 return 0; 839 } 840 841 static int stream_bzip2_end(handler_ctx *hctx) { 842 bz_stream * const bz = &(hctx->u.bz); 843 int rc = BZ2_bzCompressEnd(bz); 844 if (BZ_OK == rc || BZ_DATA_ERROR == rc) return 0; 845 846 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 847 "BZ2_bzCompressEnd error ret=%d", rc); 848 return -1; 849 } 850 851 #endif 852 853 854 #ifdef USE_BROTLI 855 856 static int stream_br_init(handler_ctx *hctx) { 857 BrotliEncoderState * const br = hctx->u.br = 858 BrotliEncoderCreateInstance(NULL, NULL, NULL); 859 if (NULL == br) return -1; 860 861 /* future: consider allowing tunables by encoder algorithm, 862 * (i.e. not generic "compression_level") */ 863 /*(note: we ignore any errors while tuning parameters here)*/ 864 const plugin_data * const p = hctx->plugin_data; 865 if (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */ 866 BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY, 867 p->conf.compression_level); 868 869 /* XXX: is this worth checking? 870 * BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */ 871 const buffer *vb = 872 http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE, 873 CONST_STR_LEN("Content-Type")); 874 if (NULL != vb) { 875 if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1) 876 || (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1) 877 && (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1) 878 || 0 == strncmp(vb->ptr+12,"ld+json", sizeof("ld+json")-1) 879 || 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1) 880 || 0 == strncmp(vb->ptr+12,"xhtml+xml", sizeof("xhtml+xml")-1) 881 || 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1))) 882 || 0 == strncmp(vb->ptr, "image/svg+xml", sizeof("image/svg+xml")-1)) 883 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); 884 else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1)) 885 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT); 886 } 887 888 return 0; 889 } 890 891 static int stream_br_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 892 const uint8_t *in = (uint8_t *)start; 893 BrotliEncoderState * const br = hctx->u.br; 894 hctx->bytes_in += st_size; 895 while (st_size || BrotliEncoderHasMoreOutput(br)) { 896 size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size) 897 ? (size_t)st_size 898 : ((~(uint32_t)0) >> 1); 899 size_t outsz = 0; 900 BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS, 901 &insz, &in, &outsz, NULL, NULL); 902 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 903 st_size -= (st_size - (off_t)insz); 904 if (outsz) { 905 hctx->bytes_out += (off_t)outsz; 906 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 907 return -1; 908 } 909 } 910 return 0; 911 } 912 913 static int stream_br_flush(handler_ctx * const hctx, int end) { 914 const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; 915 BrotliEncoderState * const br = hctx->u.br; 916 do { 917 size_t insz = 0; 918 size_t outsz = 0; 919 BrotliEncoderCompressStream(br, brmode, 920 &insz, NULL, &outsz, NULL, NULL); 921 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 922 if (outsz) { 923 hctx->bytes_out += (off_t)outsz; 924 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 925 return -1; 926 } 927 } while (BrotliEncoderHasMoreOutput(br)); 928 return 0; 929 } 930 931 static int stream_br_end(handler_ctx *hctx) { 932 BrotliEncoderState * const br = hctx->u.br; 933 BrotliEncoderDestroyInstance(br); 934 return 0; 935 } 936 937 #endif 938 939 940 static int mod_deflate_stream_init(handler_ctx *hctx) { 941 switch(hctx->compression_type) { 942 #ifdef USE_ZLIB 943 case HTTP_ACCEPT_ENCODING_GZIP: 944 case HTTP_ACCEPT_ENCODING_DEFLATE: 945 return stream_deflate_init(hctx); 946 #endif 947 #ifdef USE_BZ2LIB 948 case HTTP_ACCEPT_ENCODING_BZIP2: 949 return stream_bzip2_init(hctx); 950 #endif 951 #ifdef USE_BROTLI 952 case HTTP_ACCEPT_ENCODING_BR: 953 return stream_br_init(hctx); 954 #endif 955 default: 956 return -1; 957 } 958 } 959 960 static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 961 if (0 == st_size) return 0; 962 switch(hctx->compression_type) { 963 #ifdef USE_ZLIB 964 case HTTP_ACCEPT_ENCODING_GZIP: 965 case HTTP_ACCEPT_ENCODING_DEFLATE: 966 return stream_deflate_compress(hctx, start, st_size); 967 #endif 968 #ifdef USE_BZ2LIB 969 case HTTP_ACCEPT_ENCODING_BZIP2: 970 return stream_bzip2_compress(hctx, start, st_size); 971 #endif 972 #ifdef USE_BROTLI 973 case HTTP_ACCEPT_ENCODING_BR: 974 return stream_br_compress(hctx, start, st_size); 975 #endif 976 default: 977 UNUSED(start); 978 return -1; 979 } 980 } 981 982 static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { 983 if (0 == hctx->bytes_in) return 0; 984 switch(hctx->compression_type) { 985 #ifdef USE_ZLIB 986 case HTTP_ACCEPT_ENCODING_GZIP: 987 case HTTP_ACCEPT_ENCODING_DEFLATE: 988 return stream_deflate_flush(hctx, end); 989 #endif 990 #ifdef USE_BZ2LIB 991 case HTTP_ACCEPT_ENCODING_BZIP2: 992 return stream_bzip2_flush(hctx, end); 993 #endif 994 #ifdef USE_BROTLI 995 case HTTP_ACCEPT_ENCODING_BR: 996 return stream_br_flush(hctx, end); 997 #endif 998 default: 999 UNUSED(end); 1000 return -1; 1001 } 1002 } 1003 1004 static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) { 1005 /* store compression ratio in environment 1006 * for possible logging by mod_accesslog 1007 * (late in response handling, so not seen by most other modules) */ 1008 /*(should be called only at end of successful response compression)*/ 1009 char ratio[LI_ITOSTRING_LENGTH]; 1010 if (0 == bytes_in) return; 1011 size_t len = 1012 li_itostrn(ratio, sizeof(ratio), bytes_out * 100 / bytes_in); 1013 http_header_env_set(r, CONST_STR_LEN("ratio"), ratio, len); 1014 } 1015 1016 static int mod_deflate_stream_end(handler_ctx *hctx) { 1017 switch(hctx->compression_type) { 1018 #ifdef USE_ZLIB 1019 case HTTP_ACCEPT_ENCODING_GZIP: 1020 case HTTP_ACCEPT_ENCODING_DEFLATE: 1021 return stream_deflate_end(hctx); 1022 #endif 1023 #ifdef USE_BZ2LIB 1024 case HTTP_ACCEPT_ENCODING_BZIP2: 1025 return stream_bzip2_end(hctx); 1026 #endif 1027 #ifdef USE_BROTLI 1028 case HTTP_ACCEPT_ENCODING_BR: 1029 return stream_br_end(hctx); 1030 #endif 1031 default: 1032 return -1; 1033 } 1034 } 1035 1036 static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) { 1037 int rc = mod_deflate_stream_end(hctx); 1038 1039 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */ 1040 if (0 == rc && hctx->bytes_in < hctx->bytes_out) 1041 log_error(r->conf.errh, __FILE__, __LINE__, 1042 "uri %s in=%lld smaller than out=%lld", r->target.ptr, 1043 (long long)hctx->bytes_in, (long long)hctx->bytes_out); 1044 #endif 1045 1046 handler_ctx_free(hctx); 1047 return rc; 1048 } 1049 1050 1051 static int mod_deflate_file_chunk(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t st_size) { 1052 off_t abs_offset; 1053 off_t toSend = -1; 1054 char *start; 1055 #ifdef USE_MMAP 1056 off_t we_want_to_mmap = 2 MByte; 1057 off_t we_want_to_send = st_size; 1058 volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */ 1059 #else 1060 start = NULL; 1061 #endif 1062 1063 if (-1 == c->file.fd) { /* open the file if not already open */ 1064 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) { 1065 log_perror(r->conf.errh, __FILE__, __LINE__, "open failed %s", c->mem->ptr); 1066 return -1; 1067 } 1068 } 1069 1070 abs_offset = c->file.start + c->offset; 1071 1072 #ifdef USE_MMAP 1073 /* mmap the buffer 1074 * - first mmap 1075 * - new mmap as the we are at the end of the last one */ 1076 if (c->file.mmap.start == MAP_FAILED || 1077 abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) { 1078 1079 /* Optimizations for the future: 1080 * 1081 * adaptive mem-mapping 1082 * the problem: 1083 * we mmap() the whole file. If someone has a lot of large files and 32bit 1084 * machine the virtual address area will be exhausted and we will have a failing 1085 * mmap() call. 1086 * solution: 1087 * only mmap 16M in one chunk and move the window as soon as we have finished 1088 * the first 8M 1089 * 1090 * read-ahead buffering 1091 * the problem: 1092 * sending out several large files in parallel trashes the read-ahead of the 1093 * kernel leading to long wait-for-seek times. 1094 * solutions: (increasing complexity) 1095 * 1. use madvise 1096 * 2. use a internal read-ahead buffer in the chunk-structure 1097 * 3. use non-blocking IO for file-transfers 1098 * */ 1099 1100 /* all mmap()ed areas are 512kb expect the last which might be smaller */ 1101 off_t to_mmap; 1102 1103 /* this is a remap, move the mmap-offset */ 1104 if (c->file.mmap.start != MAP_FAILED) { 1105 munmap(c->file.mmap.start, c->file.mmap.length); 1106 c->file.mmap.offset += we_want_to_mmap; 1107 } else { 1108 /* in case the range-offset is after the first mmap()ed area we skip the area */ 1109 c->file.mmap.offset = 0; 1110 1111 while (c->file.mmap.offset + we_want_to_mmap < c->file.start) { 1112 c->file.mmap.offset += we_want_to_mmap; 1113 } 1114 } 1115 1116 /* length is rel, c->offset too, assume there is no limit at the mmap-boundaries */ 1117 to_mmap = (c->file.start + c->file.length) - c->file.mmap.offset; 1118 if (to_mmap > we_want_to_mmap) to_mmap = we_want_to_mmap; 1119 /* we have more to send than we can mmap() at once */ 1120 if (we_want_to_send > to_mmap) we_want_to_send = to_mmap; 1121 1122 if (MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset)) 1123 && (errno != EINVAL || MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset)))) { 1124 log_perror(r->conf.errh, __FILE__, __LINE__, 1125 "mmap failed %s %d", c->mem->ptr, c->file.fd); 1126 1127 return -1; 1128 } 1129 1130 c->file.mmap.length = to_mmap; 1131 #ifdef HAVE_MADVISE 1132 /* don't advise files < 64Kb */ 1133 if (c->file.mmap.length > (64 KByte) && 1134 0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) { 1135 log_perror(r->conf.errh, __FILE__, __LINE__, 1136 "madvise failed %s %d", c->mem->ptr, c->file.fd); 1137 } 1138 #endif 1139 1140 /* chunk_reset() or chunk_free() will cleanup for us */ 1141 } 1142 1143 /* to_send = abs_mmap_end - abs_offset */ 1144 toSend = (c->file.mmap.offset + c->file.mmap.length) - abs_offset; 1145 if (toSend > we_want_to_send) toSend = we_want_to_send; 1146 1147 if (toSend < 0) { 1148 log_error(r->conf.errh, __FILE__, __LINE__, 1149 "toSend is negative: %lld %lld %lld %lld", 1150 (long long)toSend, 1151 (long long)c->file.mmap.length, 1152 (long long)abs_offset, 1153 (long long)c->file.mmap.offset); 1154 force_assert(toSend < 0); 1155 } 1156 1157 start = c->file.mmap.start; 1158 mapped = 1; 1159 #endif 1160 1161 if (MAP_FAILED == c->file.mmap.start) { 1162 toSend = st_size; 1163 if (toSend > 2 MByte) toSend = 2 MByte; 1164 if (NULL == (start = malloc((size_t)toSend)) || -1 == lseek(c->file.fd, abs_offset, SEEK_SET) || toSend != read(c->file.fd, start, (size_t)toSend)) { 1165 log_perror(r->conf.errh, __FILE__, __LINE__, "reading %s failed", c->mem->ptr); 1166 1167 free(start); 1168 return -1; 1169 } 1170 abs_offset = 0; 1171 } 1172 1173 #ifdef USE_MMAP 1174 if (mapped) { 1175 signal(SIGBUS, sigbus_handler); 1176 sigbus_jmp_valid = 1; 1177 if (0 != sigsetjmp(sigbus_jmp, 1)) { 1178 sigbus_jmp_valid = 0; 1179 1180 log_error(r->conf.errh, __FILE__, __LINE__, 1181 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1182 return -1; 1183 } 1184 } 1185 #endif 1186 1187 if (mod_deflate_compress(hctx, 1188 (unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) { 1189 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1190 toSend = -1; 1191 } 1192 1193 #ifdef USE_MMAP 1194 if (mapped) 1195 sigbus_jmp_valid = 0; 1196 else 1197 #endif 1198 free(start); 1199 1200 return toSend; 1201 } 1202 1203 1204 static handler_t deflate_compress_response(request_st * const r, handler_ctx * const hctx) { 1205 off_t len, max; 1206 int close_stream; 1207 1208 /* move all chunk from write_queue into our in_queue, then adjust 1209 * counters since r->write_queue is reused for compressed output */ 1210 chunkqueue * const cq = r->write_queue; 1211 len = chunkqueue_length(cq); 1212 chunkqueue_remove_finished_chunks(cq); 1213 chunkqueue_append_chunkqueue(hctx->in_queue, cq); 1214 cq->bytes_in -= len; 1215 cq->bytes_out -= len; 1216 1217 max = chunkqueue_length(hctx->in_queue); 1218 #if 0 1219 /* calculate max bytes to compress for this call */ 1220 if (p->conf.sync_flush && max > (len = p->conf.work_block_size << 10)) { 1221 max = len; 1222 } 1223 #endif 1224 1225 /* Compress chunks from in_queue into chunks for write_queue */ 1226 while (max) { 1227 chunk *c = hctx->in_queue->first; 1228 1229 switch(c->type) { 1230 case MEM_CHUNK: 1231 len = buffer_string_length(c->mem) - c->offset; 1232 if (len > max) len = max; 1233 if (mod_deflate_compress(hctx, (unsigned char *)c->mem->ptr+c->offset, len) < 0) { 1234 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1235 return HANDLER_ERROR; 1236 } 1237 break; 1238 case FILE_CHUNK: 1239 len = c->file.length - c->offset; 1240 if (len > max) len = max; 1241 if ((len = mod_deflate_file_chunk(r, hctx, c, len)) < 0) { 1242 log_error(r->conf.errh, __FILE__, __LINE__, "compress file chunk failed."); 1243 return HANDLER_ERROR; 1244 } 1245 break; 1246 default: 1247 log_error(r->conf.errh, __FILE__, __LINE__, "%d type not known", c->type); 1248 return HANDLER_ERROR; 1249 } 1250 1251 max -= len; 1252 chunkqueue_mark_written(hctx->in_queue, len); 1253 } 1254 1255 /*(currently should always be true)*/ 1256 /*(current implementation requires response be complete)*/ 1257 close_stream = (r->resp_body_finished 1258 && chunkqueue_is_empty(hctx->in_queue)); 1259 if (mod_deflate_stream_flush(hctx, close_stream) < 0) { 1260 log_error(r->conf.errh, __FILE__, __LINE__, "flush error"); 1261 return HANDLER_ERROR; 1262 } 1263 1264 return close_stream ? HANDLER_FINISHED : HANDLER_GO_ON; 1265 } 1266 1267 1268 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { 1269 /* get client side support encodings */ 1270 int accept_encoding = 0; 1271 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) 1272 UNUSED(value); 1273 UNUSED(label); 1274 #else 1275 for (; *value; ++value) { 1276 const char *v; 1277 while (*value == ' ' || *value == ',') ++value; 1278 v = value; 1279 while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0') 1280 ++value; 1281 switch (value - v) { 1282 case 2: 1283 #ifdef USE_BROTLI 1284 if (0 == memcmp(v, "br", 2)) 1285 accept_encoding |= HTTP_ACCEPT_ENCODING_BR; 1286 #endif 1287 break; 1288 case 4: 1289 #ifdef USE_ZLIB 1290 if (0 == memcmp(v, "gzip", 4)) 1291 accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; 1292 #endif 1293 break; 1294 case 5: 1295 #ifdef USE_BZ2LIB 1296 if (0 == memcmp(v, "bzip2", 5)) 1297 accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; 1298 #endif 1299 break; 1300 case 6: 1301 #ifdef USE_ZLIB 1302 if (0 == memcmp(v, "x-gzip", 6)) 1303 accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP; 1304 #endif 1305 break; 1306 case 7: 1307 #ifdef USE_ZLIB 1308 if (0 == memcmp(v, "deflate", 7)) 1309 accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; 1310 #endif 1311 #ifdef USE_BZ2LIB 1312 if (0 == memcmp(v, "x-bzip2", 7)) 1313 accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2; 1314 #endif 1315 break; 1316 #if 0 1317 case 8: 1318 if (0 == memcmp(v, "identity", 8)) 1319 accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; 1320 else if (0 == memcmp(v, "compress", 8)) 1321 accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; 1322 break; 1323 #endif 1324 default: 1325 break; 1326 } 1327 if (*value == ';') { 1328 while (*value != ',' && *value != '\0') ++value; 1329 } 1330 if (*value == '\0') break; 1331 } 1332 #endif 1333 1334 /* mask to limit to allowed_encodings */ 1335 accept_encoding &= p->conf.allowed_encodings; 1336 1337 /* select best matching encoding */ 1338 #ifdef USE_BROTLI 1339 if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { 1340 *label = "br"; 1341 return HTTP_ACCEPT_ENCODING_BR; 1342 } else 1343 #endif 1344 #ifdef USE_BZ2LIB 1345 if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) { 1346 *label = "bzip2"; 1347 return HTTP_ACCEPT_ENCODING_BZIP2; 1348 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_BZIP2) { 1349 *label = "x-bzip2"; 1350 return HTTP_ACCEPT_ENCODING_BZIP2; 1351 } else 1352 #endif 1353 #ifdef USE_ZLIB 1354 if (accept_encoding & HTTP_ACCEPT_ENCODING_GZIP) { 1355 *label = "gzip"; 1356 return HTTP_ACCEPT_ENCODING_GZIP; 1357 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_GZIP) { 1358 *label = "x-gzip"; 1359 return HTTP_ACCEPT_ENCODING_GZIP; 1360 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_DEFLATE) { 1361 *label = "deflate"; 1362 return HTTP_ACCEPT_ENCODING_DEFLATE; 1363 } else 1364 #endif 1365 if (0 == accept_encoding) { 1366 return 0; 1367 } else { 1368 return 0; 1369 } 1370 } 1371 1372 REQUEST_FUNC(mod_deflate_handle_response_start) { 1373 plugin_data *p = p_d; 1374 const buffer *vbro; 1375 buffer *vb; 1376 handler_ctx *hctx; 1377 const char *label; 1378 off_t len; 1379 uint32_t etaglen; 1380 int compression_type; 1381 handler_t rc; 1382 int had_vary = 0; 1383 1384 /*(current implementation requires response be complete)*/ 1385 if (!r->resp_body_finished) return HANDLER_GO_ON; 1386 if (r->http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; 1387 if (r->resp_htags & HTTP_HEADER_TRANSFER_ENCODING) return HANDLER_GO_ON; 1388 if (r->resp_htags & HTTP_HEADER_CONTENT_ENCODING) return HANDLER_GO_ON; 1389 1390 /* disable compression for some http status types. */ 1391 switch(r->http_status) { 1392 case 100: 1393 case 101: 1394 case 204: 1395 case 205: 1396 case 304: 1397 /* disable compression as we have no response entity */ 1398 return HANDLER_GO_ON; 1399 default: 1400 break; 1401 } 1402 1403 mod_deflate_patch_config(r, p); 1404 1405 /* check if deflate configured for any mimetypes */ 1406 if (NULL == p->conf.mimetypes) return HANDLER_GO_ON; 1407 1408 /* check if size of response is below min-compress-size or exceeds max*/ 1409 /* (r->resp_body_finished checked at top of routine) */ 1410 len = chunkqueue_length(r->write_queue); 1411 if (len <= (off_t)p->conf.min_compress_size) return HANDLER_GO_ON; 1412 if (p->conf.max_compress_size /*(max_compress_size in KB)*/ 1413 && len > ((off_t)p->conf.max_compress_size << 10)) { 1414 return HANDLER_GO_ON; 1415 } 1416 1417 /* Check Accept-Encoding for supported encoding. */ 1418 vbro = http_header_request_get(r, HTTP_HEADER_ACCEPT_ENCODING, CONST_STR_LEN("Accept-Encoding")); 1419 if (NULL == vbro) return HANDLER_GO_ON; 1420 1421 /* find matching encodings */ 1422 compression_type = mod_deflate_choose_encoding(vbro->ptr, p, &label); 1423 if (!compression_type) return HANDLER_GO_ON; 1424 1425 /* Check mimetype in response header "Content-Type" */ 1426 if (NULL != (vbro = http_header_response_get(r, HTTP_HEADER_CONTENT_TYPE, CONST_STR_LEN("Content-Type")))) { 1427 if (NULL == array_match_value_prefix(p->conf.mimetypes, vbro)) return HANDLER_GO_ON; 1428 } else { 1429 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */ 1430 data_string *mimetype = (data_string *)p->conf.mimetypes->data[0]; 1431 if (!buffer_string_is_empty(&mimetype->value)) return HANDLER_GO_ON; 1432 } 1433 1434 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */ 1435 if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) { 1436 had_vary = 1; 1437 if (NULL == strstr(vb->ptr, "Accept-Encoding")) { 1438 buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding")); 1439 } 1440 } else { 1441 http_header_response_append(r, HTTP_HEADER_VARY, 1442 CONST_STR_LEN("Vary"), 1443 CONST_STR_LEN("Accept-Encoding")); 1444 } 1445 1446 /* check ETag as is done in http_response_handle_cachable() 1447 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */ 1448 vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag")); 1449 etaglen = (NULL != vb) ? buffer_string_length(vb) : 0; 1450 if (NULL != vb && (r->rqst_htags & HTTP_HEADER_IF_NONE_MATCH)) { 1451 const buffer *if_none_match = http_header_response_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match")); 1452 if (etaglen 1453 && r->http_status < 300 /*(want 2xx only)*/ 1454 && NULL != if_none_match 1455 && 0 == strncmp(if_none_match->ptr, vb->ptr, etaglen-1) 1456 && if_none_match->ptr[etaglen-1] == '-' 1457 && 0 == strncmp(if_none_match->ptr+etaglen, label, strlen(label))) { 1458 1459 if (http_method_get_or_head(r->http_method)) { 1460 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1461 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1462 buffer_append_string(vb, label); 1463 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1464 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1465 r->http_status = 304; 1466 } else { 1467 r->http_status = 412; 1468 } 1469 1470 /* response_start hook occurs after error docs have been handled. 1471 * For now, send back empty response body. 1472 * In the future, might extract the error doc code so that it 1473 * might be run again if response_start hooks return with 1474 * changed http_status and r->handler_module = NULL */ 1475 /* clear content length even if 304 since compressed length unknown */ 1476 http_response_body_clear(r, 0); 1477 1478 r->resp_body_finished = 1; 1479 r->handler_module = NULL; 1480 return HANDLER_GO_ON; 1481 } 1482 } 1483 1484 if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < r->con->srv->loadavg[0]) { 1485 return HANDLER_GO_ON; 1486 } 1487 1488 /* update ETag, if ETag response header is set */ 1489 if (etaglen) { 1490 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1491 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1492 buffer_append_string(vb, label); 1493 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1494 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1495 } 1496 1497 /* set Content-Encoding to show selected compression type */ 1498 http_header_response_set(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding"), label, strlen(label)); 1499 1500 /* clear Content-Length and r->write_queue if HTTP HEAD request 1501 * (alternatively, could return original Content-Length with HEAD 1502 * request if ETag not modified and Content-Encoding not added) */ 1503 if (HTTP_METHOD_HEAD == r->http_method) { 1504 /* ensure that uncompressed Content-Length is not sent in HEAD response */ 1505 http_response_body_clear(r, 0); 1506 return HANDLER_GO_ON; 1507 } 1508 1509 /* restrict items eligible for cache of compressed responses 1510 * (This module does not aim to be a full caching proxy) 1511 * response must be complete (not streaming response) 1512 * must not have prior Vary response header (before Accept-Encoding added) 1513 * must not have Range response header 1514 * must have ETag 1515 * must be file 1516 * must be single FILE_CHUNK in chunkqueue 1517 * must not be chunkqueue temporary file 1518 * must be whole file, not partial content 1519 * Note: small files (< 32k (see http_chunk.c)) will have been read into 1520 * memory (if streaming HTTP/1.1 chunked response) and will end up 1521 * getting stream-compressed rather than cached on disk as compressed 1522 * file 1523 */ 1524 buffer *tb = NULL; 1525 if (!buffer_is_empty(p->conf.cache_dir) 1526 && !had_vary 1527 && etaglen > 2 1528 && r->resp_body_finished 1529 && r->write_queue->first == r->write_queue->last 1530 && r->write_queue->first->type == FILE_CHUNK 1531 && r->write_queue->first->file.start == 0 1532 && !r->write_queue->first->file.is_temp 1533 && !http_header_response_get(r, HTTP_HEADER_RANGE, 1534 CONST_STR_LEN("Range"))) { 1535 tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb); 1536 /*(checked earlier and skipped if Transfer-Encoding had been set)*/ 1537 stat_cache_entry *sce = stat_cache_get_entry(tb); 1538 if (NULL != sce) { 1539 chunkqueue_reset(r->write_queue); 1540 if (0 != http_chunk_append_file(r, tb)) 1541 return HANDLER_ERROR; 1542 if (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH) 1543 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, 1544 CONST_STR_LEN("Content-Length")); 1545 mod_deflate_note_ratio(r, sce->st.st_size, len); 1546 return HANDLER_GO_ON; 1547 } 1548 /* sanity check that response was whole file; 1549 * (racy since using stat_cache, but cache file only if match) */ 1550 sce = stat_cache_get_entry(r->write_queue->first->mem); 1551 if (NULL == sce || sce->st.st_size != len) 1552 tb = NULL; 1553 if (0 != mkdir_for_file(tb->ptr)) 1554 tb = NULL; 1555 } 1556 1557 /* enable compression */ 1558 p->conf.sync_flush = 1559 (r->conf.stream_response_body && 0 == p->conf.output_buffer_size); 1560 hctx = handler_ctx_init(); 1561 hctx->plugin_data = p; 1562 hctx->compression_type = compression_type; 1563 hctx->r = r; 1564 /* setup output buffer */ 1565 buffer_clear(&p->tmp_buf); 1566 hctx->output = &p->tmp_buf; 1567 /* open cache file if caching compressed file */ 1568 if (tb) mod_deflate_cache_file_open(hctx, tb); 1569 if (0 != mod_deflate_stream_init(hctx)) { 1570 /*(should not happen unless ENOMEM)*/ 1571 handler_ctx_free(hctx); 1572 log_error(r->conf.errh, __FILE__, __LINE__, 1573 "Failed to initialize compression %s", label); 1574 /* restore prior Etag and unset Content-Encoding */ 1575 if (etaglen) { 1576 vb->ptr[etaglen-1] = '"'; /*(overwrite '-')*/ 1577 buffer_string_set_length(vb, etaglen); 1578 } 1579 http_header_response_unset(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding")); 1580 return HANDLER_GO_ON; 1581 } 1582 1583 #ifdef USE_BROTLI 1584 if (r->resp_body_finished 1585 && (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR) 1586 && (len >> 1) < (off_t)((~(uint32_t)0u) >> 1)) 1587 BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len); 1588 #endif 1589 1590 if (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH) { 1591 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length")); 1592 } 1593 r->plugin_ctx[p->id] = hctx; 1594 1595 rc = deflate_compress_response(r, hctx); 1596 if (HANDLER_GO_ON == rc) return HANDLER_GO_ON; 1597 if (HANDLER_FINISHED == rc) { 1598 if (-1 == hctx->cache_fd 1599 || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) { 1600 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 1601 rc = HANDLER_GO_ON; 1602 } 1603 else 1604 rc = HANDLER_ERROR; 1605 } 1606 r->plugin_ctx[p->id] = NULL; 1607 if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR; 1608 return rc; 1609 } 1610 1611 static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) { 1612 plugin_data *p = p_d; 1613 handler_ctx *hctx = r->plugin_ctx[p->id]; 1614 1615 if (NULL != hctx) { 1616 r->plugin_ctx[p->id] = NULL; 1617 deflate_compress_cleanup(r, hctx); 1618 } 1619 1620 return HANDLER_GO_ON; 1621 } 1622 1623 int mod_deflate_plugin_init(plugin *p); 1624 int mod_deflate_plugin_init(plugin *p) { 1625 p->version = LIGHTTPD_VERSION_ID; 1626 p->name = "deflate"; 1627 1628 p->init = mod_deflate_init; 1629 p->cleanup = mod_deflate_free; 1630 p->set_defaults = mod_deflate_set_defaults; 1631 p->handle_request_reset = mod_deflate_cleanup; 1632 p->handle_response_start = mod_deflate_handle_response_start; 1633 1634 return 0; 1635 } 1636