1 /* mod_deflate 2 * 3 * 4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch 5 * 6 * Bug fix and new features: 7 * 1) fix loop bug when content-length is bigger than work-block-size*k 8 * 9 * ------- 10 * 11 * lighttpd-1.4.26.mod_deflate.patch from 12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate 13 * 14 * ------- 15 * 16 * Patch further modified in this incarnation. 17 * 18 * Note: this patch only handles completed responses 19 * (r->resp_body_finished) 20 * this patch does not currently handle streaming dynamic responses, 21 * and therefore also does not worry about Transfer-Encoding: chunked 22 * (or having separate con->output_queue for chunked-encoded output) 23 * (or using separate buffers per connection instead of p->tmp_buf) 24 * (or handling interactions with block buffering and write timeouts) 25 * 26 * Bug fix: 27 * - fixed major bug with compressing chunks with offset > 0 28 * x-ref: 29 * "Response breaking in mod_deflate" 30 * https://redmine.lighttpd.net/issues/986 31 * - fix broken (in some cases) chunk accounting in deflate_compress_response() 32 * - fix broken bzip2 33 * x-ref: 34 * "mod_deflate's bzip2 broken by default" 35 * https://redmine.lighttpd.net/issues/2035 36 * - fix mismatch with current chunk interfaces 37 * x-ref: 38 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)" 39 * https://redmine.lighttpd.net/issues/1510 40 * 41 * Behavior changes from prior patch: 42 * - deflate.mimetypes must now be configured to enable compression 43 * deflate.mimetypes = ( ) # compress nothing (disabled; default) 44 * deflate.mimetypes = ( "" ) # compress all mimetypes 45 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes 46 * x-ref: 47 * "mod_deflate enabled by default" 48 * https://redmine.lighttpd.net/issues/1394 49 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes) 50 * - deflate.debug removed (was developer debug trace, not end-user debug) 51 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress) 52 * x-ref: 53 * "mod_deflate should allow limiting of compression algorithm from the configuration file" 54 * https://redmine.lighttpd.net/issues/996 55 * "mod_compress disabling methods" 56 * https://redmine.lighttpd.net/issues/1773 57 * - deflate.nocompress-url removed since disabling compression for a URL 58 * can now easily be done by setting to a blank list either directive 59 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional 60 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) } 61 * - deflate.sync-flush removed; controlled by r->conf.stream_response_body 62 * (though streaming compression not currently implemented in mod_deflate) 63 * - inactive directives in this patch 64 * (since r->resp_body_finished required) 65 * deflate.work-block-size 66 * deflate.output-buffer-size 67 * - remove weak file size check; SIGBUS is trapped, file that shrink will error 68 * x-ref: 69 * "mod_deflate: filesize check is too weak" 70 * https://redmine.lighttpd.net/issues/1512 71 * - change default deflate.min-compress-size from 0 to now be 256 72 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits 73 * Apache 2.4 mod_deflate minimum is 68 bytes 74 * Akamai recommends minimum 860 bytes 75 * Google recommends minimum be somewhere in range between 150 and 1024 bytes 76 * - deflate.max-compress-size new directive (in kb like compress.max_filesize) 77 * - deflate.mem-level removed (too many knobs for little benefit) 78 * - deflate.window-size removed (too many knobs for little benefit) 79 * 80 * Future: 81 * - config directives may be changed, renamed, or removed 82 * e.g. A set of reasonable defaults might be chosen 83 * instead of making them configurable. 84 * deflate.min-compress-size 85 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes 86 * to avoid compressing, even if a broader deflate.mimetypes matched, 87 * e.g. to compress all "text/" except "text/special". 88 * 89 * Implementation notes: 90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer() 91 * so that p->tmp_buf can be large and re-used. This results in an extra copy 92 * of compressed data before data is sent to network, though if the compressed 93 * size is larger than 64k, it ends up being sent to a temporary file on 94 * disk without suffering an extra copy in memory, and without extra chunk 95 * create and destroy. If this is ever changed to give away buffers, then use 96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across 97 * multiple requests being handled in parallel. 98 */ 99 #include "first.h" 100 101 #include <sys/types.h> 102 #include <sys/stat.h> 103 #include "sys-mmap.h" 104 105 #include <fcntl.h> 106 #include <stdlib.h> 107 #include <string.h> 108 #include <errno.h> 109 #include <time.h> 110 #include <unistd.h> /* getpid() read() unlink() write() */ 111 112 #include "base.h" 113 #include "fdevent.h" 114 #include "log.h" 115 #include "buffer.h" 116 #include "etag.h" 117 #include "http_chunk.h" 118 #include "http_header.h" 119 #include "response.h" 120 #include "stat_cache.h" 121 122 #include "plugin.h" 123 124 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ 125 # define USE_ZLIB 126 # include <zlib.h> 127 #endif 128 #ifndef Z_DEFAULT_COMPRESSION 129 #define Z_DEFAULT_COMPRESSION -1 130 #endif 131 #ifndef MAX_WBITS 132 #define MAX_WBITS 15 133 #endif 134 135 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 136 # define USE_BZ2LIB 137 /* we don't need stdio interface */ 138 # define BZ_NO_STDIO 139 # include <bzlib.h> 140 #endif 141 142 #if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI 143 # define USE_BROTLI 144 # include <brotli/encode.h> 145 #endif 146 147 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP 148 #define USE_MMAP 149 150 #include "sys-mmap.h" 151 #include <setjmp.h> 152 #include <signal.h> 153 154 static volatile int sigbus_jmp_valid; 155 static sigjmp_buf sigbus_jmp; 156 157 static void sigbus_handler(int sig) { 158 UNUSED(sig); 159 if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1); 160 log_failed_assert(__FILE__, __LINE__, "SIGBUS"); 161 } 162 #endif 163 164 /* request: accept-encoding */ 165 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) 166 #define HTTP_ACCEPT_ENCODING_GZIP BV(1) 167 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) 168 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) 169 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) 170 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) 171 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) 172 #define HTTP_ACCEPT_ENCODING_BR BV(7) 173 174 typedef struct { 175 const array *mimetypes; 176 const buffer *cache_dir; 177 unsigned int max_compress_size; 178 unsigned short min_compress_size; 179 unsigned short output_buffer_size; 180 unsigned short work_block_size; 181 unsigned short sync_flush; 182 short compression_level; 183 short allowed_encodings; 184 double max_loadavg; 185 } plugin_config; 186 187 typedef struct { 188 PLUGIN_DATA; 189 plugin_config defaults; 190 plugin_config conf; 191 192 buffer tmp_buf; 193 } plugin_data; 194 195 typedef struct { 196 union { 197 #ifdef USE_ZLIB 198 z_stream z; 199 #endif 200 #ifdef USE_BZ2LIB 201 bz_stream bz; 202 #endif 203 #ifdef USE_BROTLI 204 BrotliEncoderState *br; 205 #endif 206 int dummy; 207 } u; 208 off_t bytes_in; 209 off_t bytes_out; 210 buffer *output; 211 plugin_data *plugin_data; 212 request_st *r; 213 int compression_type; 214 int cache_fd; 215 char *cache_fn; 216 chunkqueue in_queue; 217 } handler_ctx; 218 219 static handler_ctx *handler_ctx_init() { 220 handler_ctx *hctx; 221 222 hctx = calloc(1, sizeof(*hctx)); 223 chunkqueue_init(&hctx->in_queue); 224 hctx->cache_fd = -1; 225 226 return hctx; 227 } 228 229 static void handler_ctx_free(handler_ctx *hctx) { 230 if (hctx->cache_fn) { 231 unlink(hctx->cache_fn); 232 free(hctx->cache_fn); 233 } 234 if (-1 != hctx->cache_fd) 235 close(hctx->cache_fd); 236 #if 0 237 if (hctx->output != &p->tmp_buf) { 238 buffer_free(hctx->output); 239 } 240 #endif 241 chunkqueue_reset(&hctx->in_queue); 242 free(hctx); 243 } 244 245 INIT_FUNC(mod_deflate_init) { 246 plugin_data * const p = calloc(1, sizeof(plugin_data)); 247 buffer_string_prepare_copy(&p->tmp_buf, 65536); 248 return p; 249 } 250 251 FREE_FUNC(mod_deflate_free) { 252 plugin_data *p = p_d; 253 free(p->tmp_buf.ptr); 254 } 255 256 #if defined(_WIN32) && !defined(__CYGWIN__) 257 #define mkdir(x,y) mkdir(x) 258 #endif 259 260 static int mkdir_for_file (char *fn) { 261 for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) { 262 if (p[1] == '\0') return 0; /* ignore trailing slash */ 263 *p = '\0'; 264 int rc = mkdir(fn, 0700); 265 *p = '/'; 266 if (0 != rc && errno != EEXIST) return -1; 267 } 268 return 0; 269 } 270 271 static int mkdir_recursive (char *dir) { 272 return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST) 273 ? 0 274 : -1; 275 } 276 277 static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) { 278 /* XXX: future: for shorter paths into the cache, we could checksum path, 279 * and then shard it to avoid a huge single directory. 280 * Alternatively, could use &r->uri.path, minus any 281 * (matching) &r->pathinfo suffix, with result url-encoded */ 282 buffer * const tb = r->tmp_buf; 283 buffer_copy_buffer(tb, cache_dir); 284 buffer_append_string_len(tb, CONST_BUF_LEN(&r->physical.path)); 285 buffer_append_string_len(tb, CONST_STR_LEN("-")); 286 buffer_append_string_len(tb, etag->ptr+1, buffer_string_length(etag)-2); 287 return tb; 288 } 289 290 static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) { 291 /* race exists whereby up to # workers might attempt to compress same 292 * file at same time if requested at same time, but this is unlikely 293 * and resolves itself by atomic rename into place when done */ 294 const uint32_t fnlen = buffer_string_length(fn); 295 hctx->cache_fn = malloc(fnlen+1+LI_ITOSTRING_LENGTH+1); 296 force_assert(hctx->cache_fn); 297 memcpy(hctx->cache_fn, fn->ptr, fnlen); 298 hctx->cache_fn[fnlen] = '.'; 299 const size_t ilen = 300 li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid()); 301 hctx->cache_fn[fnlen+1+ilen] = '\0'; 302 hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600); 303 if (-1 == hctx->cache_fd) { 304 free(hctx->cache_fn); 305 hctx->cache_fn = NULL; 306 } 307 } 308 309 static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) { 310 if (0 != fdevent_rename(hctx->cache_fn, fn->ptr)) 311 return -1; 312 free(hctx->cache_fn); 313 hctx->cache_fn = NULL; 314 chunkqueue_reset(&r->write_queue); 315 int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out); 316 hctx->cache_fd = -1; 317 return rc; 318 } 319 320 static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) { 321 switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */ 322 case 0: /* deflate.mimetypes */ 323 pconf->mimetypes = cpv->v.a; 324 break; 325 case 1: /* deflate.allowed-encodings */ 326 pconf->allowed_encodings = (short)cpv->v.shrt; 327 break; 328 case 2: /* deflate.max-compress-size */ 329 pconf->max_compress_size = cpv->v.u; 330 break; 331 case 3: /* deflate.min-compress-size */ 332 pconf->min_compress_size = cpv->v.shrt; 333 break; 334 case 4: /* deflate.compression-level */ 335 pconf->compression_level = (short)cpv->v.shrt; 336 break; 337 case 5: /* deflate.output-buffer-size */ 338 pconf->output_buffer_size = cpv->v.shrt; 339 break; 340 case 6: /* deflate.work-block-size */ 341 pconf->work_block_size = cpv->v.shrt; 342 break; 343 case 7: /* deflate.max-loadavg */ 344 pconf->max_loadavg = cpv->v.d; 345 break; 346 case 8: /* deflate.cache-dir */ 347 pconf->cache_dir = cpv->v.b; 348 break; 349 case 9: /* compress.filetype */ 350 pconf->mimetypes = cpv->v.a; 351 break; 352 case 10:/* compress.allowed-encodings */ 353 pconf->allowed_encodings = (short)cpv->v.shrt; 354 break; 355 case 11:/* compress.cache-dir */ 356 pconf->cache_dir = cpv->v.b; 357 break; 358 case 12:/* compress.max-filesize */ 359 pconf->max_compress_size = cpv->v.u; 360 break; 361 case 13:/* compress.max-loadavg */ 362 pconf->max_loadavg = cpv->v.d; 363 break; 364 default:/* should not happen */ 365 return; 366 } 367 } 368 369 static void mod_deflate_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) { 370 do { 371 mod_deflate_merge_config_cpv(pconf, cpv); 372 } while ((++cpv)->k_id != -1); 373 } 374 375 static void mod_deflate_patch_config(request_st * const r, plugin_data * const p) { 376 memcpy(&p->conf, &p->defaults, sizeof(plugin_config)); 377 for (int i = 1, used = p->nconfig; i < used; ++i) { 378 if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id)) 379 mod_deflate_merge_config(&p->conf, p->cvlist + p->cvlist[i].v.u2[0]); 380 } 381 } 382 383 static short mod_deflate_encodings_to_flags(const array *encodings) { 384 short allowed_encodings = 0; 385 if (encodings->used) { 386 for (uint32_t j = 0; j < encodings->used; ++j) { 387 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) 388 data_string *ds = (data_string *)encodings->data[j]; 389 #endif 390 #ifdef USE_ZLIB 391 if (NULL != strstr(ds->value.ptr, "gzip")) 392 allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP 393 | HTTP_ACCEPT_ENCODING_X_GZIP; 394 if (NULL != strstr(ds->value.ptr, "x-gzip")) 395 allowed_encodings |= HTTP_ACCEPT_ENCODING_X_GZIP; 396 if (NULL != strstr(ds->value.ptr, "deflate")) 397 allowed_encodings |= HTTP_ACCEPT_ENCODING_DEFLATE; 398 /* 399 if (NULL != strstr(ds->value.ptr, "compress")) 400 allowed_encodings |= HTTP_ACCEPT_ENCODING_COMPRESS; 401 */ 402 #endif 403 #ifdef USE_BZ2LIB 404 if (NULL != strstr(ds->value.ptr, "bzip2")) 405 allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 406 | HTTP_ACCEPT_ENCODING_X_BZIP2; 407 if (NULL != strstr(ds->value.ptr, "x-bzip2")) 408 allowed_encodings |= HTTP_ACCEPT_ENCODING_X_BZIP2; 409 #endif 410 #ifdef USE_BROTLI 411 if (NULL != strstr(ds->value.ptr, "br")) 412 allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; 413 #endif 414 } 415 } 416 else { 417 /* default encodings */ 418 #ifdef USE_ZLIB 419 allowed_encodings |= HTTP_ACCEPT_ENCODING_GZIP 420 | HTTP_ACCEPT_ENCODING_X_GZIP 421 | HTTP_ACCEPT_ENCODING_DEFLATE; 422 #endif 423 #ifdef USE_BZ2LIB 424 allowed_encodings |= HTTP_ACCEPT_ENCODING_BZIP2 425 | HTTP_ACCEPT_ENCODING_X_BZIP2; 426 #endif 427 #ifdef USE_BROTLI 428 allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; 429 #endif 430 } 431 return allowed_encodings; 432 } 433 434 SETDEFAULTS_FUNC(mod_deflate_set_defaults) { 435 static const config_plugin_keys_t cpk[] = { 436 { CONST_STR_LEN("deflate.mimetypes"), 437 T_CONFIG_ARRAY_VLIST, 438 T_CONFIG_SCOPE_CONNECTION } 439 ,{ CONST_STR_LEN("deflate.allowed-encodings"), 440 T_CONFIG_ARRAY_VLIST, 441 T_CONFIG_SCOPE_CONNECTION } 442 ,{ CONST_STR_LEN("deflate.max-compress-size"), 443 T_CONFIG_INT, 444 T_CONFIG_SCOPE_CONNECTION } 445 ,{ CONST_STR_LEN("deflate.min-compress-size"), 446 T_CONFIG_SHORT, 447 T_CONFIG_SCOPE_CONNECTION } 448 ,{ CONST_STR_LEN("deflate.compression-level"), 449 T_CONFIG_SHORT, 450 T_CONFIG_SCOPE_CONNECTION } 451 ,{ CONST_STR_LEN("deflate.output-buffer-size"), 452 T_CONFIG_SHORT, 453 T_CONFIG_SCOPE_CONNECTION } 454 ,{ CONST_STR_LEN("deflate.work-block-size"), 455 T_CONFIG_SHORT, 456 T_CONFIG_SCOPE_CONNECTION } 457 ,{ CONST_STR_LEN("deflate.max-loadavg"), 458 T_CONFIG_STRING, 459 T_CONFIG_SCOPE_CONNECTION } 460 ,{ CONST_STR_LEN("deflate.cache-dir"), 461 T_CONFIG_STRING, 462 T_CONFIG_SCOPE_CONNECTION } 463 ,{ CONST_STR_LEN("compress.filetype"), 464 T_CONFIG_ARRAY_VLIST, 465 T_CONFIG_SCOPE_CONNECTION } 466 ,{ CONST_STR_LEN("compress.allowed-encodings"), 467 T_CONFIG_ARRAY_VLIST, 468 T_CONFIG_SCOPE_CONNECTION } 469 ,{ CONST_STR_LEN("compress.cache-dir"), 470 T_CONFIG_STRING, 471 T_CONFIG_SCOPE_CONNECTION } 472 ,{ CONST_STR_LEN("compress.max-filesize"), 473 T_CONFIG_INT, 474 T_CONFIG_SCOPE_CONNECTION } 475 ,{ CONST_STR_LEN("compress.max-loadavg"), 476 T_CONFIG_STRING, 477 T_CONFIG_SCOPE_CONNECTION } 478 ,{ NULL, 0, 479 T_CONFIG_UNSET, 480 T_CONFIG_SCOPE_UNSET } 481 }; 482 483 plugin_data * const p = p_d; 484 if (!config_plugin_values_init(srv, p, cpk, "mod_deflate")) 485 return HANDLER_ERROR; 486 487 /* process and validate config directives 488 * (init i to 0 if global context; to 1 to skip empty global context) */ 489 for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) { 490 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 491 for (; -1 != cpv->k_id; ++cpv) { 492 switch (cpv->k_id) { 493 case 0: /* deflate.mimetypes */ 494 /* mod_deflate matches mimetype as prefix of Content-Type 495 * so ignore '*' at end of mimetype for end-user flexibility 496 * in specifying trailing wildcard to grouping of mimetypes */ 497 for (uint32_t m = 0; m < cpv->v.a->used; ++m) { 498 buffer *mimetype=&((data_string *)cpv->v.a->data[m])->value; 499 size_t len = buffer_string_length(mimetype); 500 if (len > 2 && mimetype->ptr[len-1] == '*') 501 buffer_string_set_length(mimetype, len-1); 502 } 503 if (0 == cpv->v.a->used) cpv->v.a = NULL; 504 break; 505 case 1: /* deflate.allowed-encodings */ 506 cpv->v.shrt = (unsigned short) 507 mod_deflate_encodings_to_flags(cpv->v.a); 508 cpv->vtype = T_CONFIG_SHORT; 509 break; 510 case 2: /* deflate.max-compress-size */ 511 case 3: /* deflate.min-compress-size */ 512 break; 513 case 4: /* deflate.compression-level */ 514 if ((cpv->v.shrt < 1 || cpv->v.shrt > 9) 515 && *(short *)&cpv->v.shrt != -1) { 516 log_error(srv->errh, __FILE__, __LINE__, 517 "compression-level must be between 1 and 9: %hu", 518 cpv->v.shrt); 519 return HANDLER_ERROR; 520 } 521 break; 522 case 5: /* deflate.output-buffer-size */ 523 case 6: /* deflate.work-block-size */ 524 break; 525 case 7: /* deflate.max-loadavg */ 526 cpv->v.d = (!buffer_string_is_empty(cpv->v.b)) 527 ? strtod(cpv->v.b->ptr, NULL) 528 : 0.0; 529 break; 530 case 8: /* deflate.cache-dir */ 531 if (!buffer_string_is_empty(cpv->v.b)) { 532 buffer *b; 533 *(const buffer **)&b = cpv->v.b; 534 const uint32_t len = buffer_string_length(b); 535 if (len > 0 && '/' == b->ptr[len-1]) 536 buffer_string_set_length(b, len-1); /*remove end slash*/ 537 struct stat st; 538 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 539 log_perror(srv->errh, __FILE__, __LINE__, 540 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 541 return HANDLER_ERROR; 542 } 543 } 544 break; 545 case 9: /* compress.filetype */ 546 log_error(srv->errh, __FILE__, __LINE__, 547 "DEPRECATED: %s replaced with deflate.mimetypes", 548 cpk[cpv->k_id].k); 549 break; 550 case 10:/* compress.allowed-encodings */ 551 log_error(srv->errh, __FILE__, __LINE__, 552 "DEPRECATED: %s replaced with deflate.allowed-encodings", 553 cpk[cpv->k_id].k); 554 cpv->v.shrt = (unsigned short) 555 mod_deflate_encodings_to_flags(cpv->v.a); 556 cpv->vtype = T_CONFIG_SHORT; 557 break; 558 case 11:/* compress.cache-dir */ 559 log_error(srv->errh, __FILE__, __LINE__, 560 "DEPRECATED: %s replaced with deflate.cache-dir", 561 cpk[cpv->k_id].k); 562 if (!buffer_string_is_empty(cpv->v.b)) { 563 buffer *b; 564 *(const buffer **)&b = cpv->v.b; 565 const uint32_t len = buffer_string_length(b); 566 if (len > 0 && '/' == b->ptr[len-1]) 567 buffer_string_set_length(b, len-1); /*remove end slash*/ 568 struct stat st; 569 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 570 log_perror(srv->errh, __FILE__, __LINE__, 571 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 572 return HANDLER_ERROR; 573 } 574 } 575 break; 576 case 12:/* compress.max-filesize */ 577 log_error(srv->errh, __FILE__, __LINE__, 578 "DEPRECATED: %s replaced with deflate.max-compress-size", 579 cpk[cpv->k_id].k); 580 break; 581 case 13:/* compress.max-loadavg */ 582 log_error(srv->errh, __FILE__, __LINE__, 583 "DEPRECATED: %s replaced with deflate.max-loadavg", 584 cpk[cpv->k_id].k); 585 cpv->v.d = (!buffer_string_is_empty(cpv->v.b)) 586 ? strtod(cpv->v.b->ptr, NULL) 587 : 0.0; 588 break; 589 default:/* should not happen */ 590 break; 591 } 592 } 593 } 594 595 p->defaults.allowed_encodings = 0; 596 p->defaults.max_compress_size = 128*1024; /*(128 MB measured as num KB)*/ 597 p->defaults.min_compress_size = 256; 598 p->defaults.compression_level = -1; 599 p->defaults.output_buffer_size = 0; 600 p->defaults.work_block_size = 2048; 601 p->defaults.max_loadavg = 0.0; 602 p->defaults.sync_flush = 0; 603 604 /* initialize p->defaults from global config context */ 605 if (p->nconfig > 0 && p->cvlist->v.u2[1]) { 606 const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0]; 607 if (-1 != cpv->k_id) 608 mod_deflate_merge_config(&p->defaults, cpv); 609 } 610 611 return HANDLER_GO_ON; 612 } 613 614 615 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) 616 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { 617 ssize_t wr; 618 do { 619 wr = write(hctx->cache_fd, out, len); 620 } while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR); 621 return (0 == len) ? 0 : -1; 622 } 623 624 static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) { 625 if (0 == len) return 0; 626 return (-1 == hctx->cache_fd) 627 ? http_chunk_append_mem(hctx->r, out, len) 628 : mod_deflate_cache_file_append(hctx, out, len); 629 } 630 #endif 631 632 633 #ifdef USE_ZLIB 634 635 static int stream_deflate_init(handler_ctx *hctx) { 636 z_stream * const z = &hctx->u.z; 637 const plugin_data * const p = hctx->plugin_data; 638 z->zalloc = Z_NULL; 639 z->zfree = Z_NULL; 640 z->opaque = Z_NULL; 641 z->total_in = 0; 642 z->total_out = 0; 643 z->next_out = (unsigned char *)hctx->output->ptr; 644 z->avail_out = hctx->output->size; 645 646 if (Z_OK != deflateInit2(z, 647 p->conf.compression_level > 0 648 ? p->conf.compression_level 649 : Z_DEFAULT_COMPRESSION, 650 Z_DEFLATED, 651 (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 652 ? (MAX_WBITS | 16) /*(0x10 flags gzip header, trailer)*/ 653 : -MAX_WBITS, /*(negate to suppress zlib header)*/ 654 8, /* default memLevel */ 655 Z_DEFAULT_STRATEGY)) { 656 return -1; 657 } 658 659 return 0; 660 } 661 662 static int stream_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 663 z_stream * const z = &(hctx->u.z); 664 size_t len; 665 666 z->next_in = start; 667 z->avail_in = st_size; 668 hctx->bytes_in += st_size; 669 670 /* compress data */ 671 do { 672 if (Z_OK != deflate(z, Z_NO_FLUSH)) return -1; 673 674 if (z->avail_out == 0 || z->avail_in > 0) { 675 len = hctx->output->size - z->avail_out; 676 hctx->bytes_out += len; 677 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 678 return -1; 679 z->next_out = (unsigned char *)hctx->output->ptr; 680 z->avail_out = hctx->output->size; 681 } 682 } while (z->avail_in > 0); 683 684 return 0; 685 } 686 687 static int stream_deflate_flush(handler_ctx * const hctx, int end) { 688 z_stream * const z = &(hctx->u.z); 689 const plugin_data *p = hctx->plugin_data; 690 size_t len; 691 int rc = 0; 692 int done; 693 694 /* compress data */ 695 do { 696 done = 1; 697 if (end) { 698 rc = deflate(z, Z_FINISH); 699 if (rc == Z_OK) { 700 done = 0; 701 } else if (rc != Z_STREAM_END) { 702 return -1; 703 } 704 } else { 705 if (p->conf.sync_flush) { 706 rc = deflate(z, Z_SYNC_FLUSH); 707 if (rc != Z_OK) return -1; 708 } else if (z->avail_in > 0) { 709 rc = deflate(z, Z_NO_FLUSH); 710 if (rc != Z_OK) return -1; 711 } 712 } 713 714 len = hctx->output->size - z->avail_out; 715 if (z->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 716 hctx->bytes_out += len; 717 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 718 return -1; 719 z->next_out = (unsigned char *)hctx->output->ptr; 720 z->avail_out = hctx->output->size; 721 } 722 } while (z->avail_in != 0 || !done); 723 724 return 0; 725 } 726 727 static int stream_deflate_end(handler_ctx *hctx) { 728 z_stream * const z = &(hctx->u.z); 729 int rc = deflateEnd(z); 730 if (Z_OK == rc || Z_DATA_ERROR == rc) return 0; 731 732 if (z->msg != NULL) { 733 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 734 "deflateEnd error ret=%d, msg=%s", rc, z->msg); 735 } else { 736 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 737 "deflateEnd error ret=%d", rc); 738 } 739 return -1; 740 } 741 742 #endif 743 744 745 #ifdef USE_BZ2LIB 746 747 static int stream_bzip2_init(handler_ctx *hctx) { 748 bz_stream * const bz = &hctx->u.bz; 749 const plugin_data * const p = hctx->plugin_data; 750 bz->bzalloc = NULL; 751 bz->bzfree = NULL; 752 bz->opaque = NULL; 753 bz->total_in_lo32 = 0; 754 bz->total_in_hi32 = 0; 755 bz->total_out_lo32 = 0; 756 bz->total_out_hi32 = 0; 757 bz->next_out = hctx->output->ptr; 758 bz->avail_out = hctx->output->size; 759 760 if (BZ_OK != BZ2_bzCompressInit(bz, 761 p->conf.compression_level > 0 762 ? p->conf.compression_level 763 : 9, /* blocksize = 900k */ 764 0, /* verbosity */ 765 0)) { /* workFactor: default */ 766 return -1; 767 } 768 769 return 0; 770 } 771 772 static int stream_bzip2_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 773 bz_stream * const bz = &(hctx->u.bz); 774 size_t len; 775 776 bz->next_in = (char *)start; 777 bz->avail_in = st_size; 778 hctx->bytes_in += st_size; 779 780 /* compress data */ 781 do { 782 if (BZ_RUN_OK != BZ2_bzCompress(bz, BZ_RUN)) return -1; 783 784 if (bz->avail_out == 0 || bz->avail_in > 0) { 785 len = hctx->output->size - bz->avail_out; 786 hctx->bytes_out += len; 787 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 788 return -1; 789 bz->next_out = hctx->output->ptr; 790 bz->avail_out = hctx->output->size; 791 } 792 } while (bz->avail_in > 0); 793 794 return 0; 795 } 796 797 static int stream_bzip2_flush(handler_ctx * const hctx, int end) { 798 bz_stream * const bz = &(hctx->u.bz); 799 const plugin_data *p = hctx->plugin_data; 800 size_t len; 801 int rc; 802 int done; 803 804 /* compress data */ 805 do { 806 done = 1; 807 if (end) { 808 rc = BZ2_bzCompress(bz, BZ_FINISH); 809 if (rc == BZ_FINISH_OK) { 810 done = 0; 811 } else if (rc != BZ_STREAM_END) { 812 return -1; 813 } 814 } else if (bz->avail_in > 0) { 815 /* p->conf.sync_flush not implemented here, 816 * which would loop on BZ_FLUSH while BZ_FLUSH_OK 817 * until BZ_RUN_OK returned */ 818 rc = BZ2_bzCompress(bz, BZ_RUN); 819 if (rc != BZ_RUN_OK) { 820 return -1; 821 } 822 } 823 824 len = hctx->output->size - bz->avail_out; 825 if (bz->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 826 hctx->bytes_out += len; 827 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 828 return -1; 829 bz->next_out = hctx->output->ptr; 830 bz->avail_out = hctx->output->size; 831 } 832 } while (bz->avail_in != 0 || !done); 833 834 return 0; 835 } 836 837 static int stream_bzip2_end(handler_ctx *hctx) { 838 bz_stream * const bz = &(hctx->u.bz); 839 int rc = BZ2_bzCompressEnd(bz); 840 if (BZ_OK == rc || BZ_DATA_ERROR == rc) return 0; 841 842 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 843 "BZ2_bzCompressEnd error ret=%d", rc); 844 return -1; 845 } 846 847 #endif 848 849 850 #ifdef USE_BROTLI 851 852 static int stream_br_init(handler_ctx *hctx) { 853 BrotliEncoderState * const br = hctx->u.br = 854 BrotliEncoderCreateInstance(NULL, NULL, NULL); 855 if (NULL == br) return -1; 856 857 /* future: consider allowing tunables by encoder algorithm, 858 * (i.e. not generic "compression_level") */ 859 /*(note: we ignore any errors while tuning parameters here)*/ 860 const plugin_data * const p = hctx->plugin_data; 861 if (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */ 862 BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY, 863 p->conf.compression_level); 864 865 /* XXX: is this worth checking? 866 * BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */ 867 const buffer *vb = 868 http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE, 869 CONST_STR_LEN("Content-Type")); 870 if (NULL != vb) { 871 if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1) 872 || (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1) 873 && (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1) 874 || 0 == strncmp(vb->ptr+12,"ld+json", sizeof("ld+json")-1) 875 || 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1) 876 || 0 == strncmp(vb->ptr+12,"xhtml+xml", sizeof("xhtml+xml")-1) 877 || 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1))) 878 || 0 == strncmp(vb->ptr, "image/svg+xml", sizeof("image/svg+xml")-1)) 879 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); 880 else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1)) 881 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT); 882 } 883 884 return 0; 885 } 886 887 static int stream_br_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 888 const uint8_t *in = (uint8_t *)start; 889 BrotliEncoderState * const br = hctx->u.br; 890 hctx->bytes_in += st_size; 891 while (st_size || BrotliEncoderHasMoreOutput(br)) { 892 size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size) 893 ? (size_t)st_size 894 : ((~(uint32_t)0) >> 1); 895 size_t outsz = 0; 896 BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS, 897 &insz, &in, &outsz, NULL, NULL); 898 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 899 st_size -= (st_size - (off_t)insz); 900 if (outsz) { 901 hctx->bytes_out += (off_t)outsz; 902 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 903 return -1; 904 } 905 } 906 return 0; 907 } 908 909 static int stream_br_flush(handler_ctx * const hctx, int end) { 910 const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; 911 BrotliEncoderState * const br = hctx->u.br; 912 do { 913 size_t insz = 0; 914 size_t outsz = 0; 915 BrotliEncoderCompressStream(br, brmode, 916 &insz, NULL, &outsz, NULL, NULL); 917 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 918 if (outsz) { 919 hctx->bytes_out += (off_t)outsz; 920 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 921 return -1; 922 } 923 } while (BrotliEncoderHasMoreOutput(br)); 924 return 0; 925 } 926 927 static int stream_br_end(handler_ctx *hctx) { 928 BrotliEncoderState * const br = hctx->u.br; 929 BrotliEncoderDestroyInstance(br); 930 return 0; 931 } 932 933 #endif 934 935 936 static int mod_deflate_stream_init(handler_ctx *hctx) { 937 switch(hctx->compression_type) { 938 #ifdef USE_ZLIB 939 case HTTP_ACCEPT_ENCODING_GZIP: 940 case HTTP_ACCEPT_ENCODING_DEFLATE: 941 return stream_deflate_init(hctx); 942 #endif 943 #ifdef USE_BZ2LIB 944 case HTTP_ACCEPT_ENCODING_BZIP2: 945 return stream_bzip2_init(hctx); 946 #endif 947 #ifdef USE_BROTLI 948 case HTTP_ACCEPT_ENCODING_BR: 949 return stream_br_init(hctx); 950 #endif 951 default: 952 return -1; 953 } 954 } 955 956 static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 957 if (0 == st_size) return 0; 958 switch(hctx->compression_type) { 959 #ifdef USE_ZLIB 960 case HTTP_ACCEPT_ENCODING_GZIP: 961 case HTTP_ACCEPT_ENCODING_DEFLATE: 962 return stream_deflate_compress(hctx, start, st_size); 963 #endif 964 #ifdef USE_BZ2LIB 965 case HTTP_ACCEPT_ENCODING_BZIP2: 966 return stream_bzip2_compress(hctx, start, st_size); 967 #endif 968 #ifdef USE_BROTLI 969 case HTTP_ACCEPT_ENCODING_BR: 970 return stream_br_compress(hctx, start, st_size); 971 #endif 972 default: 973 UNUSED(start); 974 return -1; 975 } 976 } 977 978 static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { 979 if (0 == hctx->bytes_in) return 0; 980 switch(hctx->compression_type) { 981 #ifdef USE_ZLIB 982 case HTTP_ACCEPT_ENCODING_GZIP: 983 case HTTP_ACCEPT_ENCODING_DEFLATE: 984 return stream_deflate_flush(hctx, end); 985 #endif 986 #ifdef USE_BZ2LIB 987 case HTTP_ACCEPT_ENCODING_BZIP2: 988 return stream_bzip2_flush(hctx, end); 989 #endif 990 #ifdef USE_BROTLI 991 case HTTP_ACCEPT_ENCODING_BR: 992 return stream_br_flush(hctx, end); 993 #endif 994 default: 995 UNUSED(end); 996 return -1; 997 } 998 } 999 1000 static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) { 1001 /* store compression ratio in environment 1002 * for possible logging by mod_accesslog 1003 * (late in response handling, so not seen by most other modules) */ 1004 /*(should be called only at end of successful response compression)*/ 1005 char ratio[LI_ITOSTRING_LENGTH]; 1006 if (0 == bytes_in) return; 1007 size_t len = 1008 li_itostrn(ratio, sizeof(ratio), bytes_out * 100 / bytes_in); 1009 http_header_env_set(r, CONST_STR_LEN("ratio"), ratio, len); 1010 } 1011 1012 static int mod_deflate_stream_end(handler_ctx *hctx) { 1013 switch(hctx->compression_type) { 1014 #ifdef USE_ZLIB 1015 case HTTP_ACCEPT_ENCODING_GZIP: 1016 case HTTP_ACCEPT_ENCODING_DEFLATE: 1017 return stream_deflate_end(hctx); 1018 #endif 1019 #ifdef USE_BZ2LIB 1020 case HTTP_ACCEPT_ENCODING_BZIP2: 1021 return stream_bzip2_end(hctx); 1022 #endif 1023 #ifdef USE_BROTLI 1024 case HTTP_ACCEPT_ENCODING_BR: 1025 return stream_br_end(hctx); 1026 #endif 1027 default: 1028 return -1; 1029 } 1030 } 1031 1032 static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) { 1033 int rc = mod_deflate_stream_end(hctx); 1034 1035 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */ 1036 if (0 == rc && hctx->bytes_in < hctx->bytes_out) 1037 log_error(r->conf.errh, __FILE__, __LINE__, 1038 "uri %s in=%lld smaller than out=%lld", r->target.ptr, 1039 (long long)hctx->bytes_in, (long long)hctx->bytes_out); 1040 #endif 1041 1042 handler_ctx_free(hctx); 1043 return rc; 1044 } 1045 1046 1047 static int mod_deflate_file_chunk(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t st_size) { 1048 off_t abs_offset; 1049 off_t toSend = -1; 1050 char *start; 1051 #ifdef USE_MMAP 1052 /* use large chunks since server blocks while compressing, anyway 1053 * (mod_deflate is not recommended for large files) */ 1054 const off_t mmap_chunk_size = 16 * 1024 * 1024; /* must be power-of-2 */ 1055 off_t we_want_to_send = st_size; 1056 volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */ 1057 #else 1058 start = NULL; 1059 #endif 1060 1061 if (-1 == c->file.fd) { /* open the file if not already open */ 1062 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) { 1063 log_perror(r->conf.errh, __FILE__, __LINE__, "open failed %s", c->mem->ptr); 1064 return -1; 1065 } 1066 } 1067 1068 abs_offset = c->offset; 1069 1070 #ifdef USE_MMAP 1071 /* mmap the buffer 1072 * - first mmap 1073 * - new mmap as the we are at the end of the last one */ 1074 if (c->file.mmap.start == MAP_FAILED || 1075 abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) { 1076 1077 /* Optimizations for the future: 1078 * 1079 * (comment is outdated since server blocks compressing file and then munmap()s file) 1080 * 1081 * adaptive mem-mapping 1082 * the problem: 1083 * we mmap() the whole file. If someone has a lot of large files and 32bit 1084 * machine the virtual address area will be exhausted and we will have a failing 1085 * mmap() call. 1086 * solution: 1087 * only mmap 16M in one chunk and move the window as soon as we have finished 1088 * the first 8M 1089 * 1090 * read-ahead buffering 1091 * the problem: 1092 * sending out several large files in parallel trashes the read-ahead of the 1093 * kernel leading to long wait-for-seek times. 1094 * solutions: (increasing complexity) 1095 * 1. use madvise 1096 * 2. use a internal read-ahead buffer in the chunk-structure 1097 * 3. use non-blocking IO for file-transfers 1098 * */ 1099 1100 /* all mmap()ed areas are mmap_chunk_size except the last which might be smaller */ 1101 off_t to_mmap; 1102 1103 /* this is a remap, move the mmap-offset */ 1104 if (c->file.mmap.start != MAP_FAILED) { 1105 munmap(c->file.mmap.start, c->file.mmap.length); 1106 c->file.mmap.offset += mmap_chunk_size; 1107 } else { 1108 /* in case the range-offset is after the first mmap()ed area we skip the area */ 1109 c->file.mmap.offset = c->offset & ~(mmap_chunk_size-1); 1110 } 1111 1112 to_mmap = c->file.length - c->file.mmap.offset; 1113 if (to_mmap > mmap_chunk_size) to_mmap = mmap_chunk_size; 1114 /* we have more to send than we can mmap() at once */ 1115 if (we_want_to_send > to_mmap) we_want_to_send = to_mmap; 1116 1117 if (MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset)) 1118 && (errno != EINVAL || MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset)))) { 1119 log_perror(r->conf.errh, __FILE__, __LINE__, 1120 "mmap failed %s %d", c->mem->ptr, c->file.fd); 1121 1122 return -1; 1123 } 1124 1125 c->file.mmap.length = to_mmap; 1126 #ifdef HAVE_MADVISE 1127 /* don't advise files <= 64Kb */ 1128 if (c->file.mmap.length > 65536 && 1129 0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) { 1130 log_perror(r->conf.errh, __FILE__, __LINE__, 1131 "madvise failed %s %d", c->mem->ptr, c->file.fd); 1132 } 1133 #endif 1134 1135 /* chunk_reset() or chunk_free() will cleanup for us */ 1136 } 1137 1138 /* to_send = abs_mmap_end - abs_offset */ 1139 toSend = (c->file.mmap.offset + c->file.mmap.length) - abs_offset; 1140 if (toSend > we_want_to_send) toSend = we_want_to_send; 1141 1142 if (toSend < 0) { 1143 log_error(r->conf.errh, __FILE__, __LINE__, 1144 "toSend is negative: %lld %lld %lld %lld", 1145 (long long)toSend, 1146 (long long)c->file.mmap.length, 1147 (long long)abs_offset, 1148 (long long)c->file.mmap.offset); 1149 force_assert(toSend < 0); 1150 } 1151 1152 start = c->file.mmap.start; 1153 mapped = 1; 1154 #endif 1155 1156 if (MAP_FAILED == c->file.mmap.start) { 1157 toSend = st_size; 1158 if (toSend > 2*1024*1024) toSend = 2*1024*1024; 1159 if (NULL == (start = malloc((size_t)toSend)) || -1 == lseek(c->file.fd, abs_offset, SEEK_SET) || toSend != read(c->file.fd, start, (size_t)toSend)) { 1160 log_perror(r->conf.errh, __FILE__, __LINE__, "reading %s failed", c->mem->ptr); 1161 1162 free(start); 1163 return -1; 1164 } 1165 abs_offset = 0; 1166 } 1167 1168 #ifdef USE_MMAP 1169 if (mapped) { 1170 signal(SIGBUS, sigbus_handler); 1171 sigbus_jmp_valid = 1; 1172 if (0 != sigsetjmp(sigbus_jmp, 1)) { 1173 sigbus_jmp_valid = 0; 1174 1175 log_error(r->conf.errh, __FILE__, __LINE__, 1176 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1177 return -1; 1178 } 1179 } 1180 #endif 1181 1182 if (mod_deflate_compress(hctx, 1183 (unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) { 1184 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1185 toSend = -1; 1186 } 1187 1188 #ifdef USE_MMAP 1189 if (mapped) 1190 sigbus_jmp_valid = 0; 1191 else 1192 #endif 1193 free(start); 1194 1195 return toSend; 1196 } 1197 1198 1199 static handler_t deflate_compress_response(request_st * const r, handler_ctx * const hctx) { 1200 off_t len, max; 1201 int close_stream; 1202 1203 /* move all chunk from write_queue into our in_queue, then adjust 1204 * counters since r->write_queue is reused for compressed output */ 1205 chunkqueue * const cq = &r->write_queue; 1206 len = chunkqueue_length(cq); 1207 chunkqueue_remove_finished_chunks(cq); 1208 chunkqueue_append_chunkqueue(&hctx->in_queue, cq); 1209 cq->bytes_in -= len; 1210 cq->bytes_out -= len; 1211 1212 max = chunkqueue_length(&hctx->in_queue); 1213 #if 0 1214 /* calculate max bytes to compress for this call */ 1215 if (p->conf.sync_flush && max > (len = p->conf.work_block_size << 10)) { 1216 max = len; 1217 } 1218 #endif 1219 1220 /* Compress chunks from in_queue into chunks for write_queue */ 1221 while (max) { 1222 chunk *c = hctx->in_queue.first; 1223 1224 switch(c->type) { 1225 case MEM_CHUNK: 1226 len = buffer_string_length(c->mem) - c->offset; 1227 if (len > max) len = max; 1228 if (mod_deflate_compress(hctx, (unsigned char *)c->mem->ptr+c->offset, len) < 0) { 1229 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1230 return HANDLER_ERROR; 1231 } 1232 break; 1233 case FILE_CHUNK: 1234 len = c->file.length - c->offset; 1235 if (len > max) len = max; 1236 if ((len = mod_deflate_file_chunk(r, hctx, c, len)) < 0) { 1237 log_error(r->conf.errh, __FILE__, __LINE__, "compress file chunk failed."); 1238 return HANDLER_ERROR; 1239 } 1240 break; 1241 default: 1242 log_error(r->conf.errh, __FILE__, __LINE__, "%d type not known", c->type); 1243 return HANDLER_ERROR; 1244 } 1245 1246 max -= len; 1247 chunkqueue_mark_written(&hctx->in_queue, len); 1248 } 1249 1250 /*(currently should always be true)*/ 1251 /*(current implementation requires response be complete)*/ 1252 close_stream = (r->resp_body_finished 1253 && chunkqueue_is_empty(&hctx->in_queue)); 1254 if (mod_deflate_stream_flush(hctx, close_stream) < 0) { 1255 log_error(r->conf.errh, __FILE__, __LINE__, "flush error"); 1256 return HANDLER_ERROR; 1257 } 1258 1259 return close_stream ? HANDLER_FINISHED : HANDLER_GO_ON; 1260 } 1261 1262 1263 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { 1264 /* get client side support encodings */ 1265 int accept_encoding = 0; 1266 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) 1267 UNUSED(value); 1268 UNUSED(label); 1269 #else 1270 for (; *value; ++value) { 1271 const char *v; 1272 while (*value == ' ' || *value == ',') ++value; 1273 v = value; 1274 while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0') 1275 ++value; 1276 switch (value - v) { 1277 case 2: 1278 #ifdef USE_BROTLI 1279 if (0 == memcmp(v, "br", 2)) 1280 accept_encoding |= HTTP_ACCEPT_ENCODING_BR; 1281 #endif 1282 break; 1283 case 4: 1284 #ifdef USE_ZLIB 1285 if (0 == memcmp(v, "gzip", 4)) 1286 accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; 1287 #endif 1288 break; 1289 case 5: 1290 #ifdef USE_BZ2LIB 1291 if (0 == memcmp(v, "bzip2", 5)) 1292 accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; 1293 #endif 1294 break; 1295 case 6: 1296 #ifdef USE_ZLIB 1297 if (0 == memcmp(v, "x-gzip", 6)) 1298 accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP; 1299 #endif 1300 break; 1301 case 7: 1302 #ifdef USE_ZLIB 1303 if (0 == memcmp(v, "deflate", 7)) 1304 accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; 1305 #endif 1306 #ifdef USE_BZ2LIB 1307 if (0 == memcmp(v, "x-bzip2", 7)) 1308 accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2; 1309 #endif 1310 break; 1311 #if 0 1312 case 8: 1313 if (0 == memcmp(v, "identity", 8)) 1314 accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; 1315 else if (0 == memcmp(v, "compress", 8)) 1316 accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; 1317 break; 1318 #endif 1319 default: 1320 break; 1321 } 1322 if (*value == ';') { 1323 while (*value != ',' && *value != '\0') ++value; 1324 } 1325 if (*value == '\0') break; 1326 } 1327 #endif 1328 1329 /* mask to limit to allowed_encodings */ 1330 accept_encoding &= p->conf.allowed_encodings; 1331 1332 /* select best matching encoding */ 1333 #ifdef USE_BROTLI 1334 if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { 1335 *label = "br"; 1336 return HTTP_ACCEPT_ENCODING_BR; 1337 } else 1338 #endif 1339 #ifdef USE_BZ2LIB 1340 if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) { 1341 *label = "bzip2"; 1342 return HTTP_ACCEPT_ENCODING_BZIP2; 1343 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_BZIP2) { 1344 *label = "x-bzip2"; 1345 return HTTP_ACCEPT_ENCODING_BZIP2; 1346 } else 1347 #endif 1348 #ifdef USE_ZLIB 1349 if (accept_encoding & HTTP_ACCEPT_ENCODING_GZIP) { 1350 *label = "gzip"; 1351 return HTTP_ACCEPT_ENCODING_GZIP; 1352 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_GZIP) { 1353 *label = "x-gzip"; 1354 return HTTP_ACCEPT_ENCODING_GZIP; 1355 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_DEFLATE) { 1356 *label = "deflate"; 1357 return HTTP_ACCEPT_ENCODING_DEFLATE; 1358 } else 1359 #endif 1360 if (0 == accept_encoding) { 1361 return 0; 1362 } else { 1363 return 0; 1364 } 1365 } 1366 1367 REQUEST_FUNC(mod_deflate_handle_response_start) { 1368 plugin_data *p = p_d; 1369 const buffer *vbro; 1370 buffer *vb; 1371 handler_ctx *hctx; 1372 const char *label; 1373 off_t len; 1374 uint32_t etaglen; 1375 int compression_type; 1376 handler_t rc; 1377 int had_vary = 0; 1378 1379 /*(current implementation requires response be complete)*/ 1380 if (!r->resp_body_finished) return HANDLER_GO_ON; 1381 if (r->http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; 1382 if (light_btst(r->resp_htags, HTTP_HEADER_TRANSFER_ENCODING)) 1383 return HANDLER_GO_ON; 1384 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_ENCODING)) 1385 return HANDLER_GO_ON; 1386 1387 /* disable compression for some http status types. */ 1388 switch(r->http_status) { 1389 case 100: 1390 case 101: 1391 case 204: 1392 case 205: 1393 case 304: 1394 /* disable compression as we have no response entity */ 1395 return HANDLER_GO_ON; 1396 default: 1397 break; 1398 } 1399 1400 mod_deflate_patch_config(r, p); 1401 1402 /* check if deflate configured for any mimetypes */ 1403 if (NULL == p->conf.mimetypes) return HANDLER_GO_ON; 1404 1405 /* check if size of response is below min-compress-size or exceeds max*/ 1406 /* (r->resp_body_finished checked at top of routine) */ 1407 len = chunkqueue_length(&r->write_queue); 1408 if (len <= (off_t)p->conf.min_compress_size) return HANDLER_GO_ON; 1409 if (p->conf.max_compress_size /*(max_compress_size in KB)*/ 1410 && len > ((off_t)p->conf.max_compress_size << 10)) { 1411 return HANDLER_GO_ON; 1412 } 1413 1414 /* Check Accept-Encoding for supported encoding. */ 1415 vbro = http_header_request_get(r, HTTP_HEADER_ACCEPT_ENCODING, CONST_STR_LEN("Accept-Encoding")); 1416 if (NULL == vbro) return HANDLER_GO_ON; 1417 1418 /* find matching encodings */ 1419 compression_type = mod_deflate_choose_encoding(vbro->ptr, p, &label); 1420 if (!compression_type) return HANDLER_GO_ON; 1421 1422 /* Check mimetype in response header "Content-Type" */ 1423 if (NULL != (vbro = http_header_response_get(r, HTTP_HEADER_CONTENT_TYPE, CONST_STR_LEN("Content-Type")))) { 1424 if (NULL == array_match_value_prefix(p->conf.mimetypes, vbro)) return HANDLER_GO_ON; 1425 } else { 1426 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */ 1427 data_string *mimetype = (data_string *)p->conf.mimetypes->data[0]; 1428 if (!buffer_string_is_empty(&mimetype->value)) return HANDLER_GO_ON; 1429 } 1430 1431 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */ 1432 if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) { 1433 had_vary = 1; 1434 if (NULL == strstr(vb->ptr, "Accept-Encoding")) { 1435 buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding")); 1436 } 1437 } else { 1438 http_header_response_append(r, HTTP_HEADER_VARY, 1439 CONST_STR_LEN("Vary"), 1440 CONST_STR_LEN("Accept-Encoding")); 1441 } 1442 1443 /* check ETag as is done in http_response_handle_cachable() 1444 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */ 1445 vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag")); 1446 etaglen = (NULL != vb) ? buffer_string_length(vb) : 0; 1447 if (NULL != vb && light_btst(r->rqst_htags, HTTP_HEADER_IF_NONE_MATCH)) { 1448 const buffer *if_none_match = http_header_response_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match")); 1449 if (etaglen 1450 && r->http_status < 300 /*(want 2xx only)*/ 1451 && NULL != if_none_match 1452 && 0 == strncmp(if_none_match->ptr, vb->ptr, etaglen-1) 1453 && if_none_match->ptr[etaglen-1] == '-' 1454 && 0 == strncmp(if_none_match->ptr+etaglen, label, strlen(label))) { 1455 1456 if (http_method_get_or_head(r->http_method)) { 1457 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1458 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1459 buffer_append_string(vb, label); 1460 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1461 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1462 r->http_status = 304; 1463 } else { 1464 r->http_status = 412; 1465 } 1466 1467 /* response_start hook occurs after error docs have been handled. 1468 * For now, send back empty response body. 1469 * In the future, might extract the error doc code so that it 1470 * might be run again if response_start hooks return with 1471 * changed http_status and r->handler_module = NULL */ 1472 /* clear content length even if 304 since compressed length unknown */ 1473 http_response_body_clear(r, 0); 1474 1475 r->resp_body_finished = 1; 1476 r->handler_module = NULL; 1477 return HANDLER_GO_ON; 1478 } 1479 } 1480 1481 if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < r->con->srv->loadavg[0]) { 1482 return HANDLER_GO_ON; 1483 } 1484 1485 /* update ETag, if ETag response header is set */ 1486 if (etaglen) { 1487 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1488 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1489 buffer_append_string(vb, label); 1490 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1491 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1492 } 1493 1494 /* set Content-Encoding to show selected compression type */ 1495 http_header_response_set(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding"), label, strlen(label)); 1496 1497 /* clear Content-Length and r->write_queue if HTTP HEAD request 1498 * (alternatively, could return original Content-Length with HEAD 1499 * request if ETag not modified and Content-Encoding not added) */ 1500 if (HTTP_METHOD_HEAD == r->http_method) { 1501 /* ensure that uncompressed Content-Length is not sent in HEAD response */ 1502 http_response_body_clear(r, 0); 1503 return HANDLER_GO_ON; 1504 } 1505 1506 /* restrict items eligible for cache of compressed responses 1507 * (This module does not aim to be a full caching proxy) 1508 * response must be complete (not streaming response) 1509 * must not have prior Vary response header (before Accept-Encoding added) 1510 * must not have Range response header 1511 * must have ETag 1512 * must be file 1513 * must be single FILE_CHUNK in chunkqueue 1514 * must not be chunkqueue temporary file 1515 * must be whole file, not partial content 1516 * Note: small files (< 32k (see http_chunk.c)) will have been read into 1517 * memory (if streaming HTTP/1.1 chunked response) and will end up 1518 * getting stream-compressed rather than cached on disk as compressed 1519 * file 1520 */ 1521 buffer *tb = NULL; 1522 if (!buffer_is_empty(p->conf.cache_dir) 1523 && !had_vary 1524 && etaglen > 2 1525 && r->resp_body_finished 1526 && r->write_queue.first == r->write_queue.last 1527 && r->write_queue.first->type == FILE_CHUNK 1528 && r->write_queue.first->offset == 0 1529 && !r->write_queue.first->file.is_temp 1530 && !light_btst(r->resp_htags, HTTP_HEADER_RANGE)) { 1531 tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb); 1532 /*(checked earlier and skipped if Transfer-Encoding had been set)*/ 1533 stat_cache_entry *sce = stat_cache_get_entry_open(tb, 1); 1534 if (NULL != sce) { 1535 chunkqueue_reset(&r->write_queue); 1536 if (sce->fd < 0 || 0 != http_chunk_append_file_ref(r, sce)) 1537 return HANDLER_ERROR; 1538 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) 1539 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, 1540 CONST_STR_LEN("Content-Length")); 1541 mod_deflate_note_ratio(r, sce->st.st_size, len); 1542 return HANDLER_GO_ON; 1543 } 1544 /* sanity check that response was whole file; 1545 * (racy since using stat_cache, but cache file only if match) */ 1546 sce = stat_cache_get_entry(r->write_queue.first->mem); 1547 if (NULL == sce || sce->st.st_size != len) 1548 tb = NULL; 1549 else if (0 != mkdir_for_file(tb->ptr)) 1550 tb = NULL; 1551 } 1552 1553 /* enable compression */ 1554 p->conf.sync_flush = 1555 ((r->conf.stream_response_body 1556 & (FDEVENT_STREAM_RESPONSE | FDEVENT_STREAM_RESPONSE_BUFMIN)) 1557 && 0 == p->conf.output_buffer_size); 1558 hctx = handler_ctx_init(); 1559 hctx->plugin_data = p; 1560 hctx->compression_type = compression_type; 1561 hctx->r = r; 1562 /* setup output buffer */ 1563 buffer_clear(&p->tmp_buf); 1564 hctx->output = &p->tmp_buf; 1565 /* open cache file if caching compressed file */ 1566 if (tb) mod_deflate_cache_file_open(hctx, tb); 1567 if (0 != mod_deflate_stream_init(hctx)) { 1568 /*(should not happen unless ENOMEM)*/ 1569 handler_ctx_free(hctx); 1570 log_error(r->conf.errh, __FILE__, __LINE__, 1571 "Failed to initialize compression %s", label); 1572 /* restore prior Etag and unset Content-Encoding */ 1573 if (etaglen) { 1574 vb->ptr[etaglen-1] = '"'; /*(overwrite '-')*/ 1575 buffer_string_set_length(vb, etaglen); 1576 } 1577 http_header_response_unset(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding")); 1578 return HANDLER_GO_ON; 1579 } 1580 1581 #ifdef USE_BROTLI 1582 if (r->resp_body_finished 1583 && (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR) 1584 && (len >> 1) < (off_t)((~(uint32_t)0u) >> 1)) 1585 BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len); 1586 #endif 1587 1588 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) { 1589 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length")); 1590 } 1591 r->plugin_ctx[p->id] = hctx; 1592 1593 rc = deflate_compress_response(r, hctx); 1594 if (HANDLER_GO_ON == rc) return HANDLER_GO_ON; 1595 if (HANDLER_FINISHED == rc) { 1596 #ifdef __COVERITY__ 1597 /* coverity misses if hctx->cache_fd is not -1, then tb is not NULL */ 1598 force_assert(-1 == hctx->cache_fd || NULL != tb); 1599 #endif 1600 if (-1 == hctx->cache_fd 1601 || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) { 1602 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 1603 rc = HANDLER_GO_ON; 1604 } 1605 else 1606 rc = HANDLER_ERROR; 1607 } 1608 r->plugin_ctx[p->id] = NULL; 1609 if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR; 1610 return rc; 1611 } 1612 1613 static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) { 1614 plugin_data *p = p_d; 1615 handler_ctx *hctx = r->plugin_ctx[p->id]; 1616 1617 if (NULL != hctx) { 1618 r->plugin_ctx[p->id] = NULL; 1619 deflate_compress_cleanup(r, hctx); 1620 } 1621 1622 return HANDLER_GO_ON; 1623 } 1624 1625 int mod_deflate_plugin_init(plugin *p); 1626 int mod_deflate_plugin_init(plugin *p) { 1627 p->version = LIGHTTPD_VERSION_ID; 1628 p->name = "deflate"; 1629 1630 p->init = mod_deflate_init; 1631 p->cleanup = mod_deflate_free; 1632 p->set_defaults = mod_deflate_set_defaults; 1633 p->handle_request_reset = mod_deflate_cleanup; 1634 p->handle_response_start = mod_deflate_handle_response_start; 1635 1636 return 0; 1637 } 1638