1 /* mod_deflate 2 * 3 * 4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch 5 * 6 * Bug fix and new features: 7 * 1) fix loop bug when content-length is bigger than work-block-size*k 8 * 9 * ------- 10 * 11 * lighttpd-1.4.26.mod_deflate.patch from 12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate 13 * 14 * ------- 15 * 16 * Patch further modified in this incarnation. 17 * 18 * Note: this patch only handles completed responses 19 * (r->resp_body_finished) 20 * this patch does not currently handle streaming dynamic responses, 21 * and therefore also does not worry about Transfer-Encoding: chunked 22 * (or having separate con->output_queue for chunked-encoded output) 23 * (or using separate buffers per connection instead of p->tmp_buf) 24 * (or handling interactions with block buffering and write timeouts) 25 * 26 * Bug fix: 27 * - fixed major bug with compressing chunks with offset > 0 28 * x-ref: 29 * "Response breaking in mod_deflate" 30 * https://redmine.lighttpd.net/issues/986 31 * - fix broken (in some cases) chunk accounting in deflate_compress_response() 32 * - fix broken bzip2 33 * x-ref: 34 * "mod_deflate's bzip2 broken by default" 35 * https://redmine.lighttpd.net/issues/2035 36 * - fix mismatch with current chunk interfaces 37 * x-ref: 38 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)" 39 * https://redmine.lighttpd.net/issues/1510 40 * 41 * Behavior changes from prior patch: 42 * - deflate.mimetypes must now be configured to enable compression 43 * deflate.mimetypes = ( ) # compress nothing (disabled; default) 44 * deflate.mimetypes = ( "" ) # compress all mimetypes 45 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes 46 * x-ref: 47 * "mod_deflate enabled by default" 48 * https://redmine.lighttpd.net/issues/1394 49 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes) 50 * - deflate.debug removed (was developer debug trace, not end-user debug) 51 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress) 52 * x-ref: 53 * "mod_deflate should allow limiting of compression algorithm from the configuration file" 54 * https://redmine.lighttpd.net/issues/996 55 * "mod_compress disabling methods" 56 * https://redmine.lighttpd.net/issues/1773 57 * - deflate.nocompress-url removed since disabling compression for a URL 58 * can now easily be done by setting to a blank list either directive 59 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional 60 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) } 61 * - deflate.sync-flush removed; controlled by r->conf.stream_response_body 62 * (though streaming compression not currently implemented in mod_deflate) 63 * - inactive directives in this patch 64 * (since r->resp_body_finished required) 65 * deflate.work-block-size 66 * deflate.output-buffer-size 67 * - remove weak file size check; SIGBUS is trapped, file that shrink will error 68 * x-ref: 69 * "mod_deflate: filesize check is too weak" 70 * https://redmine.lighttpd.net/issues/1512 71 * - change default deflate.min-compress-size from 0 to now be 256 72 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits 73 * Apache 2.4 mod_deflate minimum is 68 bytes 74 * Akamai recommends minimum 860 bytes 75 * Google recommends minimum be somewhere in range between 150 and 1024 bytes 76 * - deflate.max-compress-size new directive (in kb like compress.max_filesize) 77 * - deflate.mem-level removed (too many knobs for little benefit) 78 * - deflate.window-size removed (too many knobs for little benefit) 79 * 80 * Future: 81 * - config directives may be changed, renamed, or removed 82 * e.g. A set of reasonable defaults might be chosen 83 * instead of making them configurable. 84 * deflate.min-compress-size 85 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes 86 * to avoid compressing, even if a broader deflate.mimetypes matched, 87 * e.g. to compress all "text/" except "text/special". 88 * 89 * Implementation notes: 90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer() 91 * so that p->tmp_buf can be large and re-used. This results in an extra copy 92 * of compressed data before data is sent to network, though if the compressed 93 * size is larger than 64k, it ends up being sent to a temporary file on 94 * disk without suffering an extra copy in memory, and without extra chunk 95 * create and destroy. If this is ever changed to give away buffers, then use 96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across 97 * multiple requests being handled in parallel. 98 */ 99 #include "first.h" 100 101 #include <sys/types.h> 102 #include <sys/stat.h> 103 #include "sys-mmap.h" 104 #include "sys-time.h" 105 106 #include <fcntl.h> 107 #include <stdlib.h> 108 #include <string.h> 109 #include <errno.h> 110 #include <unistd.h> /* getpid() read() unlink() write() */ 111 112 #include "base.h" 113 #include "ck.h" 114 #include "fdevent.h" 115 #include "log.h" 116 #include "buffer.h" 117 #include "http_chunk.h" 118 #include "http_etag.h" 119 #include "http_header.h" 120 #include "response.h" 121 #include "stat_cache.h" 122 123 #include "plugin.h" 124 125 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ 126 # define USE_ZLIB 127 # include <zlib.h> 128 #endif 129 #ifndef Z_DEFAULT_COMPRESSION 130 #define Z_DEFAULT_COMPRESSION -1 131 #endif 132 #ifndef MAX_WBITS 133 #define MAX_WBITS 15 134 #endif 135 136 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 137 # define USE_BZ2LIB 138 /* we don't need stdio interface */ 139 # define BZ_NO_STDIO 140 # include <bzlib.h> 141 #endif 142 143 #if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI 144 # define USE_BROTLI 145 # include <brotli/encode.h> 146 #endif 147 148 #if defined HAVE_ZSTD_H && defined HAVE_ZSTD 149 # define USE_ZSTD 150 # include <zstd.h> 151 #endif 152 153 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP 154 #define USE_MMAP 155 156 #include "sys-mmap.h" 157 #include <setjmp.h> 158 #include <signal.h> 159 160 static volatile int sigbus_jmp_valid; 161 static sigjmp_buf sigbus_jmp; 162 163 static void sigbus_handler(int sig) { 164 UNUSED(sig); 165 if (sigbus_jmp_valid) siglongjmp(sigbus_jmp, 1); 166 ck_bt_abort(__FILE__, __LINE__, "SIGBUS"); 167 } 168 #endif 169 170 /* request: accept-encoding */ 171 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) 172 #define HTTP_ACCEPT_ENCODING_GZIP BV(1) 173 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) 174 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) 175 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) 176 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) 177 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) 178 #define HTTP_ACCEPT_ENCODING_BR BV(7) 179 #define HTTP_ACCEPT_ENCODING_ZSTD BV(8) 180 181 typedef struct { 182 struct { 183 int clevel; /*(compression level)*/ 184 int windowBits; 185 int memLevel; 186 int strategy; 187 } gzip; 188 struct { 189 uint32_t quality; /*(compression level)*/ 190 uint32_t window; 191 uint32_t mode; 192 } brotli; 193 struct { 194 int clevel; /*(compression level)*/ 195 int strategy; 196 int windowLog; 197 } zstd; 198 struct { 199 int clevel; /*(compression level)*/ 200 } bzip2; 201 } encparms; 202 203 typedef struct { 204 const array *mimetypes; 205 const buffer *cache_dir; 206 unsigned int max_compress_size; 207 unsigned short min_compress_size; 208 unsigned short output_buffer_size; 209 unsigned short work_block_size; 210 unsigned short sync_flush; 211 short compression_level; 212 uint16_t * allowed_encodings; 213 double max_loadavg; 214 const encparms *params; 215 } plugin_config; 216 217 typedef struct { 218 PLUGIN_DATA; 219 plugin_config defaults; 220 plugin_config conf; 221 222 buffer tmp_buf; 223 } plugin_data; 224 225 typedef struct { 226 union { 227 #ifdef USE_ZLIB 228 z_stream z; 229 #endif 230 #ifdef USE_BZ2LIB 231 bz_stream bz; 232 #endif 233 #ifdef USE_BROTLI 234 BrotliEncoderState *br; 235 #endif 236 #ifdef USE_ZSTD 237 ZSTD_CStream *cctx; 238 #endif 239 int dummy; 240 } u; 241 off_t bytes_in; 242 off_t bytes_out; 243 buffer *output; 244 plugin_data *plugin_data; 245 request_st *r; 246 int compression_type; 247 int cache_fd; 248 char *cache_fn; 249 chunkqueue in_queue; 250 } handler_ctx; 251 252 static handler_ctx *handler_ctx_init() { 253 handler_ctx *hctx; 254 255 hctx = calloc(1, sizeof(*hctx)); 256 chunkqueue_init(&hctx->in_queue); 257 hctx->cache_fd = -1; 258 259 return hctx; 260 } 261 262 static void handler_ctx_free(handler_ctx *hctx) { 263 if (hctx->cache_fn) { 264 unlink(hctx->cache_fn); 265 free(hctx->cache_fn); 266 } 267 if (-1 != hctx->cache_fd) 268 close(hctx->cache_fd); 269 #if 0 270 if (hctx->output != &p->tmp_buf) { 271 buffer_free(hctx->output); 272 } 273 #endif 274 chunkqueue_reset(&hctx->in_queue); 275 free(hctx); 276 } 277 278 INIT_FUNC(mod_deflate_init) { 279 plugin_data * const p = calloc(1, sizeof(plugin_data)); 280 #ifdef USE_ZSTD 281 buffer_string_prepare_copy(&p->tmp_buf, ZSTD_CStreamOutSize()); 282 #else 283 buffer_string_prepare_copy(&p->tmp_buf, 65536); 284 #endif 285 return p; 286 } 287 288 FREE_FUNC(mod_deflate_free) { 289 plugin_data *p = p_d; 290 free(p->tmp_buf.ptr); 291 if (NULL == p->cvlist) return; 292 /* (init i to 0 if global context; to 1 to skip empty global context) */ 293 for (int i = !p->cvlist[0].v.u2[1], used = p->nconfig; i < used; ++i) { 294 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 295 for (; -1 != cpv->k_id; ++cpv) { 296 if (cpv->vtype != T_CONFIG_LOCAL || NULL == cpv->v.v) continue; 297 switch (cpv->k_id) { 298 case 1: /* deflate.allowed-encodings */ 299 case 14:/* deflate.params */ 300 free(cpv->v.v); 301 break; 302 default: 303 break; 304 } 305 } 306 } 307 } 308 309 #if defined(_WIN32) && !defined(__CYGWIN__) 310 #define mkdir(x,y) mkdir(x) 311 #endif 312 313 static int mkdir_for_file (char *fn) { 314 for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) { 315 if (p[1] == '\0') return 0; /* ignore trailing slash */ 316 *p = '\0'; 317 int rc = mkdir(fn, 0700); 318 *p = '/'; 319 if (0 != rc && errno != EEXIST) return -1; 320 } 321 return 0; 322 } 323 324 static int mkdir_recursive (char *dir) { 325 return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST) 326 ? 0 327 : -1; 328 } 329 330 static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) { 331 /* XXX: future: for shorter paths into the cache, we could checksum path, 332 * and then shard it to avoid a huge single directory. 333 * Alternatively, could use &r->uri.path, minus any 334 * (matching) &r->pathinfo suffix, with result url-encoded 335 * Alternative, we could shard etag which is already our "checksum" */ 336 buffer * const tb = r->tmp_buf; 337 buffer_copy_path_len2(tb, BUF_PTR_LEN(cache_dir), 338 BUF_PTR_LEN(&r->physical.path)); 339 buffer_append_str2(tb, CONST_STR_LEN("-"), /*(strip surrounding '"')*/ 340 etag->ptr+1, buffer_clen(etag)-2); 341 return tb; 342 } 343 344 static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) { 345 /* race exists whereby up to # workers might attempt to compress same 346 * file at same time if requested at same time, but this is unlikely 347 * and resolves itself by atomic rename into place when done */ 348 const uint32_t fnlen = buffer_clen(fn); 349 hctx->cache_fn = malloc(fnlen+1+LI_ITOSTRING_LENGTH+1); 350 force_assert(hctx->cache_fn); 351 memcpy(hctx->cache_fn, fn->ptr, fnlen); 352 hctx->cache_fn[fnlen] = '.'; 353 const size_t ilen = 354 li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid()); 355 hctx->cache_fn[fnlen+1+ilen] = '\0'; 356 hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600); 357 if (-1 == hctx->cache_fd) { 358 free(hctx->cache_fn); 359 hctx->cache_fn = NULL; 360 } 361 } 362 363 static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) { 364 if (0 != fdevent_rename(hctx->cache_fn, fn->ptr)) 365 return -1; 366 free(hctx->cache_fn); 367 hctx->cache_fn = NULL; 368 chunkqueue_reset(&r->write_queue); 369 int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out); 370 hctx->cache_fd = -1; 371 return rc; 372 } 373 374 static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) { 375 switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */ 376 case 0: /* deflate.mimetypes */ 377 pconf->mimetypes = cpv->v.a; 378 break; 379 case 1: /* deflate.allowed-encodings */ 380 if (cpv->vtype == T_CONFIG_LOCAL) 381 pconf->allowed_encodings = cpv->v.v; 382 break; 383 case 2: /* deflate.max-compress-size */ 384 pconf->max_compress_size = cpv->v.u; 385 break; 386 case 3: /* deflate.min-compress-size */ 387 pconf->min_compress_size = cpv->v.shrt; 388 break; 389 case 4: /* deflate.compression-level */ 390 pconf->compression_level = (short)cpv->v.shrt; 391 break; 392 case 5: /* deflate.output-buffer-size */ 393 pconf->output_buffer_size = cpv->v.shrt; 394 break; 395 case 6: /* deflate.work-block-size */ 396 pconf->work_block_size = cpv->v.shrt; 397 break; 398 case 7: /* deflate.max-loadavg */ 399 pconf->max_loadavg = cpv->v.d; 400 break; 401 case 8: /* deflate.cache-dir */ 402 pconf->cache_dir = cpv->v.b; 403 break; 404 #if 0 /*(cpv->k_id remapped in mod_deflate_set_defaults())*/ 405 case 9: /* compress.filetype */ 406 pconf->mimetypes = cpv->v.a; 407 break; 408 case 10:/* compress.allowed-encodings */ 409 if (cpv->vtype == T_CONFIG_LOCAL) 410 pconf->allowed_encodings = cpv->v.v; 411 break; 412 case 11:/* compress.cache-dir */ 413 pconf->cache_dir = cpv->v.b; 414 break; 415 case 12:/* compress.max-filesize */ 416 pconf->max_compress_size = cpv->v.u; 417 break; 418 case 13:/* compress.max-loadavg */ 419 pconf->max_loadavg = cpv->v.d; 420 break; 421 #endif 422 case 14:/* deflate.params */ 423 if (cpv->vtype == T_CONFIG_LOCAL) 424 pconf->params = cpv->v.v; 425 break; 426 default:/* should not happen */ 427 return; 428 } 429 } 430 431 static void mod_deflate_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) { 432 do { 433 mod_deflate_merge_config_cpv(pconf, cpv); 434 } while ((++cpv)->k_id != -1); 435 } 436 437 static void mod_deflate_patch_config(request_st * const r, plugin_data * const p) { 438 memcpy(&p->conf, &p->defaults, sizeof(plugin_config)); 439 for (int i = 1, used = p->nconfig; i < used; ++i) { 440 if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id)) 441 mod_deflate_merge_config(&p->conf, p->cvlist + p->cvlist[i].v.u2[0]); 442 } 443 } 444 445 static encparms * mod_deflate_parse_params(const array * const a, log_error_st * const errh) { 446 encparms * params = calloc(1, sizeof(encparms)); 447 force_assert(params); 448 449 /* set defaults */ 450 #ifdef USE_ZLIB 451 params->gzip.clevel = 0; /*(unset)*/ 452 params->gzip.windowBits = MAX_WBITS; 453 params->gzip.memLevel = 8; 454 params->gzip.strategy = Z_DEFAULT_STRATEGY; 455 #endif 456 #ifdef USE_BROTLI 457 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 458 params->brotli.quality = 5; 459 params->brotli.window = BROTLI_DEFAULT_WINDOW; 460 params->brotli.mode = BROTLI_MODE_GENERIC; 461 #endif 462 #ifdef USE_ZSTD 463 params->zstd.clevel = ZSTD_CLEVEL_DEFAULT; 464 params->zstd.strategy = 0; /*(use default strategy)*/ 465 params->zstd.windowLog = 0;/*(use default windowLog)*/ 466 #endif 467 #ifdef USE_BZ2LIB 468 params->bzip2.clevel = 0; /*(unset)*/ 469 #endif 470 471 for (uint32_t i = 0; i < a->used; ++i) { 472 const data_unset * const du = a->data[i]; 473 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 474 || defined(USE_ZSTD) 475 int32_t v = config_plugin_value_to_int32(du, -1); 476 #endif 477 #ifdef USE_BROTLI 478 if (buffer_eq_icase_slen(&du->key, 479 CONST_STR_LEN("BROTLI_PARAM_QUALITY"))) { 480 /*(future: could check for string and then look for and translate 481 * BROTLI_DEFAULT_QUALITY BROTLI_MIN_QUALITY BROTLI_MAX_QUALITY)*/ 482 if (BROTLI_MIN_QUALITY <= v && v <= BROTLI_MAX_QUALITY) 483 params->brotli.quality = (uint32_t)v; /* 0 .. 11 */ 484 else 485 log_error(errh, __FILE__, __LINE__, 486 "invalid value for BROTLI_PARAM_QUALITY"); 487 continue; 488 } 489 if (buffer_eq_icase_slen(&du->key, 490 CONST_STR_LEN("BROTLI_PARAM_LGWIN"))) { 491 /*(future: could check for string and then look for and translate 492 * BROTLI_DEFAULT_WINDOW 493 * BROTLI_MIN_WINDOW_BITS BROTLI_MAX_WINDOW_BITS)*/ 494 if (BROTLI_MIN_WINDOW_BITS <= v && v <= BROTLI_MAX_WINDOW_BITS) 495 params->brotli.window = (uint32_t)v; /* 10 .. 24 */ 496 else 497 log_error(errh, __FILE__, __LINE__, 498 "invalid value for BROTLI_PARAM_LGWIN"); 499 continue; 500 } 501 if (buffer_eq_icase_slen(&du->key, 502 CONST_STR_LEN("BROTLI_PARAM_MODE"))) { 503 /*(future: could check for string and then look for and translate 504 * BROTLI_MODE_GENERIC BROTLI_MODE_TEXT BROTLI_MODE_FONT)*/ 505 if (0 <= v && v <= 2) 506 params->brotli.mode = (uint32_t)v; /* 0 .. 2 */ 507 else 508 log_error(errh, __FILE__, __LINE__, 509 "invalid value for BROTLI_PARAM_MODE"); 510 continue; 511 } 512 #endif 513 #ifdef USE_ZSTD 514 if (buffer_eq_icase_slen(&du->key, 515 CONST_STR_LEN("ZSTD_c_compressionLevel"))) { 516 params->zstd.clevel = v; 517 /*(not warning if number parse error. future: to detect, could 518 * use absurd default to config_plugin_value_to_int32 to detect)*/ 519 continue; 520 } 521 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 522 /*(XXX: (selected) experimental API params in zstd v1.4.0)*/ 523 if (buffer_eq_icase_slen(&du->key, 524 CONST_STR_LEN("ZSTD_c_strategy"))) { 525 /*(future: could check for string and then look for and translate 526 * enum ZSTD_strategy ZSTD_STRATEGY_MIN ZSTD_STRATEGY_MAX)*/ 527 #ifndef ZSTD_STRATEGY_MIN 528 #define ZSTD_STRATEGY_MIN 1 529 #endif 530 #ifndef ZSTD_STRATEGY_MAX 531 #define ZSTD_STRATEGY_MAX 9 532 #endif 533 if (ZSTD_STRATEGY_MIN <= v && v <= ZSTD_STRATEGY_MAX) 534 params->zstd.strategy = v; /* 1 .. 9 */ 535 else 536 log_error(errh, __FILE__, __LINE__, 537 "invalid value for ZSTD_c_strategy"); 538 continue; 539 } 540 if (buffer_eq_icase_slen(&du->key, 541 CONST_STR_LEN("ZSTD_c_windowLog"))) { 542 /*(future: could check for string and then look for and translate 543 * ZSTD_WINDOWLOG_MIN ZSTD_WINDOWLOG_MAX)*/ 544 #ifndef ZSTD_WINDOWLOG_MIN 545 #define ZSTD_WINDOWLOG_MIN 10 546 #endif 547 #ifndef ZSTD_WINDOWLOG_MAX_32 548 #define ZSTD_WINDOWLOG_MAX_32 30 549 #endif 550 #ifndef ZSTD_WINDOWLOG_MAX_64 551 #define ZSTD_WINDOWLOG_MAX_64 31 552 #endif 553 #ifndef ZSTD_WINDOWLOG_MAX 554 #define ZSTD_WINDOWLOG_MAX \ 555 (sizeof(size_t)==4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64) 556 #endif 557 if (ZSTD_WINDOWLOG_MIN <= v && v <= ZSTD_WINDOWLOG_MAX) 558 params->zstd.windowLog = v;/* 10 .. 31 *//*(30 max for 32-bit)*/ 559 else 560 log_error(errh, __FILE__, __LINE__, 561 "invalid value for ZSTD_c_windowLog"); 562 continue; 563 } 564 #endif 565 #endif 566 #ifdef USE_ZLIB 567 if (buffer_eq_icase_slen(&du->key, 568 CONST_STR_LEN("gzip.level"))) { 569 if (1 <= v && v <= 9) 570 params->gzip.clevel = v; /* 1 .. 9 */ 571 else 572 log_error(errh, __FILE__, __LINE__, 573 "invalid value for gzip.level"); 574 continue; 575 } 576 if (buffer_eq_icase_slen(&du->key, 577 CONST_STR_LEN("gzip.windowBits"))) { 578 if (9 <= v && v <= 15) 579 params->gzip.windowBits = v; /* 9 .. 15 */ 580 else 581 log_error(errh, __FILE__, __LINE__, 582 "invalid value for gzip.windowBits"); 583 continue; 584 } 585 if (buffer_eq_icase_slen(&du->key, 586 CONST_STR_LEN("gzip.memLevel"))) { 587 if (1 <= v && v <= 9) 588 params->gzip.memLevel = v; /* 1 .. 9 */ 589 else 590 log_error(errh, __FILE__, __LINE__, 591 "invalid value for gzip.memLevel"); 592 continue; 593 } 594 if (buffer_eq_icase_slen(&du->key, 595 CONST_STR_LEN("gzip.strategy"))) { 596 /*(future: could check for string and then look for and translate 597 * Z_DEFAULT_STRATEGY Z_FILTERED Z_HUFFMAN_ONLY Z_RLE Z_FIXED)*/ 598 if (0 <= v && v <= 4) 599 params->gzip.strategy = v; /* 0 .. 4 */ 600 else 601 log_error(errh, __FILE__, __LINE__, 602 "invalid value for gzip.strategy"); 603 continue; 604 } 605 #endif 606 #ifdef USE_BZ2LIB 607 if (buffer_eq_icase_slen(&du->key, 608 CONST_STR_LEN("bzip2.blockSize100k"))) { 609 if (1 <= v && v <= 9) /*(bzip2 blockSize100k param)*/ 610 params->bzip2.clevel = v; /* 1 .. 9 */ 611 else 612 log_error(errh, __FILE__, __LINE__, 613 "invalid value for bzip2.blockSize100k"); 614 continue; 615 } 616 #endif 617 log_error(errh, __FILE__, __LINE__, 618 "unrecognized param: %s", du->key.ptr); 619 } 620 621 return params; 622 } 623 624 static uint16_t * mod_deflate_encodings_to_flags(const array *encodings) { 625 if (encodings->used) { 626 uint16_t * const x = calloc(encodings->used+1, sizeof(short)); 627 force_assert(x); 628 int i = 0; 629 for (uint32_t j = 0; j < encodings->used; ++j) { 630 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 631 || defined(USE_ZSTD) 632 data_string *ds = (data_string *)encodings->data[j]; 633 #endif 634 #ifdef USE_ZLIB /* "gzip", "x-gzip" */ 635 if (NULL != strstr(ds->value.ptr, "gzip")) 636 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 637 | HTTP_ACCEPT_ENCODING_X_GZIP; 638 if (NULL != strstr(ds->value.ptr, "deflate")) 639 x[i++] = HTTP_ACCEPT_ENCODING_DEFLATE; 640 /* 641 if (NULL != strstr(ds->value.ptr, "compress")) 642 x[i++] = HTTP_ACCEPT_ENCODING_COMPRESS; 643 */ 644 #endif 645 #ifdef USE_BZ2LIB /* "bzip2", "x-bzip2" */ 646 if (NULL != strstr(ds->value.ptr, "bzip2")) 647 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 648 | HTTP_ACCEPT_ENCODING_X_BZIP2; 649 #endif 650 #ifdef USE_BROTLI /* "br" (also accepts "brotli") */ 651 if (NULL != strstr(ds->value.ptr, "br")) 652 x[i++] = HTTP_ACCEPT_ENCODING_BR; 653 #endif 654 #ifdef USE_ZSTD 655 if (NULL != strstr(ds->value.ptr, "zstd")) 656 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 657 #endif 658 } 659 x[i] = 0; /* end of list */ 660 return x; 661 } 662 else { 663 /* default encodings */ 664 uint16_t * const x = calloc(4+1, sizeof(short)); 665 force_assert(x); 666 int i = 0; 667 #ifdef USE_ZSTD 668 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 669 #endif 670 #ifdef USE_BROTLI 671 x[i++] = HTTP_ACCEPT_ENCODING_BR; 672 #endif 673 #ifdef USE_BZ2LIB 674 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 675 | HTTP_ACCEPT_ENCODING_X_BZIP2; 676 #endif 677 #ifdef USE_ZLIB 678 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 679 | HTTP_ACCEPT_ENCODING_X_GZIP 680 | HTTP_ACCEPT_ENCODING_DEFLATE; 681 #endif 682 x[i] = 0; /* end of list */ 683 return x; 684 } 685 } 686 687 SETDEFAULTS_FUNC(mod_deflate_set_defaults) { 688 static const config_plugin_keys_t cpk[] = { 689 { CONST_STR_LEN("deflate.mimetypes"), 690 T_CONFIG_ARRAY_VLIST, 691 T_CONFIG_SCOPE_CONNECTION } 692 ,{ CONST_STR_LEN("deflate.allowed-encodings"), 693 T_CONFIG_ARRAY_VLIST, 694 T_CONFIG_SCOPE_CONNECTION } 695 ,{ CONST_STR_LEN("deflate.max-compress-size"), 696 T_CONFIG_INT, 697 T_CONFIG_SCOPE_CONNECTION } 698 ,{ CONST_STR_LEN("deflate.min-compress-size"), 699 T_CONFIG_SHORT, 700 T_CONFIG_SCOPE_CONNECTION } 701 ,{ CONST_STR_LEN("deflate.compression-level"), 702 T_CONFIG_SHORT, 703 T_CONFIG_SCOPE_CONNECTION } 704 ,{ CONST_STR_LEN("deflate.output-buffer-size"), 705 T_CONFIG_SHORT, 706 T_CONFIG_SCOPE_CONNECTION } 707 ,{ CONST_STR_LEN("deflate.work-block-size"), 708 T_CONFIG_SHORT, 709 T_CONFIG_SCOPE_CONNECTION } 710 ,{ CONST_STR_LEN("deflate.max-loadavg"), 711 T_CONFIG_STRING, 712 T_CONFIG_SCOPE_CONNECTION } 713 ,{ CONST_STR_LEN("deflate.cache-dir"), 714 T_CONFIG_STRING, 715 T_CONFIG_SCOPE_CONNECTION } 716 ,{ CONST_STR_LEN("compress.filetype"), 717 T_CONFIG_ARRAY_VLIST, 718 T_CONFIG_SCOPE_CONNECTION } 719 ,{ CONST_STR_LEN("compress.allowed-encodings"), 720 T_CONFIG_ARRAY_VLIST, 721 T_CONFIG_SCOPE_CONNECTION } 722 ,{ CONST_STR_LEN("compress.cache-dir"), 723 T_CONFIG_STRING, 724 T_CONFIG_SCOPE_CONNECTION } 725 ,{ CONST_STR_LEN("compress.max-filesize"), 726 T_CONFIG_INT, 727 T_CONFIG_SCOPE_CONNECTION } 728 ,{ CONST_STR_LEN("compress.max-loadavg"), 729 T_CONFIG_STRING, 730 T_CONFIG_SCOPE_CONNECTION } 731 ,{ CONST_STR_LEN("deflate.params"), 732 T_CONFIG_ARRAY_KVANY, 733 T_CONFIG_SCOPE_CONNECTION } 734 ,{ NULL, 0, 735 T_CONFIG_UNSET, 736 T_CONFIG_SCOPE_UNSET } 737 }; 738 739 plugin_data * const p = p_d; 740 if (!config_plugin_values_init(srv, p, cpk, "mod_deflate")) 741 return HANDLER_ERROR; 742 743 /* process and validate config directives 744 * (init i to 0 if global context; to 1 to skip empty global context) */ 745 for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) { 746 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 747 for (; -1 != cpv->k_id; ++cpv) { 748 switch (cpv->k_id) { 749 case 9: /* compress.filetype */ 750 log_error(srv->errh, __FILE__, __LINE__, 751 "DEPRECATED: %s replaced with deflate.mimetypes", 752 cpk[cpv->k_id].k); 753 cpv->k_id = 0; /* deflate.mimetypes */ 754 __attribute_fallthrough__ 755 case 0: /* deflate.mimetypes */ 756 /* mod_deflate matches mimetype as prefix of Content-Type 757 * so ignore '*' at end of mimetype for end-user flexibility 758 * in specifying trailing wildcard to grouping of mimetypes */ 759 for (uint32_t m = 0; m < cpv->v.a->used; ++m) { 760 buffer *mimetype=&((data_string *)cpv->v.a->data[m])->value; 761 size_t len = buffer_clen(mimetype); 762 if (len > 2 && mimetype->ptr[len-1] == '*') 763 buffer_truncate(mimetype, len-1); 764 } 765 if (0 == cpv->v.a->used) cpv->v.a = NULL; 766 break; 767 case 10:/* compress.allowed-encodings */ 768 log_error(srv->errh, __FILE__, __LINE__, 769 "DEPRECATED: %s replaced with deflate.allowed-encodings", 770 cpk[cpv->k_id].k); 771 cpv->k_id = 1; /* deflate.allowed-encodings */ 772 __attribute_fallthrough__ 773 case 1: /* deflate.allowed-encodings */ 774 cpv->v.v = mod_deflate_encodings_to_flags(cpv->v.a); 775 cpv->vtype = T_CONFIG_LOCAL; 776 break; 777 case 12:/* compress.max-filesize */ 778 log_error(srv->errh, __FILE__, __LINE__, 779 "DEPRECATED: %s replaced with deflate.max-compress-size", 780 cpk[cpv->k_id].k); 781 cpv->k_id = 2; /* deflate.max-compress-size */ 782 __attribute_fallthrough__ 783 case 2: /* deflate.max-compress-size */ 784 case 3: /* deflate.min-compress-size */ 785 break; 786 case 4: /* deflate.compression-level */ 787 if ((cpv->v.shrt < 1 || cpv->v.shrt > 9) 788 && *(short *)&cpv->v.shrt != -1) { 789 log_error(srv->errh, __FILE__, __LINE__, 790 "compression-level must be between 1 and 9: %hu", 791 cpv->v.shrt); 792 return HANDLER_ERROR; 793 } 794 break; 795 case 5: /* deflate.output-buffer-size */ 796 case 6: /* deflate.work-block-size */ 797 break; 798 case 13:/* compress.max-loadavg */ 799 log_error(srv->errh, __FILE__, __LINE__, 800 "DEPRECATED: %s replaced with deflate.max-loadavg", 801 cpk[cpv->k_id].k); 802 cpv->k_id = 7; /* deflate.max-loadavg */ 803 __attribute_fallthrough__ 804 case 7: /* deflate.max-loadavg */ 805 cpv->v.d = (!buffer_is_blank(cpv->v.b)) 806 ? strtod(cpv->v.b->ptr, NULL) 807 : 0.0; 808 break; 809 case 11:/* compress.cache-dir */ 810 log_error(srv->errh, __FILE__, __LINE__, 811 "DEPRECATED: %s replaced with deflate.cache-dir", 812 cpk[cpv->k_id].k); 813 cpv->k_id = 8; /* deflate.cache-dir */ 814 __attribute_fallthrough__ 815 case 8: /* deflate.cache-dir */ 816 if (!buffer_is_blank(cpv->v.b)) { 817 buffer *b; 818 *(const buffer **)&b = cpv->v.b; 819 const uint32_t len = buffer_clen(b); 820 if (len > 0 && '/' == b->ptr[len-1]) 821 buffer_truncate(b, len-1); /*remove end slash*/ 822 struct stat st; 823 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 824 log_perror(srv->errh, __FILE__, __LINE__, 825 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 826 return HANDLER_ERROR; 827 } 828 } 829 else 830 cpv->v.b = NULL; 831 break; 832 #if 0 /*(handled further above)*/ 833 case 9: /* compress.filetype */ 834 case 10:/* compress.allowed-encodings */ 835 case 11:/* compress.cache-dir */ 836 case 12:/* compress.max-filesize */ 837 case 13:/* compress.max-loadavg */ 838 break; 839 #endif 840 case 14:/* deflate.params */ 841 cpv->v.v = mod_deflate_parse_params(cpv->v.a, srv->errh); 842 cpv->vtype = T_CONFIG_LOCAL; 843 break; 844 default:/* should not happen */ 845 break; 846 } 847 } 848 } 849 850 p->defaults.max_compress_size = 128*1024; /*(128 MB measured as num KB)*/ 851 p->defaults.min_compress_size = 256; 852 p->defaults.compression_level = -1; 853 p->defaults.output_buffer_size = 0; 854 p->defaults.work_block_size = 2048; 855 p->defaults.max_loadavg = 0.0; 856 p->defaults.sync_flush = 0; 857 858 /* initialize p->defaults from global config context */ 859 if (p->nconfig > 0 && p->cvlist->v.u2[1]) { 860 const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0]; 861 if (-1 != cpv->k_id) 862 mod_deflate_merge_config(&p->defaults, cpv); 863 } 864 865 return HANDLER_GO_ON; 866 } 867 868 869 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 870 || defined(USE_ZSTD) 871 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { 872 ssize_t wr; 873 do { 874 wr = write(hctx->cache_fd, out, len); 875 } while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR); 876 return (0 == len) ? 0 : -1; 877 } 878 879 static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) { 880 if (0 == len) return 0; 881 return (-1 == hctx->cache_fd) 882 ? http_chunk_append_mem(hctx->r, out, len) 883 : mod_deflate_cache_file_append(hctx, out, len); 884 } 885 #endif 886 887 888 #ifdef USE_ZLIB 889 890 static int stream_deflate_init(handler_ctx *hctx) { 891 z_stream * const z = &hctx->u.z; 892 z->zalloc = Z_NULL; 893 z->zfree = Z_NULL; 894 z->opaque = Z_NULL; 895 z->total_in = 0; 896 z->total_out = 0; 897 z->next_out = (unsigned char *)hctx->output->ptr; 898 z->avail_out = hctx->output->size; 899 900 const plugin_data * const p = hctx->plugin_data; 901 const encparms * const params = p->conf.params; 902 const int clevel = (NULL != params) 903 ? params->gzip.clevel 904 : p->conf.compression_level; 905 const int wbits = (NULL != params) 906 ? params->gzip.windowBits 907 : MAX_WBITS; 908 909 if (Z_OK != deflateInit2(z, 910 clevel > 0 ? clevel : Z_DEFAULT_COMPRESSION, 911 Z_DEFLATED, 912 (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 913 ? (wbits | 16) /*(0x10 flags gzip header, trailer)*/ 914 : -wbits, /*(negate to suppress zlib header)*/ 915 params ? params->gzip.memLevel : 8,/*default memLevel*/ 916 params ? params->gzip.strategy : Z_DEFAULT_STRATEGY)) { 917 return -1; 918 } 919 920 return 0; 921 } 922 923 static int stream_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 924 z_stream * const z = &(hctx->u.z); 925 size_t len; 926 927 z->next_in = start; 928 z->avail_in = st_size; 929 hctx->bytes_in += st_size; 930 931 /* compress data */ 932 do { 933 if (Z_OK != deflate(z, Z_NO_FLUSH)) return -1; 934 935 if (z->avail_out == 0 || z->avail_in > 0) { 936 len = hctx->output->size - z->avail_out; 937 hctx->bytes_out += len; 938 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 939 return -1; 940 z->next_out = (unsigned char *)hctx->output->ptr; 941 z->avail_out = hctx->output->size; 942 } 943 } while (z->avail_in > 0); 944 945 return 0; 946 } 947 948 static int stream_deflate_flush(handler_ctx * const hctx, int end) { 949 z_stream * const z = &(hctx->u.z); 950 const plugin_data *p = hctx->plugin_data; 951 size_t len; 952 int rc = 0; 953 int done; 954 955 /* compress data */ 956 do { 957 done = 1; 958 if (end) { 959 rc = deflate(z, Z_FINISH); 960 if (rc == Z_OK) { 961 done = 0; 962 } else if (rc != Z_STREAM_END) { 963 return -1; 964 } 965 } else { 966 if (p->conf.sync_flush) { 967 rc = deflate(z, Z_SYNC_FLUSH); 968 if (rc != Z_OK) return -1; 969 } else if (z->avail_in > 0) { 970 rc = deflate(z, Z_NO_FLUSH); 971 if (rc != Z_OK) return -1; 972 } 973 } 974 975 len = hctx->output->size - z->avail_out; 976 if (z->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 977 hctx->bytes_out += len; 978 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 979 return -1; 980 z->next_out = (unsigned char *)hctx->output->ptr; 981 z->avail_out = hctx->output->size; 982 } 983 } while (z->avail_in != 0 || !done); 984 985 return 0; 986 } 987 988 static int stream_deflate_end(handler_ctx *hctx) { 989 z_stream * const z = &(hctx->u.z); 990 int rc = deflateEnd(z); 991 if (Z_OK == rc || Z_DATA_ERROR == rc) return 0; 992 993 if (z->msg != NULL) { 994 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 995 "deflateEnd error ret=%d, msg=%s", rc, z->msg); 996 } else { 997 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 998 "deflateEnd error ret=%d", rc); 999 } 1000 return -1; 1001 } 1002 1003 #endif 1004 1005 1006 #ifdef USE_BZ2LIB 1007 1008 static int stream_bzip2_init(handler_ctx *hctx) { 1009 bz_stream * const bz = &hctx->u.bz; 1010 bz->bzalloc = NULL; 1011 bz->bzfree = NULL; 1012 bz->opaque = NULL; 1013 bz->total_in_lo32 = 0; 1014 bz->total_in_hi32 = 0; 1015 bz->total_out_lo32 = 0; 1016 bz->total_out_hi32 = 0; 1017 bz->next_out = hctx->output->ptr; 1018 bz->avail_out = hctx->output->size; 1019 1020 const plugin_data * const p = hctx->plugin_data; 1021 const encparms * const params = p->conf.params; 1022 const int clevel = (NULL != params) 1023 ? params->bzip2.clevel 1024 : p->conf.compression_level; 1025 1026 if (BZ_OK != BZ2_bzCompressInit(bz, 1027 clevel > 0 1028 ? p->conf.compression_level 1029 : 9, /* blocksize = 900k */ 1030 0, /* verbosity */ 1031 0)) { /* workFactor: default */ 1032 return -1; 1033 } 1034 1035 return 0; 1036 } 1037 1038 static int stream_bzip2_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 1039 bz_stream * const bz = &(hctx->u.bz); 1040 size_t len; 1041 1042 bz->next_in = (char *)start; 1043 bz->avail_in = st_size; 1044 hctx->bytes_in += st_size; 1045 1046 /* compress data */ 1047 do { 1048 if (BZ_RUN_OK != BZ2_bzCompress(bz, BZ_RUN)) return -1; 1049 1050 if (bz->avail_out == 0 || bz->avail_in > 0) { 1051 len = hctx->output->size - bz->avail_out; 1052 hctx->bytes_out += len; 1053 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1054 return -1; 1055 bz->next_out = hctx->output->ptr; 1056 bz->avail_out = hctx->output->size; 1057 } 1058 } while (bz->avail_in > 0); 1059 1060 return 0; 1061 } 1062 1063 static int stream_bzip2_flush(handler_ctx * const hctx, int end) { 1064 bz_stream * const bz = &(hctx->u.bz); 1065 const plugin_data *p = hctx->plugin_data; 1066 size_t len; 1067 int rc; 1068 int done; 1069 1070 /* compress data */ 1071 do { 1072 done = 1; 1073 if (end) { 1074 rc = BZ2_bzCompress(bz, BZ_FINISH); 1075 if (rc == BZ_FINISH_OK) { 1076 done = 0; 1077 } else if (rc != BZ_STREAM_END) { 1078 return -1; 1079 } 1080 } else if (bz->avail_in > 0) { 1081 /* p->conf.sync_flush not implemented here, 1082 * which would loop on BZ_FLUSH while BZ_FLUSH_OK 1083 * until BZ_RUN_OK returned */ 1084 rc = BZ2_bzCompress(bz, BZ_RUN); 1085 if (rc != BZ_RUN_OK) { 1086 return -1; 1087 } 1088 } 1089 1090 len = hctx->output->size - bz->avail_out; 1091 if (bz->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 1092 hctx->bytes_out += len; 1093 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1094 return -1; 1095 bz->next_out = hctx->output->ptr; 1096 bz->avail_out = hctx->output->size; 1097 } 1098 } while (bz->avail_in != 0 || !done); 1099 1100 return 0; 1101 } 1102 1103 static int stream_bzip2_end(handler_ctx *hctx) { 1104 bz_stream * const bz = &(hctx->u.bz); 1105 int rc = BZ2_bzCompressEnd(bz); 1106 if (BZ_OK == rc || BZ_DATA_ERROR == rc) return 0; 1107 1108 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 1109 "BZ2_bzCompressEnd error ret=%d", rc); 1110 return -1; 1111 } 1112 1113 #endif 1114 1115 1116 #ifdef USE_BROTLI 1117 1118 static int stream_br_init(handler_ctx *hctx) { 1119 BrotliEncoderState * const br = hctx->u.br = 1120 BrotliEncoderCreateInstance(NULL, NULL, NULL); 1121 if (NULL == br) return -1; 1122 1123 /*(note: we ignore any errors while tuning parameters here)*/ 1124 const plugin_data * const p = hctx->plugin_data; 1125 const encparms * const params = p->conf.params; 1126 const uint32_t quality = (NULL != params) 1127 ? params->brotli.quality 1128 : (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */ 1129 ? (uint32_t)p->conf.compression_level 1130 : 5; 1131 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 1132 if (quality != BROTLI_DEFAULT_QUALITY) 1133 BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY, quality); 1134 1135 if (params && params->brotli.window != BROTLI_DEFAULT_WINDOW) 1136 BrotliEncoderSetParameter(br, BROTLI_PARAM_LGWIN,params->brotli.window); 1137 1138 const buffer *vb; 1139 if (params && params->brotli.mode != BROTLI_MODE_GENERIC) 1140 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, params->brotli.mode); 1141 else if ((vb = http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE, 1142 CONST_STR_LEN("Content-Type")))) { 1143 /* BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */ 1144 const uint32_t len = buffer_clen(vb); 1145 if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1) 1146 || (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1) 1147 && (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1) 1148 || 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1) 1149 || 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1))) 1150 || (len > 4 1151 && (0 == strncmp(vb->ptr+len-5, "+json", sizeof("+json")-1) 1152 || 0 == strncmp(vb->ptr+len-4, "+xml", sizeof("+xml")-1)))) 1153 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); 1154 else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1)) 1155 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT); 1156 } 1157 1158 return 0; 1159 } 1160 1161 static int stream_br_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 1162 const uint8_t *in = (uint8_t *)start; 1163 BrotliEncoderState * const br = hctx->u.br; 1164 hctx->bytes_in += st_size; 1165 while (st_size || BrotliEncoderHasMoreOutput(br)) { 1166 size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size) 1167 ? (size_t)st_size 1168 : ((~(uint32_t)0) >> 1); 1169 size_t outsz = 0; 1170 BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS, 1171 &insz, &in, &outsz, NULL, NULL); 1172 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1173 st_size -= (st_size - (off_t)insz); 1174 if (outsz) { 1175 hctx->bytes_out += (off_t)outsz; 1176 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1177 return -1; 1178 } 1179 } 1180 return 0; 1181 } 1182 1183 static int stream_br_flush(handler_ctx * const hctx, int end) { 1184 const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; 1185 BrotliEncoderState * const br = hctx->u.br; 1186 do { 1187 size_t insz = 0; 1188 size_t outsz = 0; 1189 BrotliEncoderCompressStream(br, brmode, 1190 &insz, NULL, &outsz, NULL, NULL); 1191 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1192 if (outsz) { 1193 hctx->bytes_out += (off_t)outsz; 1194 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1195 return -1; 1196 } 1197 } while (BrotliEncoderHasMoreOutput(br)); 1198 return 0; 1199 } 1200 1201 static int stream_br_end(handler_ctx *hctx) { 1202 BrotliEncoderState * const br = hctx->u.br; 1203 BrotliEncoderDestroyInstance(br); 1204 return 0; 1205 } 1206 1207 #endif 1208 1209 1210 #ifdef USE_ZSTD 1211 1212 static int stream_zstd_init(handler_ctx *hctx) { 1213 ZSTD_CStream * const cctx = hctx->u.cctx = ZSTD_createCStream(); 1214 if (NULL == cctx) return -1; 1215 hctx->output->used = 0; 1216 1217 /*(note: we ignore any errors while tuning parameters here)*/ 1218 const plugin_data * const p = hctx->plugin_data; 1219 const encparms * const params = p->conf.params; 1220 if (params) { 1221 if (params->zstd.clevel && params->zstd.clevel != ZSTD_CLEVEL_DEFAULT) { 1222 const int level = params->zstd.clevel; 1223 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1224 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); 1225 #else 1226 ZSTD_initCStream(cctx, level); 1227 #endif 1228 } 1229 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1230 if (params->zstd.strategy) 1231 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, 1232 params->zstd.strategy); 1233 if (params->zstd.windowLog) 1234 ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 1235 params->zstd.windowLog); 1236 #endif 1237 } 1238 else if (p->conf.compression_level >= 0) { /* -1 here is "unset" */ 1239 int level = p->conf.compression_level; 1240 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1241 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, level); 1242 #else 1243 ZSTD_initCStream(cctx, level); 1244 #endif 1245 } 1246 return 0; 1247 } 1248 1249 static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 1250 ZSTD_CStream * const cctx = hctx->u.cctx; 1251 ZSTD_inBuffer zib = { start, (size_t)st_size, 0 }; 1252 ZSTD_outBuffer zob = { hctx->output->ptr, 1253 hctx->output->size, 1254 hctx->output->used }; 1255 hctx->output->used = 0; 1256 hctx->bytes_in += st_size; 1257 while (zib.pos < zib.size) { 1258 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1259 const size_t rv = ZSTD_compressStream2(cctx,&zob,&zib,ZSTD_e_continue); 1260 #else 1261 const size_t rv = ZSTD_compressStream(cctx, &zob, &zib); 1262 #endif 1263 if (ZSTD_isError(rv)) return -1; 1264 if (zib.pos == zib.size) break; /* defer flush */ 1265 hctx->bytes_out += (off_t)zob.pos; 1266 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1267 return -1; 1268 zob.pos = 0; 1269 } 1270 hctx->output->used = (uint32_t)zob.pos; 1271 return 0; 1272 } 1273 1274 static int stream_zstd_flush(handler_ctx * const hctx, int end) { 1275 ZSTD_CStream * const cctx = hctx->u.cctx; 1276 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1277 const ZSTD_EndDirective endOp = end ? ZSTD_e_end : ZSTD_e_flush; 1278 ZSTD_inBuffer zib = { NULL, 0, 0 }; 1279 #endif 1280 ZSTD_outBuffer zob = { hctx->output->ptr, 1281 hctx->output->size, 1282 hctx->output->used }; 1283 size_t rv; 1284 do { 1285 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1286 rv = ZSTD_compressStream2(cctx, &zob, &zib, endOp); 1287 #else 1288 rv = end 1289 ? ZSTD_endStream(cctx, &zob) 1290 : ZSTD_flushStream(cctx, &zob); 1291 #endif 1292 if (ZSTD_isError(rv)) return -1; 1293 hctx->bytes_out += (off_t)zob.pos; 1294 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1295 return -1; 1296 zob.pos = 0; 1297 } while (0 != rv); 1298 return 0; 1299 } 1300 1301 static int stream_zstd_end(handler_ctx *hctx) { 1302 ZSTD_CStream * const cctx = hctx->u.cctx; 1303 ZSTD_freeCStream(cctx); 1304 return 0; 1305 } 1306 1307 #endif 1308 1309 1310 static int mod_deflate_stream_init(handler_ctx *hctx) { 1311 switch(hctx->compression_type) { 1312 #ifdef USE_ZLIB 1313 case HTTP_ACCEPT_ENCODING_GZIP: 1314 case HTTP_ACCEPT_ENCODING_DEFLATE: 1315 return stream_deflate_init(hctx); 1316 #endif 1317 #ifdef USE_BZ2LIB 1318 case HTTP_ACCEPT_ENCODING_BZIP2: 1319 return stream_bzip2_init(hctx); 1320 #endif 1321 #ifdef USE_BROTLI 1322 case HTTP_ACCEPT_ENCODING_BR: 1323 return stream_br_init(hctx); 1324 #endif 1325 #ifdef USE_ZSTD 1326 case HTTP_ACCEPT_ENCODING_ZSTD: 1327 return stream_zstd_init(hctx); 1328 #endif 1329 default: 1330 return -1; 1331 } 1332 } 1333 1334 static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { 1335 if (0 == st_size) return 0; 1336 switch(hctx->compression_type) { 1337 #ifdef USE_ZLIB 1338 case HTTP_ACCEPT_ENCODING_GZIP: 1339 case HTTP_ACCEPT_ENCODING_DEFLATE: 1340 return stream_deflate_compress(hctx, start, st_size); 1341 #endif 1342 #ifdef USE_BZ2LIB 1343 case HTTP_ACCEPT_ENCODING_BZIP2: 1344 return stream_bzip2_compress(hctx, start, st_size); 1345 #endif 1346 #ifdef USE_BROTLI 1347 case HTTP_ACCEPT_ENCODING_BR: 1348 return stream_br_compress(hctx, start, st_size); 1349 #endif 1350 #ifdef USE_ZSTD 1351 case HTTP_ACCEPT_ENCODING_ZSTD: 1352 return stream_zstd_compress(hctx, start, st_size); 1353 #endif 1354 default: 1355 UNUSED(start); 1356 return -1; 1357 } 1358 } 1359 1360 static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { 1361 if (0 == hctx->bytes_in) return 0; 1362 switch(hctx->compression_type) { 1363 #ifdef USE_ZLIB 1364 case HTTP_ACCEPT_ENCODING_GZIP: 1365 case HTTP_ACCEPT_ENCODING_DEFLATE: 1366 return stream_deflate_flush(hctx, end); 1367 #endif 1368 #ifdef USE_BZ2LIB 1369 case HTTP_ACCEPT_ENCODING_BZIP2: 1370 return stream_bzip2_flush(hctx, end); 1371 #endif 1372 #ifdef USE_BROTLI 1373 case HTTP_ACCEPT_ENCODING_BR: 1374 return stream_br_flush(hctx, end); 1375 #endif 1376 #ifdef USE_ZSTD 1377 case HTTP_ACCEPT_ENCODING_ZSTD: 1378 return stream_zstd_flush(hctx, end); 1379 #endif 1380 default: 1381 UNUSED(end); 1382 return -1; 1383 } 1384 } 1385 1386 static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) { 1387 /* store compression ratio in environment 1388 * for possible logging by mod_accesslog 1389 * (late in response handling, so not seen by most other modules) */ 1390 /*(should be called only at end of successful response compression)*/ 1391 if (0 == bytes_in) return; 1392 buffer_append_int( 1393 http_header_env_set_ptr(r, CONST_STR_LEN("ratio")), 1394 bytes_out * 100 / bytes_in); 1395 } 1396 1397 static int mod_deflate_stream_end(handler_ctx *hctx) { 1398 switch(hctx->compression_type) { 1399 #ifdef USE_ZLIB 1400 case HTTP_ACCEPT_ENCODING_GZIP: 1401 case HTTP_ACCEPT_ENCODING_DEFLATE: 1402 return stream_deflate_end(hctx); 1403 #endif 1404 #ifdef USE_BZ2LIB 1405 case HTTP_ACCEPT_ENCODING_BZIP2: 1406 return stream_bzip2_end(hctx); 1407 #endif 1408 #ifdef USE_BROTLI 1409 case HTTP_ACCEPT_ENCODING_BR: 1410 return stream_br_end(hctx); 1411 #endif 1412 #ifdef USE_ZSTD 1413 case HTTP_ACCEPT_ENCODING_ZSTD: 1414 return stream_zstd_end(hctx); 1415 #endif 1416 default: 1417 return -1; 1418 } 1419 } 1420 1421 static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) { 1422 int rc = mod_deflate_stream_end(hctx); 1423 1424 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */ 1425 if (0 == rc && hctx->bytes_in < hctx->bytes_out) 1426 log_error(r->conf.errh, __FILE__, __LINE__, 1427 "uri %s in=%lld smaller than out=%lld", r->target.ptr, 1428 (long long)hctx->bytes_in, (long long)hctx->bytes_out); 1429 #endif 1430 1431 handler_ctx_free(hctx); 1432 return rc; 1433 } 1434 1435 1436 static int mod_deflate_file_chunk(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t st_size) { 1437 off_t abs_offset; 1438 off_t toSend = -1; 1439 char *start; 1440 #ifdef USE_MMAP 1441 /* use large chunks since server blocks while compressing, anyway 1442 * (mod_deflate is not recommended for large files) */ 1443 const off_t mmap_chunk_size = 16 * 1024 * 1024; /* must be power-of-2 */ 1444 off_t we_want_to_send = st_size; 1445 volatile int mapped = 0;/* quiet warning: might be clobbered by 'longjmp' */ 1446 #else 1447 start = NULL; 1448 #endif 1449 1450 if (-1 == c->file.fd) { /* open the file if not already open */ 1451 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) { 1452 log_perror(r->conf.errh, __FILE__, __LINE__, "open failed %s", c->mem->ptr); 1453 return -1; 1454 } 1455 } 1456 1457 abs_offset = c->offset; 1458 1459 #ifdef USE_MMAP 1460 /* mmap the buffer 1461 * - first mmap 1462 * - new mmap as the we are at the end of the last one */ 1463 if (c->file.mmap.start == MAP_FAILED || 1464 abs_offset == (off_t)(c->file.mmap.offset + c->file.mmap.length)) { 1465 1466 /* Optimizations for the future: 1467 * 1468 * (comment is outdated since server blocks compressing file and then munmap()s file) 1469 * 1470 * adaptive mem-mapping 1471 * the problem: 1472 * we mmap() the whole file. If someone has a lot of large files and 32bit 1473 * machine the virtual address area will be exhausted and we will have a failing 1474 * mmap() call. 1475 * solution: 1476 * only mmap 16M in one chunk and move the window as soon as we have finished 1477 * the first 8M 1478 * 1479 * read-ahead buffering 1480 * the problem: 1481 * sending out several large files in parallel trashes the read-ahead of the 1482 * kernel leading to long wait-for-seek times. 1483 * solutions: (increasing complexity) 1484 * 1. use madvise 1485 * 2. use a internal read-ahead buffer in the chunk-structure 1486 * 3. use non-blocking IO for file-transfers 1487 * */ 1488 1489 /* all mmap()ed areas are mmap_chunk_size except the last which might be smaller */ 1490 off_t to_mmap; 1491 1492 /* this is a remap, move the mmap-offset */ 1493 if (c->file.mmap.start != MAP_FAILED) { 1494 munmap(c->file.mmap.start, c->file.mmap.length); 1495 c->file.mmap.offset += mmap_chunk_size; 1496 } else { 1497 /* in case the range-offset is after the first mmap()ed area we skip the area */ 1498 c->file.mmap.offset = c->offset & ~(mmap_chunk_size-1); 1499 } 1500 1501 to_mmap = c->file.length - c->file.mmap.offset; 1502 if (to_mmap > mmap_chunk_size) to_mmap = mmap_chunk_size; 1503 /* we have more to send than we can mmap() at once */ 1504 if (we_want_to_send > to_mmap) we_want_to_send = to_mmap; 1505 1506 if (MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_SHARED, c->file.fd, c->file.mmap.offset)) 1507 && (errno != EINVAL || MAP_FAILED == (c->file.mmap.start = mmap(0, (size_t)to_mmap, PROT_READ, MAP_PRIVATE, c->file.fd, c->file.mmap.offset)))) { 1508 log_perror(r->conf.errh, __FILE__, __LINE__, 1509 "mmap failed %s %d", c->mem->ptr, c->file.fd); 1510 1511 return -1; 1512 } 1513 1514 c->file.mmap.length = to_mmap; 1515 #ifdef HAVE_MADVISE 1516 /* don't advise files <= 64Kb */ 1517 if (c->file.mmap.length > 65536 && 1518 0 != madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED)) { 1519 log_perror(r->conf.errh, __FILE__, __LINE__, 1520 "madvise failed %s %d", c->mem->ptr, c->file.fd); 1521 } 1522 #endif 1523 1524 /* chunk_reset() or chunk_free() will cleanup for us */ 1525 } 1526 1527 /* to_send = abs_mmap_end - abs_offset */ 1528 toSend = (c->file.mmap.offset + c->file.mmap.length) - abs_offset; 1529 if (toSend > we_want_to_send) toSend = we_want_to_send; 1530 1531 if (toSend < 0) { 1532 log_error(r->conf.errh, __FILE__, __LINE__, 1533 "toSend is negative: %lld %lld %lld %lld", 1534 (long long)toSend, 1535 (long long)c->file.mmap.length, 1536 (long long)abs_offset, 1537 (long long)c->file.mmap.offset); 1538 force_assert(toSend < 0); 1539 } 1540 1541 start = c->file.mmap.start; 1542 mapped = 1; 1543 #endif 1544 1545 if (MAP_FAILED == c->file.mmap.start) { 1546 toSend = st_size; 1547 if (toSend > 2*1024*1024) toSend = 2*1024*1024; 1548 if (NULL == (start = malloc((size_t)toSend)) || -1 == lseek(c->file.fd, abs_offset, SEEK_SET) || toSend != read(c->file.fd, start, (size_t)toSend)) { 1549 log_perror(r->conf.errh, __FILE__, __LINE__, "reading %s failed", c->mem->ptr); 1550 1551 free(start); 1552 return -1; 1553 } 1554 abs_offset = 0; 1555 } 1556 1557 #ifdef USE_MMAP 1558 if (mapped) { 1559 signal(SIGBUS, sigbus_handler); 1560 sigbus_jmp_valid = 1; 1561 if (0 != sigsetjmp(sigbus_jmp, 1)) { 1562 sigbus_jmp_valid = 0; 1563 1564 log_error(r->conf.errh, __FILE__, __LINE__, 1565 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1566 return -1; 1567 } 1568 } 1569 #endif 1570 1571 if (mod_deflate_compress(hctx, 1572 (unsigned char *)start + (abs_offset - c->file.mmap.offset), toSend) < 0) { 1573 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1574 toSend = -1; 1575 } 1576 1577 #ifdef USE_MMAP 1578 if (mapped) 1579 sigbus_jmp_valid = 0; 1580 else 1581 #endif 1582 free(start); 1583 1584 return toSend; 1585 } 1586 1587 1588 static handler_t deflate_compress_response(request_st * const r, handler_ctx * const hctx) { 1589 off_t len, max; 1590 int close_stream; 1591 1592 /* move all chunk from write_queue into our in_queue, then adjust 1593 * counters since r->write_queue is reused for compressed output */ 1594 chunkqueue * const cq = &r->write_queue; 1595 len = chunkqueue_length(cq); 1596 chunkqueue_remove_finished_chunks(cq); 1597 chunkqueue_append_chunkqueue(&hctx->in_queue, cq); 1598 cq->bytes_in -= len; 1599 cq->bytes_out -= len; 1600 1601 max = chunkqueue_length(&hctx->in_queue); 1602 #if 0 1603 /* calculate max bytes to compress for this call */ 1604 if (p->conf.sync_flush && max > (len = p->conf.work_block_size << 10)) { 1605 max = len; 1606 } 1607 #endif 1608 1609 /* Compress chunks from in_queue into chunks for write_queue */ 1610 while (max) { 1611 chunk *c = hctx->in_queue.first; 1612 1613 switch(c->type) { 1614 case MEM_CHUNK: 1615 len = buffer_clen(c->mem) - c->offset; 1616 if (len > max) len = max; 1617 if (mod_deflate_compress(hctx, (unsigned char *)c->mem->ptr+c->offset, len) < 0) { 1618 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1619 return HANDLER_ERROR; 1620 } 1621 break; 1622 case FILE_CHUNK: 1623 len = c->file.length - c->offset; 1624 if (len > max) len = max; 1625 if ((len = mod_deflate_file_chunk(r, hctx, c, len)) < 0) { 1626 log_error(r->conf.errh, __FILE__, __LINE__, "compress file chunk failed."); 1627 return HANDLER_ERROR; 1628 } 1629 break; 1630 default: 1631 log_error(r->conf.errh, __FILE__, __LINE__, "%d type not known", c->type); 1632 return HANDLER_ERROR; 1633 } 1634 1635 max -= len; 1636 chunkqueue_mark_written(&hctx->in_queue, len); 1637 } 1638 1639 /*(currently should always be true)*/ 1640 /*(current implementation requires response be complete)*/ 1641 close_stream = (r->resp_body_finished 1642 && chunkqueue_is_empty(&hctx->in_queue)); 1643 if (mod_deflate_stream_flush(hctx, close_stream) < 0) { 1644 log_error(r->conf.errh, __FILE__, __LINE__, "flush error"); 1645 return HANDLER_ERROR; 1646 } 1647 1648 return close_stream ? HANDLER_FINISHED : HANDLER_GO_ON; 1649 } 1650 1651 1652 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { 1653 /* get client side support encodings */ 1654 int accept_encoding = 0; 1655 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \ 1656 && !defined(USE_ZSTD) 1657 UNUSED(value); 1658 UNUSED(label); 1659 #else 1660 for (; *value; ++value) { 1661 const char *v; 1662 while (*value == ' ' || *value == ',') ++value; 1663 v = value; 1664 while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0') 1665 ++value; 1666 switch (value - v) { 1667 case 2: 1668 #ifdef USE_BROTLI 1669 if (0 == memcmp(v, "br", 2)) 1670 accept_encoding |= HTTP_ACCEPT_ENCODING_BR; 1671 #endif 1672 break; 1673 case 4: 1674 #ifdef USE_ZLIB 1675 if (0 == memcmp(v, "gzip", 4)) 1676 accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; 1677 #endif 1678 #ifdef USE_ZSTD 1679 #ifdef USE_ZLIB 1680 else 1681 #endif 1682 if (0 == memcmp(v, "zstd", 4)) 1683 accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD; 1684 #endif 1685 break; 1686 case 5: 1687 #ifdef USE_BZ2LIB 1688 if (0 == memcmp(v, "bzip2", 5)) 1689 accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; 1690 #endif 1691 break; 1692 case 6: 1693 #ifdef USE_ZLIB 1694 if (0 == memcmp(v, "x-gzip", 6)) 1695 accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP; 1696 #endif 1697 break; 1698 case 7: 1699 #ifdef USE_ZLIB 1700 if (0 == memcmp(v, "deflate", 7)) 1701 accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; 1702 #endif 1703 #ifdef USE_BZ2LIB 1704 if (0 == memcmp(v, "x-bzip2", 7)) 1705 accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2; 1706 #endif 1707 break; 1708 #if 0 1709 case 8: 1710 if (0 == memcmp(v, "identity", 8)) 1711 accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; 1712 else if (0 == memcmp(v, "compress", 8)) 1713 accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; 1714 break; 1715 #endif 1716 default: 1717 break; 1718 } 1719 if (*value == ';') { 1720 while (*value != ',' && *value != '\0') ++value; 1721 } 1722 if (*value == '\0') break; 1723 } 1724 #endif 1725 1726 /* select best matching encoding */ 1727 const uint16_t *x = p->conf.allowed_encodings; 1728 if (NULL == x) return 0; 1729 while (*x && !(*x & accept_encoding)) ++x; 1730 accept_encoding &= *x; 1731 #ifdef USE_ZSTD 1732 if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) { 1733 *label = "zstd"; 1734 return HTTP_ACCEPT_ENCODING_ZSTD; 1735 } else 1736 #endif 1737 #ifdef USE_BROTLI 1738 if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { 1739 *label = "br"; 1740 return HTTP_ACCEPT_ENCODING_BR; 1741 } else 1742 #endif 1743 #ifdef USE_BZ2LIB 1744 if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) { 1745 *label = "bzip2"; 1746 return HTTP_ACCEPT_ENCODING_BZIP2; 1747 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_BZIP2) { 1748 *label = "x-bzip2"; 1749 return HTTP_ACCEPT_ENCODING_BZIP2; 1750 } else 1751 #endif 1752 #ifdef USE_ZLIB 1753 if (accept_encoding & HTTP_ACCEPT_ENCODING_GZIP) { 1754 *label = "gzip"; 1755 return HTTP_ACCEPT_ENCODING_GZIP; 1756 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_GZIP) { 1757 *label = "x-gzip"; 1758 return HTTP_ACCEPT_ENCODING_GZIP; 1759 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_DEFLATE) { 1760 *label = "deflate"; 1761 return HTTP_ACCEPT_ENCODING_DEFLATE; 1762 } else 1763 #endif 1764 if (0 == accept_encoding) { 1765 return 0; 1766 } else { 1767 return 0; 1768 } 1769 } 1770 1771 REQUEST_FUNC(mod_deflate_handle_response_start) { 1772 plugin_data *p = p_d; 1773 const buffer *vbro; 1774 buffer *vb; 1775 handler_ctx *hctx; 1776 const char *label; 1777 off_t len; 1778 uint32_t etaglen; 1779 int compression_type; 1780 handler_t rc; 1781 int had_vary = 0; 1782 1783 /*(current implementation requires response be complete)*/ 1784 if (!r->resp_body_finished) return HANDLER_GO_ON; 1785 if (r->http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; 1786 if (light_btst(r->resp_htags, HTTP_HEADER_TRANSFER_ENCODING)) 1787 return HANDLER_GO_ON; 1788 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_ENCODING)) 1789 return HANDLER_GO_ON; 1790 1791 /* disable compression for some http status types. */ 1792 switch(r->http_status) { 1793 case 100: 1794 case 101: 1795 case 204: 1796 case 205: 1797 case 304: 1798 /* disable compression as we have no response entity */ 1799 return HANDLER_GO_ON; 1800 default: 1801 break; 1802 } 1803 1804 mod_deflate_patch_config(r, p); 1805 1806 /* check if deflate configured for any mimetypes */ 1807 if (NULL == p->conf.mimetypes) return HANDLER_GO_ON; 1808 1809 /* check if size of response is below min-compress-size or exceeds max*/ 1810 /* (r->resp_body_finished checked at top of routine) */ 1811 len = chunkqueue_length(&r->write_queue); 1812 if (len <= (off_t)p->conf.min_compress_size) return HANDLER_GO_ON; 1813 if (p->conf.max_compress_size /*(max_compress_size in KB)*/ 1814 && len > ((off_t)p->conf.max_compress_size << 10)) { 1815 return HANDLER_GO_ON; 1816 } 1817 1818 /* Check Accept-Encoding for supported encoding. */ 1819 vbro = http_header_request_get(r, HTTP_HEADER_ACCEPT_ENCODING, CONST_STR_LEN("Accept-Encoding")); 1820 if (NULL == vbro) return HANDLER_GO_ON; 1821 1822 /* find matching encodings */ 1823 compression_type = mod_deflate_choose_encoding(vbro->ptr, p, &label); 1824 if (!compression_type) return HANDLER_GO_ON; 1825 1826 /* Check mimetype in response header "Content-Type" */ 1827 if (NULL != (vbro = http_header_response_get(r, HTTP_HEADER_CONTENT_TYPE, CONST_STR_LEN("Content-Type")))) { 1828 if (NULL == array_match_value_prefix(p->conf.mimetypes, vbro)) return HANDLER_GO_ON; 1829 } else { 1830 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */ 1831 data_string *mimetype = (data_string *)p->conf.mimetypes->data[0]; 1832 if (!buffer_is_blank(&mimetype->value)) return HANDLER_GO_ON; 1833 } 1834 1835 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */ 1836 if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) { 1837 had_vary = 1; 1838 if (!http_header_str_contains_token(BUF_PTR_LEN(vb), 1839 CONST_STR_LEN("Accept-Encoding"))) 1840 buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding")); 1841 } else { 1842 http_header_response_append(r, HTTP_HEADER_VARY, 1843 CONST_STR_LEN("Vary"), 1844 CONST_STR_LEN("Accept-Encoding")); 1845 } 1846 1847 /* check ETag as is done in http_response_handle_cachable() 1848 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */ 1849 vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag")); 1850 etaglen = vb ? buffer_clen(vb) : 0; 1851 if (etaglen && light_btst(r->rqst_htags, HTTP_HEADER_IF_NONE_MATCH)) { 1852 const buffer *if_none_match = http_header_response_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match")); 1853 if ( r->http_status < 300 /*(want 2xx only)*/ 1854 && NULL != if_none_match 1855 && 0 == strncmp(if_none_match->ptr, vb->ptr, etaglen-1) 1856 && if_none_match->ptr[etaglen-1] == '-' 1857 && 0 == strncmp(if_none_match->ptr+etaglen, label, strlen(label))) { 1858 1859 if (http_method_get_or_head(r->http_method)) { 1860 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1861 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1862 buffer_append_string(vb, label); 1863 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1864 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1865 r->http_status = 304; 1866 } else { 1867 r->http_status = 412; 1868 } 1869 1870 /* response_start hook occurs after error docs have been handled. 1871 * For now, send back empty response body. 1872 * In the future, might extract the error doc code so that it 1873 * might be run again if response_start hooks return with 1874 * changed http_status and r->handler_module = NULL */ 1875 /* clear content length even if 304 since compressed length unknown */ 1876 http_response_body_clear(r, 0); 1877 1878 r->resp_body_finished = 1; 1879 r->handler_module = NULL; 1880 return HANDLER_GO_ON; 1881 } 1882 } 1883 1884 if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < r->con->srv->loadavg[0]) { 1885 return HANDLER_GO_ON; 1886 } 1887 1888 /* update ETag, if ETag response header is set */ 1889 if (etaglen) { 1890 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1891 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1892 buffer_append_string(vb, label); 1893 buffer_append_string_len(vb, CONST_STR_LEN("\"")); 1894 /*buffer_copy_buffer(&r->physical.etag, vb);*//*(keep in sync?)*/ 1895 } 1896 1897 /* set Content-Encoding to show selected compression type */ 1898 http_header_response_set(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding"), label, strlen(label)); 1899 1900 /* clear Content-Length and r->write_queue if HTTP HEAD request 1901 * (alternatively, could return original Content-Length with HEAD 1902 * request if ETag not modified and Content-Encoding not added) 1903 * (see top of this func where short-circuit is done on HTTP HEAD) */ 1904 if (HTTP_METHOD_HEAD == r->http_method) { 1905 /* ensure that uncompressed Content-Length is not sent in HEAD response */ 1906 http_response_body_clear(r, 0); 1907 return HANDLER_GO_ON; 1908 } 1909 1910 /* restrict items eligible for cache of compressed responses 1911 * (This module does not aim to be a full caching proxy) 1912 * response must be complete (not streaming response) 1913 * must not have prior Vary response header (before Accept-Encoding added) 1914 * must have ETag 1915 * must be file 1916 * must be single FILE_CHUNK in chunkqueue 1917 * must not be chunkqueue temporary file 1918 * must be whole file, not partial content 1919 * must not be HTTP status 206 Partial Content 1920 * Note: small files (< 32k (see http_chunk.c)) will have been read into 1921 * memory (if streaming HTTP/1.1 chunked response) and will end up 1922 * getting stream-compressed rather than cached on disk as compressed 1923 * file 1924 */ 1925 buffer *tb = NULL; 1926 if (p->conf.cache_dir 1927 && !had_vary 1928 && etaglen > 2 1929 && r->resp_body_finished 1930 && r->write_queue.first == r->write_queue.last 1931 && r->write_queue.first->type == FILE_CHUNK 1932 && r->write_queue.first->offset == 0 1933 && !r->write_queue.first->file.is_temp 1934 && r->http_status != 206) { 1935 tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb); 1936 /*(checked earlier and skipped if Transfer-Encoding had been set)*/ 1937 stat_cache_entry *sce = stat_cache_get_entry_open(tb, 1); 1938 if (NULL != sce) { 1939 chunkqueue_reset(&r->write_queue); 1940 if (sce->fd < 0 || 0 != http_chunk_append_file_ref(r, sce)) 1941 return HANDLER_ERROR; 1942 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) 1943 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, 1944 CONST_STR_LEN("Content-Length")); 1945 mod_deflate_note_ratio(r, sce->st.st_size, len); 1946 return HANDLER_GO_ON; 1947 } 1948 /* sanity check that response was whole file; 1949 * (racy since using stat_cache, but cache file only if match) */ 1950 sce = stat_cache_get_entry(r->write_queue.first->mem); 1951 if (NULL == sce || sce->st.st_size != len) 1952 tb = NULL; 1953 else if (0 != mkdir_for_file(tb->ptr)) 1954 tb = NULL; 1955 } 1956 1957 /* enable compression */ 1958 p->conf.sync_flush = 1959 ((r->conf.stream_response_body 1960 & (FDEVENT_STREAM_RESPONSE | FDEVENT_STREAM_RESPONSE_BUFMIN)) 1961 && 0 == p->conf.output_buffer_size); 1962 hctx = handler_ctx_init(); 1963 hctx->plugin_data = p; 1964 hctx->compression_type = compression_type; 1965 hctx->r = r; 1966 /* setup output buffer */ 1967 buffer_clear(&p->tmp_buf); 1968 hctx->output = &p->tmp_buf; 1969 /* open cache file if caching compressed file */ 1970 if (tb) mod_deflate_cache_file_open(hctx, tb); 1971 if (0 != mod_deflate_stream_init(hctx)) { 1972 /*(should not happen unless ENOMEM)*/ 1973 handler_ctx_free(hctx); 1974 log_error(r->conf.errh, __FILE__, __LINE__, 1975 "Failed to initialize compression %s", label); 1976 /* restore prior Etag and unset Content-Encoding */ 1977 if (etaglen) { 1978 vb->ptr[etaglen-1] = '"'; /*(overwrite '-')*/ 1979 buffer_truncate(vb, etaglen); 1980 } 1981 http_header_response_unset(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding")); 1982 return HANDLER_GO_ON; 1983 } 1984 1985 #ifdef USE_BROTLI 1986 if (r->resp_body_finished 1987 && (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR) 1988 && (len >> 1) < (off_t)((~(uint32_t)0u) >> 1)) 1989 BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len); 1990 #endif 1991 1992 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) { 1993 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length")); 1994 } 1995 r->plugin_ctx[p->id] = hctx; 1996 1997 rc = deflate_compress_response(r, hctx); 1998 if (HANDLER_GO_ON == rc) return HANDLER_GO_ON; 1999 if (HANDLER_FINISHED == rc) { 2000 #ifdef __COVERITY__ 2001 /* coverity misses if hctx->cache_fd is not -1, then tb is not NULL */ 2002 force_assert(-1 == hctx->cache_fd || NULL != tb); 2003 #endif 2004 if (-1 == hctx->cache_fd 2005 || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) { 2006 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2007 rc = HANDLER_GO_ON; 2008 } 2009 else 2010 rc = HANDLER_ERROR; 2011 } 2012 r->plugin_ctx[p->id] = NULL; 2013 if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR; 2014 return rc; 2015 } 2016 2017 static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) { 2018 plugin_data *p = p_d; 2019 handler_ctx *hctx = r->plugin_ctx[p->id]; 2020 2021 if (NULL != hctx) { 2022 r->plugin_ctx[p->id] = NULL; 2023 deflate_compress_cleanup(r, hctx); 2024 } 2025 2026 return HANDLER_GO_ON; 2027 } 2028 2029 int mod_deflate_plugin_init(plugin *p); 2030 int mod_deflate_plugin_init(plugin *p) { 2031 p->version = LIGHTTPD_VERSION_ID; 2032 p->name = "deflate"; 2033 2034 p->init = mod_deflate_init; 2035 p->cleanup = mod_deflate_free; 2036 p->set_defaults = mod_deflate_set_defaults; 2037 p->handle_request_reset = mod_deflate_cleanup; 2038 p->handle_response_start = mod_deflate_handle_response_start; 2039 2040 return 0; 2041 } 2042