1 /* mod_deflate 2 * 3 * 4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch 5 * 6 * Bug fix and new features: 7 * 1) fix loop bug when content-length is bigger than work-block-size*k 8 * 9 * ------- 10 * 11 * lighttpd-1.4.26.mod_deflate.patch from 12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate 13 * 14 * ------- 15 * 16 * Patch further modified in this incarnation. 17 * 18 * Note: this patch only handles completed responses 19 * (r->resp_body_finished) 20 * this patch does not currently handle streaming dynamic responses, 21 * and therefore also does not worry about Transfer-Encoding: chunked 22 * (or having separate con->output_queue for chunked-encoded output) 23 * (or using separate buffers per connection instead of p->tmp_buf) 24 * (or handling interactions with block buffering and write timeouts) 25 * 26 * Bug fix: 27 * - fixed major bug with compressing chunks with offset > 0 28 * x-ref: 29 * "Response breaking in mod_deflate" 30 * https://redmine.lighttpd.net/issues/986 31 * - fix broken (in some cases) chunk accounting in deflate_compress_response() 32 * - fix broken bzip2 33 * x-ref: 34 * "mod_deflate's bzip2 broken by default" 35 * https://redmine.lighttpd.net/issues/2035 36 * - fix mismatch with current chunk interfaces 37 * x-ref: 38 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)" 39 * https://redmine.lighttpd.net/issues/1510 40 * 41 * Behavior changes from prior patch: 42 * - deflate.mimetypes must now be configured to enable compression 43 * deflate.mimetypes = ( ) # compress nothing (disabled; default) 44 * deflate.mimetypes = ( "" ) # compress all mimetypes 45 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes 46 * x-ref: 47 * "mod_deflate enabled by default" 48 * https://redmine.lighttpd.net/issues/1394 49 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes) 50 * - deflate.debug removed (was developer debug trace, not end-user debug) 51 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress) 52 * x-ref: 53 * "mod_deflate should allow limiting of compression algorithm from the configuration file" 54 * https://redmine.lighttpd.net/issues/996 55 * "mod_compress disabling methods" 56 * https://redmine.lighttpd.net/issues/1773 57 * - deflate.nocompress-url removed since disabling compression for a URL 58 * can now easily be done by setting to a blank list either directive 59 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional 60 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) } 61 * - deflate.sync-flush removed; controlled by r->conf.stream_response_body 62 * (though streaming compression not currently implemented in mod_deflate) 63 * - inactive directives in this patch 64 * (since r->resp_body_finished required) 65 * deflate.work-block-size 66 * deflate.output-buffer-size 67 * - remove weak file size check; SIGBUS is trapped, file that shrink will error 68 * x-ref: 69 * "mod_deflate: filesize check is too weak" 70 * https://redmine.lighttpd.net/issues/1512 71 * - change default deflate.min-compress-size from 0 to now be 256 72 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits 73 * Apache 2.4 mod_deflate minimum is 68 bytes 74 * Akamai recommends minimum 860 bytes 75 * Google recommends minimum be somewhere in range between 150 and 1024 bytes 76 * - deflate.max-compress-size new directive (in kb like compress.max_filesize) 77 * - deflate.mem-level removed (too many knobs for little benefit) 78 * - deflate.window-size removed (too many knobs for little benefit) 79 * 80 * Future: 81 * - config directives may be changed, renamed, or removed 82 * e.g. A set of reasonable defaults might be chosen 83 * instead of making them configurable. 84 * deflate.min-compress-size 85 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes 86 * to avoid compressing, even if a broader deflate.mimetypes matched, 87 * e.g. to compress all "text/" except "text/special". 88 * 89 * Implementation notes: 90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer() 91 * so that p->tmp_buf can be large and re-used. This results in an extra copy 92 * of compressed data before data is sent to network, though if the compressed 93 * size is larger than 64k, it ends up being sent to a temporary file on 94 * disk without suffering an extra copy in memory, and without extra chunk 95 * create and destroy. If this is ever changed to give away buffers, then use 96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across 97 * multiple requests being handled in parallel. 98 */ 99 #include "first.h" 100 101 #include <sys/types.h> 102 #include <sys/stat.h> 103 #include "sys-mmap.h" 104 #ifdef HAVE_MMAP 105 #include "sys-setjmp.h" 106 #endif 107 #include "sys-time.h" 108 109 #include <fcntl.h> 110 #include <stdlib.h> 111 #include <string.h> 112 #include <errno.h> 113 #include <unistd.h> /* getpid() read() unlink() write() */ 114 115 #include "base.h" 116 #include "ck.h" 117 #include "fdevent.h" 118 #include "log.h" 119 #include "buffer.h" 120 #include "http_chunk.h" 121 #include "http_etag.h" 122 #include "http_header.h" 123 #include "response.h" 124 #include "stat_cache.h" 125 126 #include "plugin.h" 127 128 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ 129 # define USE_ZLIB 130 # include <zlib.h> 131 #endif 132 #ifndef Z_DEFAULT_COMPRESSION 133 #define Z_DEFAULT_COMPRESSION -1 134 #endif 135 #ifndef MAX_WBITS 136 #define MAX_WBITS 15 137 #endif 138 139 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 140 # define USE_BZ2LIB 141 /* we don't need stdio interface */ 142 # define BZ_NO_STDIO 143 # include <bzlib.h> 144 #endif 145 146 #if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI 147 # define USE_BROTLI 148 # include <brotli/encode.h> 149 #endif 150 151 #if defined HAVE_ZSTD_H && defined HAVE_ZSTD 152 # define USE_ZSTD 153 # include <zstd.h> 154 #endif 155 156 #ifndef HAVE_LIBZ 157 #undef HAVE_LIBDEFLATE 158 #endif 159 160 /* request: accept-encoding */ 161 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) 162 #define HTTP_ACCEPT_ENCODING_GZIP BV(1) 163 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) 164 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) 165 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) 166 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) 167 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) 168 #define HTTP_ACCEPT_ENCODING_BR BV(7) 169 #define HTTP_ACCEPT_ENCODING_ZSTD BV(8) 170 171 typedef struct { 172 struct { 173 int clevel; /*(compression level)*/ 174 int windowBits; 175 int memLevel; 176 int strategy; 177 } gzip; 178 struct { 179 uint32_t quality; /*(compression level)*/ 180 uint32_t window; 181 uint32_t mode; 182 } brotli; 183 struct { 184 int clevel; /*(compression level)*/ 185 int strategy; 186 int windowLog; 187 } zstd; 188 struct { 189 int clevel; /*(compression level)*/ 190 } bzip2; 191 } encparms; 192 193 typedef struct { 194 const array *mimetypes; 195 const buffer *cache_dir; 196 unsigned int max_compress_size; 197 unsigned short min_compress_size; 198 unsigned short output_buffer_size; 199 unsigned short work_block_size; 200 unsigned short sync_flush; 201 short compression_level; 202 uint16_t * allowed_encodings; 203 double max_loadavg; 204 const encparms *params; 205 } plugin_config; 206 207 typedef struct { 208 PLUGIN_DATA; 209 plugin_config defaults; 210 plugin_config conf; 211 212 buffer tmp_buf; 213 } plugin_data; 214 215 typedef struct { 216 union { 217 #ifdef USE_ZLIB 218 z_stream z; 219 #endif 220 #ifdef USE_BZ2LIB 221 bz_stream bz; 222 #endif 223 #ifdef USE_BROTLI 224 BrotliEncoderState *br; 225 #endif 226 #ifdef USE_ZSTD 227 ZSTD_CStream *cctx; 228 #endif 229 int dummy; 230 } u; 231 off_t bytes_in; 232 off_t bytes_out; 233 buffer *output; 234 plugin_data *plugin_data; 235 request_st *r; 236 int compression_type; 237 int cache_fd; 238 char *cache_fn; 239 chunkqueue in_queue; 240 } handler_ctx; 241 242 static handler_ctx *handler_ctx_init() { 243 handler_ctx *hctx; 244 245 hctx = calloc(1, sizeof(*hctx)); 246 chunkqueue_init(&hctx->in_queue); 247 hctx->cache_fd = -1; 248 249 return hctx; 250 } 251 252 static void handler_ctx_free(handler_ctx *hctx) { 253 if (hctx->cache_fn) { 254 unlink(hctx->cache_fn); 255 free(hctx->cache_fn); 256 } 257 if (-1 != hctx->cache_fd) 258 close(hctx->cache_fd); 259 #if 0 260 if (hctx->output != &p->tmp_buf) { 261 buffer_free(hctx->output); 262 } 263 #endif 264 chunkqueue_reset(&hctx->in_queue); 265 free(hctx); 266 } 267 268 INIT_FUNC(mod_deflate_init) { 269 plugin_data * const p = calloc(1, sizeof(plugin_data)); 270 #ifdef USE_ZSTD 271 buffer_string_prepare_copy(&p->tmp_buf, ZSTD_CStreamOutSize()); 272 #else 273 buffer_string_prepare_copy(&p->tmp_buf, 65536); 274 #endif 275 return p; 276 } 277 278 FREE_FUNC(mod_deflate_free) { 279 plugin_data *p = p_d; 280 free(p->tmp_buf.ptr); 281 if (NULL == p->cvlist) return; 282 /* (init i to 0 if global context; to 1 to skip empty global context) */ 283 for (int i = !p->cvlist[0].v.u2[1], used = p->nconfig; i < used; ++i) { 284 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 285 for (; -1 != cpv->k_id; ++cpv) { 286 if (cpv->vtype != T_CONFIG_LOCAL || NULL == cpv->v.v) continue; 287 switch (cpv->k_id) { 288 case 1: /* deflate.allowed-encodings */ 289 case 14:/* deflate.params */ 290 free(cpv->v.v); 291 break; 292 default: 293 break; 294 } 295 } 296 } 297 } 298 299 #if defined(_WIN32) && !defined(__CYGWIN__) 300 #define mkdir(x,y) mkdir(x) 301 #endif 302 303 static int mkdir_for_file (char *fn) { 304 for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) { 305 if (p[1] == '\0') return 0; /* ignore trailing slash */ 306 *p = '\0'; 307 int rc = mkdir(fn, 0700); 308 *p = '/'; 309 if (0 != rc && errno != EEXIST) return -1; 310 } 311 return 0; 312 } 313 314 static int mkdir_recursive (char *dir) { 315 return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST) 316 ? 0 317 : -1; 318 } 319 320 static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) { 321 /* XXX: future: for shorter paths into the cache, we could checksum path, 322 * and then shard it to avoid a huge single directory. 323 * Alternatively, could use &r->uri.path, minus any 324 * (matching) &r->pathinfo suffix, with result url-encoded 325 * Alternative, we could shard etag which is already our "checksum" */ 326 buffer * const tb = r->tmp_buf; 327 buffer_copy_path_len2(tb, BUF_PTR_LEN(cache_dir), 328 BUF_PTR_LEN(&r->physical.path)); 329 buffer_append_str2(tb, CONST_STR_LEN("-"), /*(strip surrounding '"')*/ 330 etag->ptr+1, buffer_clen(etag)-2); 331 return tb; 332 } 333 334 static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) { 335 /* race exists whereby up to # workers might attempt to compress same 336 * file at same time if requested at same time, but this is unlikely 337 * and resolves itself by atomic rename into place when done */ 338 const uint32_t fnlen = buffer_clen(fn); 339 hctx->cache_fn = malloc(fnlen+1+LI_ITOSTRING_LENGTH+1); 340 force_assert(hctx->cache_fn); 341 memcpy(hctx->cache_fn, fn->ptr, fnlen); 342 hctx->cache_fn[fnlen] = '.'; 343 const size_t ilen = 344 li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid()); 345 hctx->cache_fn[fnlen+1+ilen] = '\0'; 346 hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600); 347 if (-1 == hctx->cache_fd) { 348 free(hctx->cache_fn); 349 hctx->cache_fn = NULL; 350 } 351 } 352 353 static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) { 354 if (0 != fdevent_rename(hctx->cache_fn, fn->ptr)) 355 return -1; 356 free(hctx->cache_fn); 357 hctx->cache_fn = NULL; 358 chunkqueue_reset(&r->write_queue); 359 int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out); 360 hctx->cache_fd = -1; 361 return rc; 362 } 363 364 static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) { 365 switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */ 366 case 0: /* deflate.mimetypes */ 367 pconf->mimetypes = cpv->v.a; 368 break; 369 case 1: /* deflate.allowed-encodings */ 370 if (cpv->vtype == T_CONFIG_LOCAL) 371 pconf->allowed_encodings = cpv->v.v; 372 break; 373 case 2: /* deflate.max-compress-size */ 374 pconf->max_compress_size = cpv->v.u; 375 break; 376 case 3: /* deflate.min-compress-size */ 377 pconf->min_compress_size = cpv->v.shrt; 378 break; 379 case 4: /* deflate.compression-level */ 380 pconf->compression_level = (short)cpv->v.shrt; 381 break; 382 case 5: /* deflate.output-buffer-size */ 383 pconf->output_buffer_size = cpv->v.shrt; 384 break; 385 case 6: /* deflate.work-block-size */ 386 pconf->work_block_size = cpv->v.shrt; 387 break; 388 case 7: /* deflate.max-loadavg */ 389 pconf->max_loadavg = cpv->v.d; 390 break; 391 case 8: /* deflate.cache-dir */ 392 pconf->cache_dir = cpv->v.b; 393 break; 394 #if 0 /*(cpv->k_id remapped in mod_deflate_set_defaults())*/ 395 case 9: /* compress.filetype */ 396 pconf->mimetypes = cpv->v.a; 397 break; 398 case 10:/* compress.allowed-encodings */ 399 if (cpv->vtype == T_CONFIG_LOCAL) 400 pconf->allowed_encodings = cpv->v.v; 401 break; 402 case 11:/* compress.cache-dir */ 403 pconf->cache_dir = cpv->v.b; 404 break; 405 case 12:/* compress.max-filesize */ 406 pconf->max_compress_size = cpv->v.u; 407 break; 408 case 13:/* compress.max-loadavg */ 409 pconf->max_loadavg = cpv->v.d; 410 break; 411 #endif 412 case 14:/* deflate.params */ 413 if (cpv->vtype == T_CONFIG_LOCAL) 414 pconf->params = cpv->v.v; 415 break; 416 default:/* should not happen */ 417 return; 418 } 419 } 420 421 static void mod_deflate_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) { 422 do { 423 mod_deflate_merge_config_cpv(pconf, cpv); 424 } while ((++cpv)->k_id != -1); 425 } 426 427 static void mod_deflate_patch_config(request_st * const r, plugin_data * const p) { 428 memcpy(&p->conf, &p->defaults, sizeof(plugin_config)); 429 for (int i = 1, used = p->nconfig; i < used; ++i) { 430 if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id)) 431 mod_deflate_merge_config(&p->conf, p->cvlist + p->cvlist[i].v.u2[0]); 432 } 433 } 434 435 static encparms * mod_deflate_parse_params(const array * const a, log_error_st * const errh) { 436 encparms * params = calloc(1, sizeof(encparms)); 437 force_assert(params); 438 439 /* set defaults */ 440 #ifdef USE_ZLIB 441 params->gzip.clevel = 0; /*(unset)*/ 442 params->gzip.windowBits = MAX_WBITS; 443 params->gzip.memLevel = 8; 444 params->gzip.strategy = Z_DEFAULT_STRATEGY; 445 #endif 446 #ifdef USE_BROTLI 447 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 448 params->brotli.quality = 5; 449 params->brotli.window = BROTLI_DEFAULT_WINDOW; 450 params->brotli.mode = BROTLI_MODE_GENERIC; 451 #endif 452 #ifdef USE_ZSTD 453 params->zstd.clevel = ZSTD_CLEVEL_DEFAULT; 454 params->zstd.strategy = 0; /*(use default strategy)*/ 455 params->zstd.windowLog = 0;/*(use default windowLog)*/ 456 #endif 457 #ifdef USE_BZ2LIB 458 params->bzip2.clevel = 0; /*(unset)*/ 459 #endif 460 461 for (uint32_t i = 0; i < a->used; ++i) { 462 const data_unset * const du = a->data[i]; 463 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 464 || defined(USE_ZSTD) 465 int32_t v = config_plugin_value_to_int32(du, -1); 466 #endif 467 #ifdef USE_BROTLI 468 if (buffer_eq_icase_slen(&du->key, 469 CONST_STR_LEN("BROTLI_PARAM_QUALITY"))) { 470 /*(future: could check for string and then look for and translate 471 * BROTLI_DEFAULT_QUALITY BROTLI_MIN_QUALITY BROTLI_MAX_QUALITY)*/ 472 if (BROTLI_MIN_QUALITY <= v && v <= BROTLI_MAX_QUALITY) 473 params->brotli.quality = (uint32_t)v; /* 0 .. 11 */ 474 else 475 log_error(errh, __FILE__, __LINE__, 476 "invalid value for BROTLI_PARAM_QUALITY"); 477 continue; 478 } 479 if (buffer_eq_icase_slen(&du->key, 480 CONST_STR_LEN("BROTLI_PARAM_LGWIN"))) { 481 /*(future: could check for string and then look for and translate 482 * BROTLI_DEFAULT_WINDOW 483 * BROTLI_MIN_WINDOW_BITS BROTLI_MAX_WINDOW_BITS)*/ 484 if (BROTLI_MIN_WINDOW_BITS <= v && v <= BROTLI_MAX_WINDOW_BITS) 485 params->brotli.window = (uint32_t)v; /* 10 .. 24 */ 486 else 487 log_error(errh, __FILE__, __LINE__, 488 "invalid value for BROTLI_PARAM_LGWIN"); 489 continue; 490 } 491 if (buffer_eq_icase_slen(&du->key, 492 CONST_STR_LEN("BROTLI_PARAM_MODE"))) { 493 /*(future: could check for string and then look for and translate 494 * BROTLI_MODE_GENERIC BROTLI_MODE_TEXT BROTLI_MODE_FONT)*/ 495 if (0 <= v && v <= 2) 496 params->brotli.mode = (uint32_t)v; /* 0 .. 2 */ 497 else 498 log_error(errh, __FILE__, __LINE__, 499 "invalid value for BROTLI_PARAM_MODE"); 500 continue; 501 } 502 #endif 503 #ifdef USE_ZSTD 504 if (buffer_eq_icase_slen(&du->key, 505 CONST_STR_LEN("ZSTD_c_compressionLevel"))) { 506 params->zstd.clevel = v; 507 /*(not warning if number parse error. future: to detect, could 508 * use absurd default to config_plugin_value_to_int32 to detect)*/ 509 continue; 510 } 511 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 512 /*(XXX: (selected) experimental API params in zstd v1.4.0)*/ 513 if (buffer_eq_icase_slen(&du->key, 514 CONST_STR_LEN("ZSTD_c_strategy"))) { 515 /*(future: could check for string and then look for and translate 516 * enum ZSTD_strategy ZSTD_STRATEGY_MIN ZSTD_STRATEGY_MAX)*/ 517 #ifndef ZSTD_STRATEGY_MIN 518 #define ZSTD_STRATEGY_MIN 1 519 #endif 520 #ifndef ZSTD_STRATEGY_MAX 521 #define ZSTD_STRATEGY_MAX 9 522 #endif 523 if (ZSTD_STRATEGY_MIN <= v && v <= ZSTD_STRATEGY_MAX) 524 params->zstd.strategy = v; /* 1 .. 9 */ 525 else 526 log_error(errh, __FILE__, __LINE__, 527 "invalid value for ZSTD_c_strategy"); 528 continue; 529 } 530 if (buffer_eq_icase_slen(&du->key, 531 CONST_STR_LEN("ZSTD_c_windowLog"))) { 532 /*(future: could check for string and then look for and translate 533 * ZSTD_WINDOWLOG_MIN ZSTD_WINDOWLOG_MAX)*/ 534 #ifndef ZSTD_WINDOWLOG_MIN 535 #define ZSTD_WINDOWLOG_MIN 10 536 #endif 537 #ifndef ZSTD_WINDOWLOG_MAX_32 538 #define ZSTD_WINDOWLOG_MAX_32 30 539 #endif 540 #ifndef ZSTD_WINDOWLOG_MAX_64 541 #define ZSTD_WINDOWLOG_MAX_64 31 542 #endif 543 #ifndef ZSTD_WINDOWLOG_MAX 544 #define ZSTD_WINDOWLOG_MAX \ 545 (sizeof(size_t)==4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64) 546 #endif 547 if (ZSTD_WINDOWLOG_MIN <= v && v <= ZSTD_WINDOWLOG_MAX) 548 params->zstd.windowLog = v;/* 10 .. 31 *//*(30 max for 32-bit)*/ 549 else 550 log_error(errh, __FILE__, __LINE__, 551 "invalid value for ZSTD_c_windowLog"); 552 continue; 553 } 554 #endif 555 #endif 556 #ifdef USE_ZLIB 557 if (buffer_eq_icase_slen(&du->key, 558 CONST_STR_LEN("gzip.level"))) { 559 if (1 <= v && v <= 9) 560 params->gzip.clevel = v; /* 1 .. 9 */ 561 else 562 log_error(errh, __FILE__, __LINE__, 563 "invalid value for gzip.level"); 564 continue; 565 } 566 if (buffer_eq_icase_slen(&du->key, 567 CONST_STR_LEN("gzip.windowBits"))) { 568 if (9 <= v && v <= 15) 569 params->gzip.windowBits = v; /* 9 .. 15 */ 570 else 571 log_error(errh, __FILE__, __LINE__, 572 "invalid value for gzip.windowBits"); 573 continue; 574 } 575 if (buffer_eq_icase_slen(&du->key, 576 CONST_STR_LEN("gzip.memLevel"))) { 577 if (1 <= v && v <= 9) 578 params->gzip.memLevel = v; /* 1 .. 9 */ 579 else 580 log_error(errh, __FILE__, __LINE__, 581 "invalid value for gzip.memLevel"); 582 continue; 583 } 584 if (buffer_eq_icase_slen(&du->key, 585 CONST_STR_LEN("gzip.strategy"))) { 586 /*(future: could check for string and then look for and translate 587 * Z_DEFAULT_STRATEGY Z_FILTERED Z_HUFFMAN_ONLY Z_RLE Z_FIXED)*/ 588 if (0 <= v && v <= 4) 589 params->gzip.strategy = v; /* 0 .. 4 */ 590 else 591 log_error(errh, __FILE__, __LINE__, 592 "invalid value for gzip.strategy"); 593 continue; 594 } 595 #endif 596 #ifdef USE_BZ2LIB 597 if (buffer_eq_icase_slen(&du->key, 598 CONST_STR_LEN("bzip2.blockSize100k"))) { 599 if (1 <= v && v <= 9) /*(bzip2 blockSize100k param)*/ 600 params->bzip2.clevel = v; /* 1 .. 9 */ 601 else 602 log_error(errh, __FILE__, __LINE__, 603 "invalid value for bzip2.blockSize100k"); 604 continue; 605 } 606 #endif 607 log_error(errh, __FILE__, __LINE__, 608 "unrecognized param: %s", du->key.ptr); 609 } 610 611 return params; 612 } 613 614 static uint16_t * mod_deflate_encodings_to_flags(const array *encodings) { 615 if (encodings->used) { 616 uint16_t * const x = calloc(encodings->used+1, sizeof(short)); 617 force_assert(x); 618 int i = 0; 619 for (uint32_t j = 0; j < encodings->used; ++j) { 620 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 621 || defined(USE_ZSTD) 622 data_string *ds = (data_string *)encodings->data[j]; 623 #endif 624 #ifdef USE_ZLIB /* "gzip", "x-gzip" */ 625 if (NULL != strstr(ds->value.ptr, "gzip")) 626 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 627 | HTTP_ACCEPT_ENCODING_X_GZIP; 628 if (NULL != strstr(ds->value.ptr, "deflate")) 629 x[i++] = HTTP_ACCEPT_ENCODING_DEFLATE; 630 /* 631 if (NULL != strstr(ds->value.ptr, "compress")) 632 x[i++] = HTTP_ACCEPT_ENCODING_COMPRESS; 633 */ 634 #endif 635 #ifdef USE_BZ2LIB /* "bzip2", "x-bzip2" */ 636 if (NULL != strstr(ds->value.ptr, "bzip2")) 637 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 638 | HTTP_ACCEPT_ENCODING_X_BZIP2; 639 #endif 640 #ifdef USE_BROTLI /* "br" (also accepts "brotli") */ 641 if (NULL != strstr(ds->value.ptr, "br")) 642 x[i++] = HTTP_ACCEPT_ENCODING_BR; 643 #endif 644 #ifdef USE_ZSTD 645 if (NULL != strstr(ds->value.ptr, "zstd")) 646 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 647 #endif 648 } 649 x[i] = 0; /* end of list */ 650 return x; 651 } 652 else { 653 /* default encodings */ 654 uint16_t * const x = calloc(4+1, sizeof(short)); 655 force_assert(x); 656 int i = 0; 657 #ifdef USE_ZSTD 658 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 659 #endif 660 #ifdef USE_BROTLI 661 x[i++] = HTTP_ACCEPT_ENCODING_BR; 662 #endif 663 #ifdef USE_BZ2LIB 664 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 665 | HTTP_ACCEPT_ENCODING_X_BZIP2; 666 #endif 667 #ifdef USE_ZLIB 668 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 669 | HTTP_ACCEPT_ENCODING_X_GZIP 670 | HTTP_ACCEPT_ENCODING_DEFLATE; 671 #endif 672 x[i] = 0; /* end of list */ 673 return x; 674 } 675 } 676 677 SETDEFAULTS_FUNC(mod_deflate_set_defaults) { 678 static const config_plugin_keys_t cpk[] = { 679 { CONST_STR_LEN("deflate.mimetypes"), 680 T_CONFIG_ARRAY_VLIST, 681 T_CONFIG_SCOPE_CONNECTION } 682 ,{ CONST_STR_LEN("deflate.allowed-encodings"), 683 T_CONFIG_ARRAY_VLIST, 684 T_CONFIG_SCOPE_CONNECTION } 685 ,{ CONST_STR_LEN("deflate.max-compress-size"), 686 T_CONFIG_INT, 687 T_CONFIG_SCOPE_CONNECTION } 688 ,{ CONST_STR_LEN("deflate.min-compress-size"), 689 T_CONFIG_SHORT, 690 T_CONFIG_SCOPE_CONNECTION } 691 ,{ CONST_STR_LEN("deflate.compression-level"), 692 T_CONFIG_SHORT, 693 T_CONFIG_SCOPE_CONNECTION } 694 ,{ CONST_STR_LEN("deflate.output-buffer-size"), 695 T_CONFIG_SHORT, 696 T_CONFIG_SCOPE_CONNECTION } 697 ,{ CONST_STR_LEN("deflate.work-block-size"), 698 T_CONFIG_SHORT, 699 T_CONFIG_SCOPE_CONNECTION } 700 ,{ CONST_STR_LEN("deflate.max-loadavg"), 701 T_CONFIG_STRING, 702 T_CONFIG_SCOPE_CONNECTION } 703 ,{ CONST_STR_LEN("deflate.cache-dir"), 704 T_CONFIG_STRING, 705 T_CONFIG_SCOPE_CONNECTION } 706 ,{ CONST_STR_LEN("compress.filetype"), 707 T_CONFIG_ARRAY_VLIST, 708 T_CONFIG_SCOPE_CONNECTION } 709 ,{ CONST_STR_LEN("compress.allowed-encodings"), 710 T_CONFIG_ARRAY_VLIST, 711 T_CONFIG_SCOPE_CONNECTION } 712 ,{ CONST_STR_LEN("compress.cache-dir"), 713 T_CONFIG_STRING, 714 T_CONFIG_SCOPE_CONNECTION } 715 ,{ CONST_STR_LEN("compress.max-filesize"), 716 T_CONFIG_INT, 717 T_CONFIG_SCOPE_CONNECTION } 718 ,{ CONST_STR_LEN("compress.max-loadavg"), 719 T_CONFIG_STRING, 720 T_CONFIG_SCOPE_CONNECTION } 721 ,{ CONST_STR_LEN("deflate.params"), 722 T_CONFIG_ARRAY_KVANY, 723 T_CONFIG_SCOPE_CONNECTION } 724 ,{ NULL, 0, 725 T_CONFIG_UNSET, 726 T_CONFIG_SCOPE_UNSET } 727 }; 728 729 plugin_data * const p = p_d; 730 if (!config_plugin_values_init(srv, p, cpk, "mod_deflate")) 731 return HANDLER_ERROR; 732 733 /* process and validate config directives 734 * (init i to 0 if global context; to 1 to skip empty global context) */ 735 for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) { 736 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 737 for (; -1 != cpv->k_id; ++cpv) { 738 switch (cpv->k_id) { 739 case 9: /* compress.filetype */ 740 log_error(srv->errh, __FILE__, __LINE__, 741 "DEPRECATED: %s replaced with deflate.mimetypes", 742 cpk[cpv->k_id].k); 743 cpv->k_id = 0; /* deflate.mimetypes */ 744 __attribute_fallthrough__ 745 case 0: /* deflate.mimetypes */ 746 /* mod_deflate matches mimetype as prefix of Content-Type 747 * so ignore '*' at end of mimetype for end-user flexibility 748 * in specifying trailing wildcard to grouping of mimetypes */ 749 for (uint32_t m = 0; m < cpv->v.a->used; ++m) { 750 buffer *mimetype=&((data_string *)cpv->v.a->data[m])->value; 751 size_t len = buffer_clen(mimetype); 752 if (len > 2 && mimetype->ptr[len-1] == '*') 753 buffer_truncate(mimetype, len-1); 754 if (buffer_eq_slen(mimetype, 755 CONST_STR_LEN("application/javascript"))) 756 buffer_copy_string_len(mimetype, "text/javascript", 15); 757 } 758 if (0 == cpv->v.a->used) cpv->v.a = NULL; 759 break; 760 case 10:/* compress.allowed-encodings */ 761 log_error(srv->errh, __FILE__, __LINE__, 762 "DEPRECATED: %s replaced with deflate.allowed-encodings", 763 cpk[cpv->k_id].k); 764 cpv->k_id = 1; /* deflate.allowed-encodings */ 765 __attribute_fallthrough__ 766 case 1: /* deflate.allowed-encodings */ 767 cpv->v.v = mod_deflate_encodings_to_flags(cpv->v.a); 768 cpv->vtype = T_CONFIG_LOCAL; 769 break; 770 case 12:/* compress.max-filesize */ 771 log_error(srv->errh, __FILE__, __LINE__, 772 "DEPRECATED: %s replaced with deflate.max-compress-size", 773 cpk[cpv->k_id].k); 774 cpv->k_id = 2; /* deflate.max-compress-size */ 775 __attribute_fallthrough__ 776 case 2: /* deflate.max-compress-size */ 777 case 3: /* deflate.min-compress-size */ 778 break; 779 case 4: /* deflate.compression-level */ 780 if ((cpv->v.shrt < 1 || cpv->v.shrt > 9) 781 && *(short *)&cpv->v.shrt != -1) { 782 log_error(srv->errh, __FILE__, __LINE__, 783 "compression-level must be between 1 and 9: %hu", 784 cpv->v.shrt); 785 return HANDLER_ERROR; 786 } 787 break; 788 case 5: /* deflate.output-buffer-size */ 789 case 6: /* deflate.work-block-size */ 790 break; 791 case 13:/* compress.max-loadavg */ 792 log_error(srv->errh, __FILE__, __LINE__, 793 "DEPRECATED: %s replaced with deflate.max-loadavg", 794 cpk[cpv->k_id].k); 795 cpv->k_id = 7; /* deflate.max-loadavg */ 796 __attribute_fallthrough__ 797 case 7: /* deflate.max-loadavg */ 798 cpv->v.d = (!buffer_is_blank(cpv->v.b)) 799 ? strtod(cpv->v.b->ptr, NULL) 800 : 0.0; 801 break; 802 case 11:/* compress.cache-dir */ 803 log_error(srv->errh, __FILE__, __LINE__, 804 "DEPRECATED: %s replaced with deflate.cache-dir", 805 cpk[cpv->k_id].k); 806 cpv->k_id = 8; /* deflate.cache-dir */ 807 __attribute_fallthrough__ 808 case 8: /* deflate.cache-dir */ 809 if (!buffer_is_blank(cpv->v.b)) { 810 buffer *b; 811 *(const buffer **)&b = cpv->v.b; 812 const uint32_t len = buffer_clen(b); 813 if (len > 0 && '/' == b->ptr[len-1]) 814 buffer_truncate(b, len-1); /*remove end slash*/ 815 struct stat st; 816 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 817 log_perror(srv->errh, __FILE__, __LINE__, 818 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 819 return HANDLER_ERROR; 820 } 821 } 822 else 823 cpv->v.b = NULL; 824 break; 825 #if 0 /*(handled further above)*/ 826 case 9: /* compress.filetype */ 827 case 10:/* compress.allowed-encodings */ 828 case 11:/* compress.cache-dir */ 829 case 12:/* compress.max-filesize */ 830 case 13:/* compress.max-loadavg */ 831 break; 832 #endif 833 case 14:/* deflate.params */ 834 cpv->v.v = mod_deflate_parse_params(cpv->v.a, srv->errh); 835 cpv->vtype = T_CONFIG_LOCAL; 836 break; 837 default:/* should not happen */ 838 break; 839 } 840 } 841 } 842 843 p->defaults.max_compress_size = 128*1024; /*(128 MB measured as num KB)*/ 844 p->defaults.min_compress_size = 256; 845 p->defaults.compression_level = -1; 846 p->defaults.output_buffer_size = 0; 847 p->defaults.work_block_size = 2048; 848 p->defaults.max_loadavg = 0.0; 849 p->defaults.sync_flush = 0; 850 851 /* initialize p->defaults from global config context */ 852 if (p->nconfig > 0 && p->cvlist->v.u2[1]) { 853 const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0]; 854 if (-1 != cpv->k_id) 855 mod_deflate_merge_config(&p->defaults, cpv); 856 } 857 858 return HANDLER_GO_ON; 859 } 860 861 862 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 863 || defined(USE_ZSTD) 864 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { 865 ssize_t wr; 866 do { 867 wr = write(hctx->cache_fd, out, len); 868 } while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR); 869 return (0 == len) ? 0 : -1; 870 } 871 872 static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) { 873 if (0 == len) return 0; 874 return (-1 == hctx->cache_fd) 875 ? http_chunk_append_mem(hctx->r, out, len) 876 : mod_deflate_cache_file_append(hctx, out, len); 877 } 878 #endif 879 880 881 #ifdef USE_ZLIB 882 883 static int stream_deflate_init(handler_ctx *hctx) { 884 z_stream * const z = &hctx->u.z; 885 z->zalloc = Z_NULL; 886 z->zfree = Z_NULL; 887 z->opaque = Z_NULL; 888 z->total_in = 0; 889 z->total_out = 0; 890 z->next_out = (unsigned char *)hctx->output->ptr; 891 z->avail_out = hctx->output->size; 892 893 const plugin_data * const p = hctx->plugin_data; 894 const encparms * const params = p->conf.params; 895 const int clevel = (NULL != params) 896 ? params->gzip.clevel 897 : p->conf.compression_level; 898 const int wbits = (NULL != params) 899 ? params->gzip.windowBits 900 : MAX_WBITS; 901 902 if (Z_OK != deflateInit2(z, 903 clevel > 0 ? clevel : Z_DEFAULT_COMPRESSION, 904 Z_DEFLATED, 905 (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 906 ? (wbits | 16) /*(0x10 flags gzip header, trailer)*/ 907 : -wbits, /*(negate to suppress zlib header)*/ 908 params ? params->gzip.memLevel : 8,/*default memLevel*/ 909 params ? params->gzip.strategy : Z_DEFAULT_STRATEGY)) { 910 return -1; 911 } 912 913 return 0; 914 } 915 916 static int stream_deflate_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 917 z_stream * const z = &(hctx->u.z); 918 size_t len; 919 920 /*(unknown whether or not linked zlib was built with ZLIB_CONST defined)*/ 921 *((const unsigned char **)&z->next_in) = start; 922 z->avail_in = st_size; 923 hctx->bytes_in += st_size; 924 925 /* compress data */ 926 do { 927 if (Z_OK != deflate(z, Z_NO_FLUSH)) return -1; 928 929 if (z->avail_out == 0 || z->avail_in > 0) { 930 len = hctx->output->size - z->avail_out; 931 hctx->bytes_out += len; 932 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 933 return -1; 934 z->next_out = (unsigned char *)hctx->output->ptr; 935 z->avail_out = hctx->output->size; 936 } 937 } while (z->avail_in > 0); 938 939 return 0; 940 } 941 942 static int stream_deflate_flush(handler_ctx * const hctx, int end) { 943 z_stream * const z = &(hctx->u.z); 944 const plugin_data *p = hctx->plugin_data; 945 size_t len; 946 int rc = 0; 947 int done; 948 949 /* compress data */ 950 do { 951 done = 1; 952 if (end) { 953 rc = deflate(z, Z_FINISH); 954 if (rc == Z_OK) { 955 done = 0; 956 } else if (rc != Z_STREAM_END) { 957 return -1; 958 } 959 } else { 960 if (p->conf.sync_flush) { 961 rc = deflate(z, Z_SYNC_FLUSH); 962 if (rc != Z_OK) return -1; 963 } else if (z->avail_in > 0) { 964 rc = deflate(z, Z_NO_FLUSH); 965 if (rc != Z_OK) return -1; 966 } 967 } 968 969 len = hctx->output->size - z->avail_out; 970 if (z->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 971 hctx->bytes_out += len; 972 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 973 return -1; 974 z->next_out = (unsigned char *)hctx->output->ptr; 975 z->avail_out = hctx->output->size; 976 } 977 } while (z->avail_in != 0 || !done); 978 979 return 0; 980 } 981 982 static int stream_deflate_end(handler_ctx *hctx) { 983 z_stream * const z = &(hctx->u.z); 984 int rc = deflateEnd(z); 985 if (Z_OK == rc || Z_DATA_ERROR == rc) return 0; 986 987 if (z->msg != NULL) { 988 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 989 "deflateEnd error ret=%d, msg=%s", rc, z->msg); 990 } else { 991 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 992 "deflateEnd error ret=%d", rc); 993 } 994 return -1; 995 } 996 997 #endif 998 999 1000 #ifdef USE_BZ2LIB 1001 1002 static int stream_bzip2_init(handler_ctx *hctx) { 1003 bz_stream * const bz = &hctx->u.bz; 1004 bz->bzalloc = NULL; 1005 bz->bzfree = NULL; 1006 bz->opaque = NULL; 1007 bz->total_in_lo32 = 0; 1008 bz->total_in_hi32 = 0; 1009 bz->total_out_lo32 = 0; 1010 bz->total_out_hi32 = 0; 1011 bz->next_out = hctx->output->ptr; 1012 bz->avail_out = hctx->output->size; 1013 1014 const plugin_data * const p = hctx->plugin_data; 1015 const encparms * const params = p->conf.params; 1016 const int clevel = (NULL != params) 1017 ? params->bzip2.clevel 1018 : p->conf.compression_level; 1019 1020 if (BZ_OK != BZ2_bzCompressInit(bz, 1021 clevel > 0 1022 ? p->conf.compression_level 1023 : 9, /* blocksize = 900k */ 1024 0, /* verbosity */ 1025 0)) { /* workFactor: default */ 1026 return -1; 1027 } 1028 1029 return 0; 1030 } 1031 1032 static int stream_bzip2_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1033 bz_stream * const bz = &(hctx->u.bz); 1034 size_t len; 1035 1036 bz->next_in = (char *)start; 1037 bz->avail_in = st_size; 1038 hctx->bytes_in += st_size; 1039 1040 /* compress data */ 1041 do { 1042 if (BZ_RUN_OK != BZ2_bzCompress(bz, BZ_RUN)) return -1; 1043 1044 if (bz->avail_out == 0 || bz->avail_in > 0) { 1045 len = hctx->output->size - bz->avail_out; 1046 hctx->bytes_out += len; 1047 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1048 return -1; 1049 bz->next_out = hctx->output->ptr; 1050 bz->avail_out = hctx->output->size; 1051 } 1052 } while (bz->avail_in > 0); 1053 1054 return 0; 1055 } 1056 1057 static int stream_bzip2_flush(handler_ctx * const hctx, int end) { 1058 bz_stream * const bz = &(hctx->u.bz); 1059 const plugin_data *p = hctx->plugin_data; 1060 size_t len; 1061 int rc; 1062 int done; 1063 1064 /* compress data */ 1065 do { 1066 done = 1; 1067 if (end) { 1068 rc = BZ2_bzCompress(bz, BZ_FINISH); 1069 if (rc == BZ_FINISH_OK) { 1070 done = 0; 1071 } else if (rc != BZ_STREAM_END) { 1072 return -1; 1073 } 1074 } else if (bz->avail_in > 0) { 1075 /* p->conf.sync_flush not implemented here, 1076 * which would loop on BZ_FLUSH while BZ_FLUSH_OK 1077 * until BZ_RUN_OK returned */ 1078 rc = BZ2_bzCompress(bz, BZ_RUN); 1079 if (rc != BZ_RUN_OK) { 1080 return -1; 1081 } 1082 } 1083 1084 len = hctx->output->size - bz->avail_out; 1085 if (bz->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 1086 hctx->bytes_out += len; 1087 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1088 return -1; 1089 bz->next_out = hctx->output->ptr; 1090 bz->avail_out = hctx->output->size; 1091 } 1092 } while (bz->avail_in != 0 || !done); 1093 1094 return 0; 1095 } 1096 1097 static int stream_bzip2_end(handler_ctx *hctx) { 1098 bz_stream * const bz = &(hctx->u.bz); 1099 int rc = BZ2_bzCompressEnd(bz); 1100 if (BZ_OK == rc || BZ_DATA_ERROR == rc) return 0; 1101 1102 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 1103 "BZ2_bzCompressEnd error ret=%d", rc); 1104 return -1; 1105 } 1106 1107 #endif 1108 1109 1110 #ifdef USE_BROTLI 1111 1112 static int stream_br_init(handler_ctx *hctx) { 1113 BrotliEncoderState * const br = hctx->u.br = 1114 BrotliEncoderCreateInstance(NULL, NULL, NULL); 1115 if (NULL == br) return -1; 1116 1117 /*(note: we ignore any errors while tuning parameters here)*/ 1118 const plugin_data * const p = hctx->plugin_data; 1119 const encparms * const params = p->conf.params; 1120 const uint32_t quality = (NULL != params) 1121 ? params->brotli.quality 1122 : (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */ 1123 ? (uint32_t)p->conf.compression_level 1124 : 5; 1125 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 1126 if (quality != BROTLI_DEFAULT_QUALITY) 1127 BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY, quality); 1128 1129 if (params && params->brotli.window != BROTLI_DEFAULT_WINDOW) 1130 BrotliEncoderSetParameter(br, BROTLI_PARAM_LGWIN,params->brotli.window); 1131 1132 const buffer *vb; 1133 if (params && params->brotli.mode != BROTLI_MODE_GENERIC) 1134 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, params->brotli.mode); 1135 else if ((vb = http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE, 1136 CONST_STR_LEN("Content-Type")))) { 1137 /* BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */ 1138 const uint32_t len = buffer_clen(vb); 1139 if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1) 1140 || (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1) 1141 && (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1) 1142 || 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1) 1143 || 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1))) 1144 || (len > 4 1145 && (0 == strncmp(vb->ptr+len-5, "+json", sizeof("+json")-1) 1146 || 0 == strncmp(vb->ptr+len-4, "+xml", sizeof("+xml")-1)))) 1147 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); 1148 else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1)) 1149 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT); 1150 } 1151 1152 return 0; 1153 } 1154 1155 static int stream_br_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1156 const uint8_t *in = (uint8_t *)start; 1157 BrotliEncoderState * const br = hctx->u.br; 1158 hctx->bytes_in += st_size; 1159 while (st_size || BrotliEncoderHasMoreOutput(br)) { 1160 size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size) 1161 ? (size_t)st_size 1162 : ((~(uint32_t)0) >> 1); 1163 size_t outsz = 0; 1164 BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS, 1165 &insz, &in, &outsz, NULL, NULL); 1166 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1167 st_size -= (st_size - (off_t)insz); 1168 if (outsz) { 1169 hctx->bytes_out += (off_t)outsz; 1170 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1171 return -1; 1172 } 1173 } 1174 return 0; 1175 } 1176 1177 static int stream_br_flush(handler_ctx * const hctx, int end) { 1178 const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; 1179 BrotliEncoderState * const br = hctx->u.br; 1180 do { 1181 size_t insz = 0; 1182 size_t outsz = 0; 1183 BrotliEncoderCompressStream(br, brmode, 1184 &insz, NULL, &outsz, NULL, NULL); 1185 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1186 if (outsz) { 1187 hctx->bytes_out += (off_t)outsz; 1188 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1189 return -1; 1190 } 1191 } while (BrotliEncoderHasMoreOutput(br)); 1192 return 0; 1193 } 1194 1195 static int stream_br_end(handler_ctx *hctx) { 1196 BrotliEncoderState * const br = hctx->u.br; 1197 BrotliEncoderDestroyInstance(br); 1198 return 0; 1199 } 1200 1201 #endif 1202 1203 1204 #ifdef USE_ZSTD 1205 1206 static int stream_zstd_init(handler_ctx *hctx) { 1207 ZSTD_CStream * const cctx = hctx->u.cctx = ZSTD_createCStream(); 1208 if (NULL == cctx) return -1; 1209 hctx->output->used = 0; 1210 1211 /*(note: we ignore any errors while tuning parameters here)*/ 1212 const plugin_data * const p = hctx->plugin_data; 1213 const encparms * const params = p->conf.params; 1214 if (params) { 1215 if (params->zstd.clevel && params->zstd.clevel != ZSTD_CLEVEL_DEFAULT) { 1216 const int level = params->zstd.clevel; 1217 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1218 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); 1219 #else 1220 ZSTD_initCStream(cctx, level); 1221 #endif 1222 } 1223 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1224 if (params->zstd.strategy) 1225 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, 1226 params->zstd.strategy); 1227 if (params->zstd.windowLog) 1228 ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 1229 params->zstd.windowLog); 1230 #endif 1231 } 1232 else if (p->conf.compression_level >= 0) { /* -1 here is "unset" */ 1233 int level = p->conf.compression_level; 1234 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1235 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, level); 1236 #else 1237 ZSTD_initCStream(cctx, level); 1238 #endif 1239 } 1240 return 0; 1241 } 1242 1243 static int stream_zstd_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1244 ZSTD_CStream * const cctx = hctx->u.cctx; 1245 ZSTD_inBuffer zib = { start, (size_t)st_size, 0 }; 1246 ZSTD_outBuffer zob = { hctx->output->ptr, 1247 hctx->output->size, 1248 hctx->output->used }; 1249 hctx->output->used = 0; 1250 hctx->bytes_in += st_size; 1251 while (zib.pos < zib.size) { 1252 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1253 const size_t rv = ZSTD_compressStream2(cctx,&zob,&zib,ZSTD_e_continue); 1254 #else 1255 const size_t rv = ZSTD_compressStream(cctx, &zob, &zib); 1256 #endif 1257 if (ZSTD_isError(rv)) return -1; 1258 if (zib.pos == zib.size) break; /* defer flush */ 1259 hctx->bytes_out += (off_t)zob.pos; 1260 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1261 return -1; 1262 zob.pos = 0; 1263 } 1264 hctx->output->used = (uint32_t)zob.pos; 1265 return 0; 1266 } 1267 1268 static int stream_zstd_flush(handler_ctx * const hctx, int end) { 1269 ZSTD_CStream * const cctx = hctx->u.cctx; 1270 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1271 const ZSTD_EndDirective endOp = end ? ZSTD_e_end : ZSTD_e_flush; 1272 ZSTD_inBuffer zib = { NULL, 0, 0 }; 1273 #endif 1274 ZSTD_outBuffer zob = { hctx->output->ptr, 1275 hctx->output->size, 1276 hctx->output->used }; 1277 size_t rv; 1278 do { 1279 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1280 rv = ZSTD_compressStream2(cctx, &zob, &zib, endOp); 1281 #else 1282 rv = end 1283 ? ZSTD_endStream(cctx, &zob) 1284 : ZSTD_flushStream(cctx, &zob); 1285 #endif 1286 if (ZSTD_isError(rv)) return -1; 1287 hctx->bytes_out += (off_t)zob.pos; 1288 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1289 return -1; 1290 zob.pos = 0; 1291 } while (0 != rv); 1292 return 0; 1293 } 1294 1295 static int stream_zstd_end(handler_ctx *hctx) { 1296 ZSTD_CStream * const cctx = hctx->u.cctx; 1297 ZSTD_freeCStream(cctx); 1298 return 0; 1299 } 1300 1301 #endif 1302 1303 1304 static int mod_deflate_stream_init(handler_ctx *hctx) { 1305 switch(hctx->compression_type) { 1306 #ifdef USE_ZLIB 1307 case HTTP_ACCEPT_ENCODING_GZIP: 1308 case HTTP_ACCEPT_ENCODING_DEFLATE: 1309 return stream_deflate_init(hctx); 1310 #endif 1311 #ifdef USE_BZ2LIB 1312 case HTTP_ACCEPT_ENCODING_BZIP2: 1313 return stream_bzip2_init(hctx); 1314 #endif 1315 #ifdef USE_BROTLI 1316 case HTTP_ACCEPT_ENCODING_BR: 1317 return stream_br_init(hctx); 1318 #endif 1319 #ifdef USE_ZSTD 1320 case HTTP_ACCEPT_ENCODING_ZSTD: 1321 return stream_zstd_init(hctx); 1322 #endif 1323 default: 1324 return -1; 1325 } 1326 } 1327 1328 static int mod_deflate_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1329 if (0 == st_size) return 0; 1330 switch(hctx->compression_type) { 1331 #ifdef USE_ZLIB 1332 case HTTP_ACCEPT_ENCODING_GZIP: 1333 case HTTP_ACCEPT_ENCODING_DEFLATE: 1334 return stream_deflate_compress(hctx, start, st_size); 1335 #endif 1336 #ifdef USE_BZ2LIB 1337 case HTTP_ACCEPT_ENCODING_BZIP2: 1338 return stream_bzip2_compress(hctx, start, st_size); 1339 #endif 1340 #ifdef USE_BROTLI 1341 case HTTP_ACCEPT_ENCODING_BR: 1342 return stream_br_compress(hctx, start, st_size); 1343 #endif 1344 #ifdef USE_ZSTD 1345 case HTTP_ACCEPT_ENCODING_ZSTD: 1346 return stream_zstd_compress(hctx, start, st_size); 1347 #endif 1348 default: 1349 UNUSED(start); 1350 return -1; 1351 } 1352 } 1353 1354 static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { 1355 if (0 == hctx->bytes_in) return 0; 1356 switch(hctx->compression_type) { 1357 #ifdef USE_ZLIB 1358 case HTTP_ACCEPT_ENCODING_GZIP: 1359 case HTTP_ACCEPT_ENCODING_DEFLATE: 1360 return stream_deflate_flush(hctx, end); 1361 #endif 1362 #ifdef USE_BZ2LIB 1363 case HTTP_ACCEPT_ENCODING_BZIP2: 1364 return stream_bzip2_flush(hctx, end); 1365 #endif 1366 #ifdef USE_BROTLI 1367 case HTTP_ACCEPT_ENCODING_BR: 1368 return stream_br_flush(hctx, end); 1369 #endif 1370 #ifdef USE_ZSTD 1371 case HTTP_ACCEPT_ENCODING_ZSTD: 1372 return stream_zstd_flush(hctx, end); 1373 #endif 1374 default: 1375 UNUSED(end); 1376 return -1; 1377 } 1378 } 1379 1380 static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) { 1381 /* store compression ratio in environment 1382 * for possible logging by mod_accesslog 1383 * (late in response handling, so not seen by most other modules) */ 1384 /*(should be called only at end of successful response compression)*/ 1385 if (0 == bytes_in) return; 1386 buffer_append_int( 1387 http_header_env_set_ptr(r, CONST_STR_LEN("ratio")), 1388 bytes_out * 100 / bytes_in); 1389 } 1390 1391 static int mod_deflate_stream_end(handler_ctx *hctx) { 1392 switch(hctx->compression_type) { 1393 #ifdef USE_ZLIB 1394 case HTTP_ACCEPT_ENCODING_GZIP: 1395 case HTTP_ACCEPT_ENCODING_DEFLATE: 1396 return stream_deflate_end(hctx); 1397 #endif 1398 #ifdef USE_BZ2LIB 1399 case HTTP_ACCEPT_ENCODING_BZIP2: 1400 return stream_bzip2_end(hctx); 1401 #endif 1402 #ifdef USE_BROTLI 1403 case HTTP_ACCEPT_ENCODING_BR: 1404 return stream_br_end(hctx); 1405 #endif 1406 #ifdef USE_ZSTD 1407 case HTTP_ACCEPT_ENCODING_ZSTD: 1408 return stream_zstd_end(hctx); 1409 #endif 1410 default: 1411 return -1; 1412 } 1413 } 1414 1415 static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) { 1416 int rc = mod_deflate_stream_end(hctx); 1417 1418 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */ 1419 if (0 == rc && hctx->bytes_in < hctx->bytes_out) 1420 log_error(r->conf.errh, __FILE__, __LINE__, 1421 "uri %s in=%lld smaller than out=%lld", r->target.ptr, 1422 (long long)hctx->bytes_in, (long long)hctx->bytes_out); 1423 #endif 1424 1425 handler_ctx_free(hctx); 1426 return rc; 1427 } 1428 1429 1430 #ifdef HAVE_LIBDEFLATE 1431 #include <libdeflate.h> 1432 1433 __attribute_cold__ 1434 __attribute_noinline__ 1435 static int mod_deflate_using_libdeflate_err (handler_ctx * const hctx, buffer * const fn, const int fd) 1436 { 1437 if (-1 == fd) { 1438 } 1439 else if (fd == hctx->cache_fd) { 1440 if (0 != ftruncate(fd, 0)) 1441 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "ftruncate"); 1442 if (0 != lseek(fd, 0, SEEK_SET)) 1443 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "lseek"); 1444 } 1445 else { 1446 if (0 != unlink(fn->ptr)) 1447 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "unlink"); 1448 close(fd); 1449 } 1450 buffer_clear(fn); /*(&p->tmp_buf)*/ 1451 return 0; 1452 } 1453 1454 1455 static int mod_deflate_using_libdeflate_sm (handler_ctx * const hctx, const plugin_data * const p) 1456 { 1457 const encparms * const params = p->conf.params; 1458 const int clevel = (NULL != params) 1459 ? params->gzip.clevel 1460 : p->conf.compression_level; 1461 struct libdeflate_compressor * const compressor = 1462 libdeflate_alloc_compressor(clevel > 0 ? clevel : 6); 1463 /* Z_DEFAULT_COMPRESSION -1 not supported */ 1464 if (NULL == compressor) 1465 return 0; 1466 1467 char * const in = hctx->r->write_queue.first->mem->ptr; 1468 const size_t in_nbytes = (size_t)hctx->bytes_in; 1469 buffer * const addrb = hctx->output; /*(&p->tmp_buf)*/ 1470 size_t sz = buffer_string_space(addrb)+1; 1471 sz = (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 1472 ? libdeflate_gzip_compress(compressor, in, in_nbytes, addrb->ptr, sz) 1473 : libdeflate_deflate_compress(compressor, in, in_nbytes, addrb->ptr, sz); 1474 libdeflate_free_compressor(compressor); 1475 1476 if (0 == sz) { 1477 buffer_clear(addrb); 1478 return 0; 1479 } 1480 1481 chunkqueue_reset(&hctx->r->write_queue); 1482 hctx->bytes_out = (off_t)sz; 1483 if (0 != stream_http_chunk_append_mem(hctx, addrb->ptr, sz)) 1484 return mod_deflate_using_libdeflate_err(hctx, addrb, hctx->cache_fd); 1485 1486 buffer * const vb = 1487 http_header_response_set_ptr(hctx->r, HTTP_HEADER_CONTENT_LENGTH, 1488 CONST_STR_LEN("Content-Length")); 1489 buffer_append_int(vb, hctx->bytes_out); 1490 return 1; 1491 } 1492 1493 1494 #ifdef HAVE_MMAP 1495 #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) 1496 1497 struct mod_deflate_setjmp_params { 1498 struct libdeflate_compressor *compressor; 1499 void *out; 1500 size_t outsz; 1501 }; 1502 1503 static off_t mod_deflate_using_libdeflate_setjmp_cb (void *dst, const void *src, off_t len) 1504 { 1505 const struct mod_deflate_setjmp_params * const params = dst; 1506 const handler_ctx * const hctx = src; 1507 const chunk * const c = hctx->r->write_queue.first; 1508 const char *in = chunk_file_view_dptr(c->file.view, c->offset); 1509 return (off_t)((hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 1510 ? libdeflate_gzip_compress(params->compressor, in, (size_t)len, 1511 params->out, params->outsz) 1512 : libdeflate_deflate_compress(params->compressor, in, (size_t)len, 1513 params->out, params->outsz)); 1514 } 1515 1516 1517 static int mod_deflate_using_libdeflate (handler_ctx * const hctx, const plugin_data * const p) 1518 { 1519 buffer * const fn = hctx->output; /*(&p->tmp_buf)*/ 1520 int fd = hctx->cache_fd; 1521 if (-1 == fd) { 1522 chunkqueue * const cq = &hctx->r->write_queue; 1523 if (cq->tempdirs && cq->tempdir_idx < cq->tempdirs->used) { 1524 data_string *ds =(data_string *)cq->tempdirs->data[cq->tempdir_idx]; 1525 buffer_copy_string_len(fn, BUF_PTR_LEN(&ds->value)); 1526 } 1527 else 1528 buffer_copy_string_len(fn, CONST_STR_LEN("/var/tmp")); 1529 buffer_append_path_len(fn, CONST_STR_LEN("lighttpd-XXXXXX")); 1530 fd = fdevent_mkostemp(fn->ptr, 0); 1531 if (-1 == fd) return 0; 1532 } 1533 1534 const size_t sz = 1535 libdeflate_deflate_compress_bound(NULL, (size_t)hctx->bytes_in); 1536 /*(XXX: consider trying posix_fallocate() first, 1537 * with fallback to ftrunctate() if EOPNOTSUPP)*/ 1538 if (0 != ftruncate(fd, (off_t)sz)) { 1539 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "ftruncate"); 1540 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1541 } 1542 1543 /*void *addr = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);*/ 1544 void * const addr = mmap(NULL, sz, PROT_WRITE, MAP_SHARED, fd, 0); 1545 if (MAP_FAILED == addr) { 1546 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "mmap"); 1547 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1548 } 1549 1550 const encparms * const params = p->conf.params; 1551 const int clevel = (NULL != params) 1552 ? params->gzip.clevel 1553 : p->conf.compression_level; 1554 struct libdeflate_compressor * const compressor = 1555 libdeflate_alloc_compressor(clevel > 0 ? clevel : 6); 1556 /* Z_DEFAULT_COMPRESSION -1 not supported */ 1557 if (NULL != compressor) { 1558 struct mod_deflate_setjmp_params outparams = { compressor, addr, sz }; 1559 hctx->bytes_out = 1560 sys_setjmp_eval3(mod_deflate_using_libdeflate_setjmp_cb, 1561 &outparams, hctx, hctx->bytes_in); 1562 libdeflate_free_compressor(compressor); 1563 } 1564 1565 /*(XXX: we theoretically could assign mmap to FILE_CHUNK in output 1566 * r->write_queue, for potential use by TLS modules, or if not using 1567 * sendfile in network_write.c, but we do not (easily) know if either 1568 * are configured or in use for this request. Might consider heuristic: 1569 * (*srv->srvconf.network_backend->ptr == 'w') (writev or write), or 1570 * to check con->is_ssl_sock.) These files would be safe to pass to 1571 * TLS modules to read from mmap without catching SIGBUS since these files 1572 * are created by lighttpd, either in deflate cache or as temporary file */ 1573 if (0 != munmap(addr, sz)) 1574 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "munmap"); 1575 1576 if (0 == hctx->bytes_out) { 1577 const chunk * const c = hctx->r->write_queue.first; 1578 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 1579 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1580 } 1581 else if (0 != ftruncate(fd, hctx->bytes_out)) 1582 hctx->bytes_out = 0; 1583 1584 if (0 == hctx->bytes_out) 1585 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1586 1587 if (fd != hctx->cache_fd) { 1588 chunkqueue * const cq = &hctx->r->write_queue; 1589 chunkqueue_reset(cq); 1590 http_chunk_append_file_fd_range(hctx->r, fn, fd, 0, hctx->bytes_out); 1591 cq->last->file.is_temp = 1; 1592 } 1593 1594 buffer * const vb = 1595 http_header_response_set_ptr(hctx->r, HTTP_HEADER_CONTENT_LENGTH, 1596 CONST_STR_LEN("Content-Length")); 1597 buffer_append_int(vb, hctx->bytes_out); 1598 return 1; 1599 } 1600 1601 #endif /* defined(_LP64) || defined(__LP64__) || defined(_WIN64) */ 1602 #endif /* HAVE_MMAP */ 1603 1604 #endif /* HAVE_LIBDEFLATE */ 1605 1606 1607 static off_t mod_deflate_file_chunk_no_mmap(request_st * const r, handler_ctx * const hctx, const chunk * const c, off_t n) 1608 { 1609 const off_t insz = n; 1610 const size_t psz = (n < 2*1024*1024) ? (size_t)n : 2*1024*1024; 1611 char * const p = malloc(psz); 1612 if (NULL == p) { 1613 log_perror(r->conf.errh, __FILE__, __LINE__, "malloc"); 1614 return -1; 1615 } 1616 1617 ssize_t rd = 0; 1618 for (n = 0; n < insz; n += rd) { 1619 rd = chunk_file_pread(c->file.fd, p, (size_t)psz, c->offset+n); 1620 if (__builtin_expect( (rd > 0), 1)) { 1621 if (0 == mod_deflate_compress(hctx, (unsigned char *)p, rd)) 1622 continue; 1623 /*(else error trace printed upon return)*/ 1624 } 1625 else if (-1 == rd) 1626 log_perror(r->conf.errh, __FILE__, __LINE__, 1627 "reading %s failed", c->mem->ptr); 1628 else /*(0 == rd)*/ 1629 log_error(r->conf.errh, __FILE__, __LINE__, 1630 "file truncated %s", c->mem->ptr); 1631 n = -1; 1632 break; 1633 } 1634 1635 free(p); 1636 return n; 1637 } 1638 1639 1640 #ifdef ENABLE_MMAP 1641 1642 static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t len) 1643 { 1644 return mod_deflate_compress(dst, (const unsigned char *)src, len); 1645 } 1646 1647 1648 static off_t mod_deflate_file_chunk_mmap(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n) 1649 { 1650 /* n is length of entire file since server blocks while compressing 1651 * (mod_deflate is not recommended for large files; 1652 * mod_deflate default upper limit is 128MB; deflate.max-compress-size) */ 1653 1654 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 1655 ? chunkqueue_chunk_file_view(c, n, r->conf.errh) 1656 : NULL; 1657 if (NULL == cfv) 1658 return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); 1659 1660 const char * const p = chunk_file_view_dptr(cfv, c->offset); 1661 off_t len = chunk_file_view_dlen(cfv, c->offset); 1662 if (len > n) len = n; 1663 off_t rc = sys_setjmp_eval3(mod_deflate_file_chunk_setjmp_cb, hctx, p, len); 1664 if (__builtin_expect( (rc < 0), 0)) { 1665 if (errno == EFAULT) 1666 log_error(r->conf.errh, __FILE__, __LINE__, 1667 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1668 else 1669 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1670 len = -1; /*return -1;*/ 1671 } 1672 return len; 1673 } 1674 1675 #endif 1676 1677 1678 static off_t mod_deflate_file_chunk(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n) { 1679 if (-1 == c->file.fd) { /* open the file if not already open */ 1680 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) { 1681 log_perror(r->conf.errh, __FILE__, __LINE__, "open failed %s", c->mem->ptr); 1682 return -1; 1683 } 1684 } 1685 #ifdef ENABLE_MMAP 1686 return mod_deflate_file_chunk_mmap(r, hctx, c, n); 1687 #else 1688 return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); 1689 #endif 1690 } 1691 1692 1693 static handler_t deflate_compress_response(request_st * const r, handler_ctx * const hctx) { 1694 off_t len, max; 1695 int close_stream; 1696 1697 /* move all chunk from write_queue into our in_queue, then adjust 1698 * counters since r->write_queue is reused for compressed output */ 1699 chunkqueue * const cq = &r->write_queue; 1700 len = chunkqueue_length(cq); 1701 chunkqueue_remove_finished_chunks(cq); 1702 chunkqueue_append_chunkqueue(&hctx->in_queue, cq); 1703 cq->bytes_in -= len; 1704 cq->bytes_out -= len; 1705 1706 max = chunkqueue_length(&hctx->in_queue); 1707 #if 0 1708 /* calculate max bytes to compress for this call */ 1709 if (p->conf.sync_flush && max > (len = p->conf.work_block_size << 10)) { 1710 max = len; 1711 } 1712 #endif 1713 1714 /* Compress chunks from in_queue into chunks for write_queue */ 1715 while (max) { 1716 chunk *c = hctx->in_queue.first; 1717 1718 switch(c->type) { 1719 case MEM_CHUNK: 1720 len = buffer_clen(c->mem) - c->offset; 1721 if (len > max) len = max; 1722 if (mod_deflate_compress(hctx, (unsigned char *)c->mem->ptr+c->offset, len) < 0) { 1723 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1724 return HANDLER_ERROR; 1725 } 1726 break; 1727 case FILE_CHUNK: 1728 len = c->file.length - c->offset; 1729 if (len > max) len = max; 1730 if ((len = mod_deflate_file_chunk(r, hctx, c, len)) < 0) { 1731 log_error(r->conf.errh, __FILE__, __LINE__, 1732 "compress file chunk failed %s", c->mem->ptr); 1733 return HANDLER_ERROR; 1734 } 1735 break; 1736 default: 1737 log_error(r->conf.errh, __FILE__, __LINE__, "%d type not known", c->type); 1738 return HANDLER_ERROR; 1739 } 1740 1741 max -= len; 1742 chunkqueue_mark_written(&hctx->in_queue, len); 1743 } 1744 1745 /*(currently should always be true)*/ 1746 /*(current implementation requires response be complete)*/ 1747 close_stream = (r->resp_body_finished 1748 && chunkqueue_is_empty(&hctx->in_queue)); 1749 if (mod_deflate_stream_flush(hctx, close_stream) < 0) { 1750 log_error(r->conf.errh, __FILE__, __LINE__, "flush error"); 1751 return HANDLER_ERROR; 1752 } 1753 1754 return close_stream ? HANDLER_FINISHED : HANDLER_GO_ON; 1755 } 1756 1757 1758 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { 1759 /* get client side support encodings */ 1760 int accept_encoding = 0; 1761 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \ 1762 && !defined(USE_ZSTD) 1763 UNUSED(value); 1764 UNUSED(label); 1765 #else 1766 for (; *value; ++value) { 1767 const char *v; 1768 while (*value == ' ' || *value == ',') ++value; 1769 v = value; 1770 while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0') 1771 ++value; 1772 switch (value - v) { 1773 case 2: 1774 #ifdef USE_BROTLI 1775 if (0 == memcmp(v, "br", 2)) 1776 accept_encoding |= HTTP_ACCEPT_ENCODING_BR; 1777 #endif 1778 break; 1779 case 4: 1780 #ifdef USE_ZLIB 1781 if (0 == memcmp(v, "gzip", 4)) 1782 accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; 1783 #endif 1784 #ifdef USE_ZSTD 1785 #ifdef USE_ZLIB 1786 else 1787 #endif 1788 if (0 == memcmp(v, "zstd", 4)) 1789 accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD; 1790 #endif 1791 break; 1792 case 5: 1793 #ifdef USE_BZ2LIB 1794 if (0 == memcmp(v, "bzip2", 5)) 1795 accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; 1796 #endif 1797 break; 1798 case 6: 1799 #ifdef USE_ZLIB 1800 if (0 == memcmp(v, "x-gzip", 6)) 1801 accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP; 1802 #endif 1803 break; 1804 case 7: 1805 #ifdef USE_ZLIB 1806 if (0 == memcmp(v, "deflate", 7)) 1807 accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; 1808 #endif 1809 #ifdef USE_BZ2LIB 1810 if (0 == memcmp(v, "x-bzip2", 7)) 1811 accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2; 1812 #endif 1813 break; 1814 #if 0 1815 case 8: 1816 if (0 == memcmp(v, "identity", 8)) 1817 accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; 1818 else if (0 == memcmp(v, "compress", 8)) 1819 accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; 1820 break; 1821 #endif 1822 default: 1823 break; 1824 } 1825 if (*value == ';') { 1826 while (*value != ',' && *value != '\0') ++value; 1827 } 1828 if (*value == '\0') break; 1829 } 1830 #endif 1831 1832 /* select best matching encoding */ 1833 const uint16_t *x = p->conf.allowed_encodings; 1834 if (NULL == x) return 0; 1835 while (*x && !(*x & accept_encoding)) ++x; 1836 accept_encoding &= *x; 1837 #ifdef USE_ZSTD 1838 if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) { 1839 *label = "zstd"; 1840 return HTTP_ACCEPT_ENCODING_ZSTD; 1841 } else 1842 #endif 1843 #ifdef USE_BROTLI 1844 if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { 1845 *label = "br"; 1846 return HTTP_ACCEPT_ENCODING_BR; 1847 } else 1848 #endif 1849 #ifdef USE_BZ2LIB 1850 if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) { 1851 *label = "bzip2"; 1852 return HTTP_ACCEPT_ENCODING_BZIP2; 1853 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_BZIP2) { 1854 *label = "x-bzip2"; 1855 return HTTP_ACCEPT_ENCODING_BZIP2; 1856 } else 1857 #endif 1858 #ifdef USE_ZLIB 1859 if (accept_encoding & HTTP_ACCEPT_ENCODING_GZIP) { 1860 *label = "gzip"; 1861 return HTTP_ACCEPT_ENCODING_GZIP; 1862 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_GZIP) { 1863 *label = "x-gzip"; 1864 return HTTP_ACCEPT_ENCODING_GZIP; 1865 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_DEFLATE) { 1866 *label = "deflate"; 1867 return HTTP_ACCEPT_ENCODING_DEFLATE; 1868 } else 1869 #endif 1870 if (0 == accept_encoding) { 1871 return 0; 1872 } else { 1873 return 0; 1874 } 1875 } 1876 1877 REQUEST_FUNC(mod_deflate_handle_response_start) { 1878 plugin_data *p = p_d; 1879 const buffer *vbro; 1880 buffer *vb; 1881 handler_ctx *hctx; 1882 const char *label; 1883 off_t len; 1884 uint32_t etaglen; 1885 int compression_type; 1886 handler_t rc; 1887 int had_vary = 0; 1888 1889 /*(current implementation requires response be complete)*/ 1890 if (!r->resp_body_finished) return HANDLER_GO_ON; 1891 if (r->http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; 1892 if (light_btst(r->resp_htags, HTTP_HEADER_TRANSFER_ENCODING)) 1893 return HANDLER_GO_ON; 1894 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_ENCODING)) 1895 return HANDLER_GO_ON; 1896 1897 /* disable compression for some http status types. */ 1898 switch(r->http_status) { 1899 case 100: 1900 case 101: 1901 case 204: 1902 case 205: 1903 case 304: 1904 /* disable compression as we have no response entity */ 1905 return HANDLER_GO_ON; 1906 default: 1907 break; 1908 } 1909 1910 mod_deflate_patch_config(r, p); 1911 1912 /* check if deflate configured for any mimetypes */ 1913 if (NULL == p->conf.mimetypes) return HANDLER_GO_ON; 1914 1915 /* check if size of response is below min-compress-size or exceeds max*/ 1916 /* (r->resp_body_finished checked at top of routine) */ 1917 len = chunkqueue_length(&r->write_queue); 1918 if (len <= (off_t)p->conf.min_compress_size) return HANDLER_GO_ON; 1919 if (p->conf.max_compress_size /*(max_compress_size in KB)*/ 1920 && len > ((off_t)p->conf.max_compress_size << 10)) { 1921 return HANDLER_GO_ON; 1922 } 1923 1924 /* Check Accept-Encoding for supported encoding. */ 1925 vbro = http_header_request_get(r, HTTP_HEADER_ACCEPT_ENCODING, CONST_STR_LEN("Accept-Encoding")); 1926 if (NULL == vbro) return HANDLER_GO_ON; 1927 1928 /* find matching encodings */ 1929 compression_type = mod_deflate_choose_encoding(vbro->ptr, p, &label); 1930 if (!compression_type) return HANDLER_GO_ON; 1931 1932 /* Check mimetype in response header "Content-Type" */ 1933 if (NULL != (vbro = http_header_response_get(r, HTTP_HEADER_CONTENT_TYPE, CONST_STR_LEN("Content-Type")))) { 1934 if (NULL == array_match_value_prefix(p->conf.mimetypes, vbro)) return HANDLER_GO_ON; 1935 } else { 1936 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */ 1937 data_string *mimetype = (data_string *)p->conf.mimetypes->data[0]; 1938 if (!buffer_is_blank(&mimetype->value)) return HANDLER_GO_ON; 1939 } 1940 1941 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */ 1942 if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) { 1943 had_vary = 1; 1944 if (!http_header_str_contains_token(BUF_PTR_LEN(vb), 1945 CONST_STR_LEN("Accept-Encoding"))) 1946 buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding")); 1947 } else { 1948 http_header_response_append(r, HTTP_HEADER_VARY, 1949 CONST_STR_LEN("Vary"), 1950 CONST_STR_LEN("Accept-Encoding")); 1951 } 1952 1953 /* check ETag as is done in http_response_handle_cachable() 1954 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */ 1955 vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag")); 1956 etaglen = vb ? buffer_clen(vb) : 0; 1957 if (etaglen && light_btst(r->rqst_htags, HTTP_HEADER_IF_NONE_MATCH)) { 1958 const buffer *if_none_match = http_header_request_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match")); 1959 if ( r->http_status < 300 /*(want 2xx only)*/ 1960 && NULL != if_none_match 1961 && 0 == strncmp(if_none_match->ptr, vb->ptr, etaglen-1) 1962 && if_none_match->ptr[etaglen-1] == '-' 1963 && 0 == strncmp(if_none_match->ptr+etaglen, label, strlen(label))) { 1964 1965 if (http_method_get_head_query(r->http_method)) { 1966 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1967 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1968 buffer_append_string(vb, label); 1969 buffer_append_char(vb, '"'); 1970 r->http_status = 304; 1971 } else { 1972 r->http_status = 412; 1973 } 1974 1975 /* response_start hook occurs after error docs have been handled. 1976 * For now, send back empty response body. 1977 * In the future, might extract the error doc code so that it 1978 * might be run again if response_start hooks return with 1979 * changed http_status and r->handler_module = NULL */ 1980 /* clear content length even if 304 since compressed length unknown */ 1981 http_response_body_clear(r, 0); 1982 1983 r->resp_body_finished = 1; 1984 r->handler_module = NULL; 1985 return HANDLER_GO_ON; 1986 } 1987 } 1988 1989 if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < r->con->srv->loadavg[0]) { 1990 return HANDLER_GO_ON; 1991 } 1992 1993 /* update ETag, if ETag response header is set */ 1994 if (etaglen) { 1995 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1996 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1997 buffer_append_string(vb, label); 1998 buffer_append_char(vb, '"'); 1999 } 2000 2001 /* set Content-Encoding to show selected compression type */ 2002 http_header_response_set(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding"), label, strlen(label)); 2003 2004 /* clear Content-Length and r->write_queue if HTTP HEAD request 2005 * (alternatively, could return original Content-Length with HEAD 2006 * request if ETag not modified and Content-Encoding not added) 2007 * (see top of this func where short-circuit is done on HTTP HEAD) */ 2008 if (HTTP_METHOD_HEAD == r->http_method) { 2009 /* ensure that uncompressed Content-Length is not sent in HEAD response */ 2010 http_response_body_clear(r, 0); 2011 r->resp_body_finished = 1; 2012 return HANDLER_GO_ON; 2013 } 2014 2015 /* restrict items eligible for cache of compressed responses 2016 * (This module does not aim to be a full caching proxy) 2017 * response must be complete (not streaming response) 2018 * must not have prior Vary response header (before Accept-Encoding added) 2019 * must have ETag 2020 * must be file 2021 * must be single FILE_CHUNK in chunkqueue 2022 * must not be chunkqueue temporary file 2023 * must be whole file, not partial content 2024 * must not be HTTP status 206 Partial Content 2025 * must not have Cache-Control 'private' or 'no-store' 2026 * Note: small files (< 32k (see http_chunk.c)) will have been read into 2027 * memory (if streaming HTTP/1.1 chunked response) and will end up 2028 * getting stream-compressed rather than cached on disk as compressed 2029 * file 2030 */ 2031 buffer *tb = NULL; 2032 if (p->conf.cache_dir 2033 && !had_vary 2034 && etaglen > 2 2035 && r->resp_body_finished 2036 && r->write_queue.first == r->write_queue.last 2037 && r->write_queue.first->type == FILE_CHUNK 2038 && r->write_queue.first->offset == 0 2039 && !r->write_queue.first->file.is_temp 2040 && r->http_status != 206 2041 && (!(vbro = http_header_response_get(r, HTTP_HEADER_CACHE_CONTROL, 2042 CONST_STR_LEN("Cache-Control"))) 2043 || (!http_header_str_contains_token(BUF_PTR_LEN(vbro), 2044 CONST_STR_LEN("private")) 2045 && !http_header_str_contains_token(BUF_PTR_LEN(vbro), 2046 CONST_STR_LEN("no-store")))) 2047 ) { 2048 tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb); 2049 /*(checked earlier and skipped if Transfer-Encoding had been set)*/ 2050 stat_cache_entry *sce = stat_cache_get_entry_open(tb, 1); 2051 if (NULL != sce) { 2052 chunkqueue_reset(&r->write_queue); 2053 if (sce->fd < 0 || 0 != http_chunk_append_file_ref(r, sce)) 2054 return HANDLER_ERROR; 2055 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) 2056 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, 2057 CONST_STR_LEN("Content-Length")); 2058 mod_deflate_note_ratio(r, sce->st.st_size, len); 2059 return HANDLER_GO_ON; 2060 } 2061 /* sanity check that response was whole file; 2062 * (racy since using stat_cache, but cache file only if match) */ 2063 sce = stat_cache_get_entry(r->write_queue.first->mem); 2064 if (NULL == sce || sce->st.st_size != len) 2065 tb = NULL; 2066 else if (0 != mkdir_for_file(tb->ptr)) 2067 tb = NULL; 2068 } 2069 2070 /* enable compression */ 2071 p->conf.sync_flush = 2072 ((r->conf.stream_response_body 2073 & (FDEVENT_STREAM_RESPONSE | FDEVENT_STREAM_RESPONSE_BUFMIN)) 2074 && 0 == p->conf.output_buffer_size); 2075 hctx = handler_ctx_init(); 2076 hctx->plugin_data = p; 2077 hctx->compression_type = compression_type; 2078 hctx->r = r; 2079 /* setup output buffer */ 2080 buffer_clear(&p->tmp_buf); 2081 hctx->output = &p->tmp_buf; 2082 /* open cache file if caching compressed file */ 2083 if (tb) mod_deflate_cache_file_open(hctx, tb); 2084 2085 #ifdef HAVE_LIBDEFLATE 2086 chunk * const c = r->write_queue.first; /*(invalid after compression)*/ 2087 #ifdef HAVE_MMAP 2088 #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) 2089 /* optimization to compress single file in one-shot to writeable mmap */ 2090 /*(future: might extend to other compression types)*/ 2091 /*(chunkqueue_chunk_file_view() current min size for mmap is 128k)*/ 2092 if (len > 131072 /* XXX: TBD what should min size be for optimization?*/ 2093 && (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP 2094 || hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE) 2095 && c == r->write_queue.last 2096 && c->type == FILE_CHUNK 2097 && chunkqueue_chunk_file_view(c, len, r->conf.errh) 2098 && chunk_file_view_dlen(c->file.view, c->offset) >= len) { /*(cfv)*/ 2099 rc = HANDLER_GO_ON; 2100 hctx->bytes_in = len; 2101 if (mod_deflate_using_libdeflate(hctx, p)) { 2102 if (NULL == tb || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) 2103 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2104 else 2105 rc = HANDLER_ERROR; 2106 handler_ctx_free(hctx); 2107 return rc; 2108 } 2109 hctx->bytes_in = hctx->bytes_out = 0; 2110 } 2111 else 2112 #endif 2113 #endif 2114 if (len <= 65536 /*(p->tmp_buf is at least 64k)*/ 2115 && (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP 2116 || hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE) 2117 && c == r->write_queue.last 2118 && c->type == MEM_CHUNK) { 2119 /*(skip if FILE_CHUNK; not worth mmap/munmap overhead on small file)*/ 2120 rc = HANDLER_GO_ON; 2121 hctx->bytes_in = len; 2122 if (mod_deflate_using_libdeflate_sm(hctx, p)) { 2123 if (NULL == tb || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) 2124 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2125 else 2126 rc = HANDLER_ERROR; 2127 handler_ctx_free(hctx); 2128 return rc; 2129 } 2130 hctx->bytes_in = hctx->bytes_out = 0; 2131 } 2132 #endif /* HAVE_LIBDEFLATE */ 2133 2134 if (0 != mod_deflate_stream_init(hctx)) { 2135 /*(should not happen unless ENOMEM)*/ 2136 handler_ctx_free(hctx); 2137 log_error(r->conf.errh, __FILE__, __LINE__, 2138 "Failed to initialize compression %s", label); 2139 /* restore prior Etag and unset Content-Encoding */ 2140 if (etaglen) { 2141 vb->ptr[etaglen-1] = '"'; /*(overwrite '-')*/ 2142 buffer_truncate(vb, etaglen); 2143 } 2144 http_header_response_unset(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding")); 2145 return HANDLER_GO_ON; 2146 } 2147 2148 #ifdef USE_BROTLI 2149 if (r->resp_body_finished 2150 && (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR) 2151 && (len >> 1) < (off_t)((~(uint32_t)0u) >> 1)) 2152 BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len); 2153 #endif 2154 2155 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) { 2156 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length")); 2157 } 2158 r->plugin_ctx[p->id] = hctx; 2159 2160 rc = deflate_compress_response(r, hctx); 2161 if (HANDLER_GO_ON == rc) return HANDLER_GO_ON; 2162 if (HANDLER_FINISHED == rc) { 2163 #ifdef __COVERITY__ 2164 /* coverity misses if hctx->cache_fd is not -1, then tb is not NULL */ 2165 force_assert(-1 == hctx->cache_fd || NULL != tb); 2166 #endif 2167 if (-1 == hctx->cache_fd 2168 || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) { 2169 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2170 rc = HANDLER_GO_ON; 2171 } 2172 else 2173 rc = HANDLER_ERROR; 2174 } 2175 r->plugin_ctx[p->id] = NULL; 2176 if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR; 2177 return rc; 2178 } 2179 2180 static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) { 2181 plugin_data *p = p_d; 2182 handler_ctx *hctx = r->plugin_ctx[p->id]; 2183 2184 if (NULL != hctx) { 2185 r->plugin_ctx[p->id] = NULL; 2186 deflate_compress_cleanup(r, hctx); 2187 } 2188 2189 return HANDLER_GO_ON; 2190 } 2191 2192 2193 __attribute_cold__ 2194 int mod_deflate_plugin_init(plugin *p); 2195 int mod_deflate_plugin_init(plugin *p) { 2196 p->version = LIGHTTPD_VERSION_ID; 2197 p->name = "deflate"; 2198 2199 p->init = mod_deflate_init; 2200 p->cleanup = mod_deflate_free; 2201 p->set_defaults = mod_deflate_set_defaults; 2202 p->handle_request_reset = mod_deflate_cleanup; 2203 p->handle_response_start = mod_deflate_handle_response_start; 2204 2205 return 0; 2206 } 2207