1 /* mod_deflate 2 * 3 * 4 * bug fix on Robert Jakabosky from alphatrade.com's lighttp 1.4.10 mod_deflate patch 5 * 6 * Bug fix and new features: 7 * 1) fix loop bug when content-length is bigger than work-block-size*k 8 * 9 * ------- 10 * 11 * lighttpd-1.4.26.mod_deflate.patch from 12 * https://redmine.lighttpd.net/projects/1/wiki/Docs_ModDeflate 13 * 14 * ------- 15 * 16 * Patch further modified in this incarnation. 17 * 18 * Note: this patch only handles completed responses 19 * (r->resp_body_finished) 20 * this patch does not currently handle streaming dynamic responses, 21 * and therefore also does not worry about Transfer-Encoding: chunked 22 * (or having separate con->output_queue for chunked-encoded output) 23 * (or using separate buffers per connection instead of p->tmp_buf) 24 * (or handling interactions with block buffering and write timeouts) 25 * 26 * Bug fix: 27 * - fixed major bug with compressing chunks with offset > 0 28 * x-ref: 29 * "Response breaking in mod_deflate" 30 * https://redmine.lighttpd.net/issues/986 31 * - fix broken (in some cases) chunk accounting in deflate_compress_response() 32 * - fix broken bzip2 33 * x-ref: 34 * "mod_deflate's bzip2 broken by default" 35 * https://redmine.lighttpd.net/issues/2035 36 * - fix mismatch with current chunk interfaces 37 * x-ref: 38 * "Weird things in chunk.c (functions only handling specific cases, unexpected behaviour)" 39 * https://redmine.lighttpd.net/issues/1510 40 * 41 * Behavior changes from prior patch: 42 * - deflate.mimetypes must now be configured to enable compression 43 * deflate.mimetypes = ( ) # compress nothing (disabled; default) 44 * deflate.mimetypes = ( "" ) # compress all mimetypes 45 * deflate.mimetypes = ( "text/" ) # compress text/... mimetypes 46 * x-ref: 47 * "mod_deflate enabled by default" 48 * https://redmine.lighttpd.net/issues/1394 49 * - deflate.enabled directive removed (see new behavior of deflate.mimetypes) 50 * - deflate.debug removed (was developer debug trace, not end-user debug) 51 * - deflate.bzip2 replaced with deflate.allowed-encodings (like mod_compress) 52 * x-ref: 53 * "mod_deflate should allow limiting of compression algorithm from the configuration file" 54 * https://redmine.lighttpd.net/issues/996 55 * "mod_compress disabling methods" 56 * https://redmine.lighttpd.net/issues/1773 57 * - deflate.nocompress-url removed since disabling compression for a URL 58 * can now easily be done by setting to a blank list either directive 59 * deflate.accept_encodings = () or deflate.mimetypes = () in a conditional 60 * block, e.g. $HTTP["url"] =~ "....." { deflate.mimetypes = ( ) } 61 * - deflate.sync-flush removed; controlled by r->conf.stream_response_body 62 * (though streaming compression not currently implemented in mod_deflate) 63 * - inactive directives in this patch 64 * (since r->resp_body_finished required) 65 * deflate.work-block-size 66 * deflate.output-buffer-size 67 * - remove weak file size check; SIGBUS is trapped, file that shrink will error 68 * x-ref: 69 * "mod_deflate: filesize check is too weak" 70 * https://redmine.lighttpd.net/issues/1512 71 * - change default deflate.min-compress-size from 0 to now be 256 72 * http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits 73 * Apache 2.4 mod_deflate minimum is 68 bytes 74 * Akamai recommends minimum 860 bytes 75 * Google recommends minimum be somewhere in range between 150 and 1024 bytes 76 * - deflate.max-compress-size new directive (in kb like compress.max_filesize) 77 * - deflate.mem-level removed (too many knobs for little benefit) 78 * - deflate.window-size removed (too many knobs for little benefit) 79 * 80 * Future: 81 * - config directives may be changed, renamed, or removed 82 * e.g. A set of reasonable defaults might be chosen 83 * instead of making them configurable. 84 * deflate.min-compress-size 85 * - might add deflate.mimetypes-exclude = ( ... ) for list of mimetypes 86 * to avoid compressing, even if a broader deflate.mimetypes matched, 87 * e.g. to compress all "text/" except "text/special". 88 * 89 * Implementation notes: 90 * - http_chunk_append_mem() used instead of http_chunk_append_buffer() 91 * so that p->tmp_buf can be large and re-used. This results in an extra copy 92 * of compressed data before data is sent to network, though if the compressed 93 * size is larger than 64k, it ends up being sent to a temporary file on 94 * disk without suffering an extra copy in memory, and without extra chunk 95 * create and destroy. If this is ever changed to give away buffers, then use 96 * a unique hctx->output buffer per hctx; do not reuse p->tmp_buf across 97 * multiple requests being handled in parallel. 98 */ 99 #include "first.h" 100 101 #include <sys/types.h> 102 #include <sys/stat.h> 103 #include "sys-mmap.h" 104 #ifdef HAVE_MMAP 105 #include "sys-setjmp.h" 106 #endif 107 #include "sys-time.h" 108 109 #include <fcntl.h> 110 #include <stdlib.h> 111 #include <string.h> 112 #include <errno.h> 113 #include <unistd.h> /* getpid() read() unlink() write() */ 114 115 #include "base.h" 116 #include "ck.h" 117 #include "fdevent.h" 118 #include "log.h" 119 #include "buffer.h" 120 #include "http_chunk.h" 121 #include "http_etag.h" 122 #include "http_header.h" 123 #include "response.h" 124 #include "stat_cache.h" 125 126 #include "plugin.h" 127 128 #if defined HAVE_ZLIB_H && defined HAVE_LIBZ 129 # define USE_ZLIB 130 # include <zlib.h> 131 #endif 132 #ifndef Z_DEFAULT_COMPRESSION 133 #define Z_DEFAULT_COMPRESSION -1 134 #endif 135 #ifndef MAX_WBITS 136 #define MAX_WBITS 15 137 #endif 138 139 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 140 # define USE_BZ2LIB 141 /* we don't need stdio interface */ 142 # define BZ_NO_STDIO 143 # include <bzlib.h> 144 #endif 145 146 #if defined HAVE_BROTLI_ENCODE_H && defined HAVE_BROTLI 147 # define USE_BROTLI 148 # include <brotli/encode.h> 149 #endif 150 151 #if defined HAVE_ZSTD_H && defined HAVE_ZSTD 152 # define USE_ZSTD 153 # include <zstd.h> 154 #endif 155 156 #ifndef HAVE_LIBZ 157 #undef HAVE_LIBDEFLATE 158 #endif 159 160 /* request: accept-encoding */ 161 #define HTTP_ACCEPT_ENCODING_IDENTITY BV(0) 162 #define HTTP_ACCEPT_ENCODING_GZIP BV(1) 163 #define HTTP_ACCEPT_ENCODING_DEFLATE BV(2) 164 #define HTTP_ACCEPT_ENCODING_COMPRESS BV(3) 165 #define HTTP_ACCEPT_ENCODING_BZIP2 BV(4) 166 #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) 167 #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) 168 #define HTTP_ACCEPT_ENCODING_BR BV(7) 169 #define HTTP_ACCEPT_ENCODING_ZSTD BV(8) 170 171 typedef struct { 172 struct { 173 int clevel; /*(compression level)*/ 174 int windowBits; 175 int memLevel; 176 int strategy; 177 } gzip; 178 struct { 179 uint32_t quality; /*(compression level)*/ 180 uint32_t window; 181 uint32_t mode; 182 } brotli; 183 struct { 184 int clevel; /*(compression level)*/ 185 int strategy; 186 int windowLog; 187 } zstd; 188 struct { 189 int clevel; /*(compression level)*/ 190 } bzip2; 191 } encparms; 192 193 typedef struct { 194 const array *mimetypes; 195 const buffer *cache_dir; 196 unsigned int max_compress_size; 197 unsigned short min_compress_size; 198 unsigned short output_buffer_size; 199 unsigned short work_block_size; 200 unsigned short sync_flush; 201 short compression_level; 202 uint16_t * allowed_encodings; 203 double max_loadavg; 204 const encparms *params; 205 } plugin_config; 206 207 typedef struct { 208 PLUGIN_DATA; 209 plugin_config defaults; 210 plugin_config conf; 211 212 buffer tmp_buf; 213 } plugin_data; 214 215 typedef struct { 216 union { 217 #ifdef USE_ZLIB 218 z_stream z; 219 #endif 220 #ifdef USE_BZ2LIB 221 bz_stream bz; 222 #endif 223 #ifdef USE_BROTLI 224 BrotliEncoderState *br; 225 #endif 226 #ifdef USE_ZSTD 227 ZSTD_CStream *cctx; 228 #endif 229 int dummy; 230 } u; 231 off_t bytes_in; 232 off_t bytes_out; 233 buffer *output; 234 plugin_data *plugin_data; 235 request_st *r; 236 int compression_type; 237 int cache_fd; 238 char *cache_fn; 239 chunkqueue in_queue; 240 } handler_ctx; 241 242 static handler_ctx *handler_ctx_init() { 243 handler_ctx * const hctx = ck_calloc(1, sizeof(*hctx)); 244 chunkqueue_init(&hctx->in_queue); 245 hctx->cache_fd = -1; 246 return hctx; 247 } 248 249 static void handler_ctx_free(handler_ctx *hctx) { 250 if (hctx->cache_fn) { 251 unlink(hctx->cache_fn); 252 free(hctx->cache_fn); 253 } 254 if (-1 != hctx->cache_fd) 255 close(hctx->cache_fd); 256 #if 0 257 if (hctx->output != &p->tmp_buf) { 258 buffer_free(hctx->output); 259 } 260 #endif 261 chunkqueue_reset(&hctx->in_queue); 262 free(hctx); 263 } 264 265 INIT_FUNC(mod_deflate_init) { 266 plugin_data * const p = ck_calloc(1, sizeof(plugin_data)); 267 #ifdef USE_ZSTD 268 buffer_string_prepare_copy(&p->tmp_buf, ZSTD_CStreamOutSize()); 269 #else 270 buffer_string_prepare_copy(&p->tmp_buf, 65536); 271 #endif 272 return p; 273 } 274 275 FREE_FUNC(mod_deflate_free) { 276 plugin_data *p = p_d; 277 free(p->tmp_buf.ptr); 278 if (NULL == p->cvlist) return; 279 /* (init i to 0 if global context; to 1 to skip empty global context) */ 280 for (int i = !p->cvlist[0].v.u2[1], used = p->nconfig; i < used; ++i) { 281 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 282 for (; -1 != cpv->k_id; ++cpv) { 283 if (cpv->vtype != T_CONFIG_LOCAL || NULL == cpv->v.v) continue; 284 switch (cpv->k_id) { 285 case 1: /* deflate.allowed-encodings */ 286 case 14:/* deflate.params */ 287 free(cpv->v.v); 288 break; 289 default: 290 break; 291 } 292 } 293 } 294 } 295 296 #if defined(_WIN32) && !defined(__CYGWIN__) 297 #define mkdir(x,y) mkdir(x) 298 #endif 299 300 static int mkdir_for_file (char *fn) { 301 for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) { 302 if (p[1] == '\0') return 0; /* ignore trailing slash */ 303 *p = '\0'; 304 int rc = mkdir(fn, 0700); 305 *p = '/'; 306 if (0 != rc && errno != EEXIST) return -1; 307 } 308 return 0; 309 } 310 311 static int mkdir_recursive (char *dir) { 312 return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST) 313 ? 0 314 : -1; 315 } 316 317 static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) { 318 /* XXX: future: for shorter paths into the cache, we could checksum path, 319 * and then shard it to avoid a huge single directory. 320 * Alternatively, could use &r->uri.path, minus any 321 * (matching) &r->pathinfo suffix, with result url-encoded 322 * Alternative, we could shard etag which is already our "checksum" */ 323 buffer * const tb = r->tmp_buf; 324 buffer_copy_path_len2(tb, BUF_PTR_LEN(cache_dir), 325 BUF_PTR_LEN(&r->physical.path)); 326 buffer_append_str2(tb, CONST_STR_LEN("-"), /*(strip surrounding '"')*/ 327 etag->ptr+1, buffer_clen(etag)-2); 328 return tb; 329 } 330 331 static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) { 332 /* race exists whereby up to # workers might attempt to compress same 333 * file at same time if requested at same time, but this is unlikely 334 * and resolves itself by atomic rename into place when done */ 335 const uint32_t fnlen = buffer_clen(fn); 336 hctx->cache_fn = ck_malloc(fnlen+1+LI_ITOSTRING_LENGTH+1); 337 memcpy(hctx->cache_fn, fn->ptr, fnlen); 338 hctx->cache_fn[fnlen] = '.'; 339 const size_t ilen = 340 li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid()); 341 hctx->cache_fn[fnlen+1+ilen] = '\0'; 342 hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600); 343 if (-1 == hctx->cache_fd) { 344 free(hctx->cache_fn); 345 hctx->cache_fn = NULL; 346 } 347 } 348 349 static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) { 350 if (0 != fdevent_rename(hctx->cache_fn, fn->ptr)) 351 return -1; 352 free(hctx->cache_fn); 353 hctx->cache_fn = NULL; 354 chunkqueue_reset(&r->write_queue); 355 int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out); 356 hctx->cache_fd = -1; 357 return rc; 358 } 359 360 static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) { 361 switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */ 362 case 0: /* deflate.mimetypes */ 363 pconf->mimetypes = cpv->v.a; 364 break; 365 case 1: /* deflate.allowed-encodings */ 366 if (cpv->vtype == T_CONFIG_LOCAL) 367 pconf->allowed_encodings = cpv->v.v; 368 break; 369 case 2: /* deflate.max-compress-size */ 370 pconf->max_compress_size = cpv->v.u; 371 break; 372 case 3: /* deflate.min-compress-size */ 373 pconf->min_compress_size = cpv->v.shrt; 374 break; 375 case 4: /* deflate.compression-level */ 376 pconf->compression_level = (short)cpv->v.shrt; 377 break; 378 case 5: /* deflate.output-buffer-size */ 379 pconf->output_buffer_size = cpv->v.shrt; 380 break; 381 case 6: /* deflate.work-block-size */ 382 pconf->work_block_size = cpv->v.shrt; 383 break; 384 case 7: /* deflate.max-loadavg */ 385 pconf->max_loadavg = cpv->v.d; 386 break; 387 case 8: /* deflate.cache-dir */ 388 pconf->cache_dir = cpv->v.b; 389 break; 390 #if 0 /*(cpv->k_id remapped in mod_deflate_set_defaults())*/ 391 case 9: /* compress.filetype */ 392 pconf->mimetypes = cpv->v.a; 393 break; 394 case 10:/* compress.allowed-encodings */ 395 if (cpv->vtype == T_CONFIG_LOCAL) 396 pconf->allowed_encodings = cpv->v.v; 397 break; 398 case 11:/* compress.cache-dir */ 399 pconf->cache_dir = cpv->v.b; 400 break; 401 case 12:/* compress.max-filesize */ 402 pconf->max_compress_size = cpv->v.u; 403 break; 404 case 13:/* compress.max-loadavg */ 405 pconf->max_loadavg = cpv->v.d; 406 break; 407 #endif 408 case 14:/* deflate.params */ 409 if (cpv->vtype == T_CONFIG_LOCAL) 410 pconf->params = cpv->v.v; 411 break; 412 default:/* should not happen */ 413 return; 414 } 415 } 416 417 static void mod_deflate_merge_config(plugin_config * const pconf, const config_plugin_value_t *cpv) { 418 do { 419 mod_deflate_merge_config_cpv(pconf, cpv); 420 } while ((++cpv)->k_id != -1); 421 } 422 423 static void mod_deflate_patch_config(request_st * const r, plugin_data * const p) { 424 memcpy(&p->conf, &p->defaults, sizeof(plugin_config)); 425 for (int i = 1, used = p->nconfig; i < used; ++i) { 426 if (config_check_cond(r, (uint32_t)p->cvlist[i].k_id)) 427 mod_deflate_merge_config(&p->conf, p->cvlist + p->cvlist[i].v.u2[0]); 428 } 429 } 430 431 static encparms * mod_deflate_parse_params(const array * const a, log_error_st * const errh) { 432 encparms * const params = ck_calloc(1, sizeof(encparms)); 433 434 /* set defaults */ 435 #ifdef USE_ZLIB 436 params->gzip.clevel = 0; /*(unset)*/ 437 params->gzip.windowBits = MAX_WBITS; 438 params->gzip.memLevel = 8; 439 params->gzip.strategy = Z_DEFAULT_STRATEGY; 440 #endif 441 #ifdef USE_BROTLI 442 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 443 params->brotli.quality = 5; 444 params->brotli.window = BROTLI_DEFAULT_WINDOW; 445 params->brotli.mode = BROTLI_MODE_GENERIC; 446 #endif 447 #ifdef USE_ZSTD 448 params->zstd.clevel = ZSTD_CLEVEL_DEFAULT; 449 params->zstd.strategy = 0; /*(use default strategy)*/ 450 params->zstd.windowLog = 0;/*(use default windowLog)*/ 451 #endif 452 #ifdef USE_BZ2LIB 453 params->bzip2.clevel = 0; /*(unset)*/ 454 #endif 455 456 for (uint32_t i = 0; i < a->used; ++i) { 457 const data_unset * const du = a->data[i]; 458 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 459 || defined(USE_ZSTD) 460 int32_t v = config_plugin_value_to_int32(du, -1); 461 #endif 462 #ifdef USE_BROTLI 463 if (buffer_eq_icase_slen(&du->key, 464 CONST_STR_LEN("BROTLI_PARAM_QUALITY"))) { 465 /*(future: could check for string and then look for and translate 466 * BROTLI_DEFAULT_QUALITY BROTLI_MIN_QUALITY BROTLI_MAX_QUALITY)*/ 467 if (BROTLI_MIN_QUALITY <= v && v <= BROTLI_MAX_QUALITY) 468 params->brotli.quality = (uint32_t)v; /* 0 .. 11 */ 469 else 470 log_error(errh, __FILE__, __LINE__, 471 "invalid value for BROTLI_PARAM_QUALITY"); 472 continue; 473 } 474 if (buffer_eq_icase_slen(&du->key, 475 CONST_STR_LEN("BROTLI_PARAM_LGWIN"))) { 476 /*(future: could check for string and then look for and translate 477 * BROTLI_DEFAULT_WINDOW 478 * BROTLI_MIN_WINDOW_BITS BROTLI_MAX_WINDOW_BITS)*/ 479 if (BROTLI_MIN_WINDOW_BITS <= v && v <= BROTLI_MAX_WINDOW_BITS) 480 params->brotli.window = (uint32_t)v; /* 10 .. 24 */ 481 else 482 log_error(errh, __FILE__, __LINE__, 483 "invalid value for BROTLI_PARAM_LGWIN"); 484 continue; 485 } 486 if (buffer_eq_icase_slen(&du->key, 487 CONST_STR_LEN("BROTLI_PARAM_MODE"))) { 488 /*(future: could check for string and then look for and translate 489 * BROTLI_MODE_GENERIC BROTLI_MODE_TEXT BROTLI_MODE_FONT)*/ 490 if (0 <= v && v <= 2) 491 params->brotli.mode = (uint32_t)v; /* 0 .. 2 */ 492 else 493 log_error(errh, __FILE__, __LINE__, 494 "invalid value for BROTLI_PARAM_MODE"); 495 continue; 496 } 497 #endif 498 #ifdef USE_ZSTD 499 if (buffer_eq_icase_slen(&du->key, 500 CONST_STR_LEN("ZSTD_c_compressionLevel"))) { 501 params->zstd.clevel = v; 502 /*(not warning if number parse error. future: to detect, could 503 * use absurd default to config_plugin_value_to_int32 to detect)*/ 504 continue; 505 } 506 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 507 /*(XXX: (selected) experimental API params in zstd v1.4.0)*/ 508 if (buffer_eq_icase_slen(&du->key, 509 CONST_STR_LEN("ZSTD_c_strategy"))) { 510 /*(future: could check for string and then look for and translate 511 * enum ZSTD_strategy ZSTD_STRATEGY_MIN ZSTD_STRATEGY_MAX)*/ 512 #ifndef ZSTD_STRATEGY_MIN 513 #define ZSTD_STRATEGY_MIN 1 514 #endif 515 #ifndef ZSTD_STRATEGY_MAX 516 #define ZSTD_STRATEGY_MAX 9 517 #endif 518 if (ZSTD_STRATEGY_MIN <= v && v <= ZSTD_STRATEGY_MAX) 519 params->zstd.strategy = v; /* 1 .. 9 */ 520 else 521 log_error(errh, __FILE__, __LINE__, 522 "invalid value for ZSTD_c_strategy"); 523 continue; 524 } 525 if (buffer_eq_icase_slen(&du->key, 526 CONST_STR_LEN("ZSTD_c_windowLog"))) { 527 /*(future: could check for string and then look for and translate 528 * ZSTD_WINDOWLOG_MIN ZSTD_WINDOWLOG_MAX)*/ 529 #ifndef ZSTD_WINDOWLOG_MIN 530 #define ZSTD_WINDOWLOG_MIN 10 531 #endif 532 #ifndef ZSTD_WINDOWLOG_MAX_32 533 #define ZSTD_WINDOWLOG_MAX_32 30 534 #endif 535 #ifndef ZSTD_WINDOWLOG_MAX_64 536 #define ZSTD_WINDOWLOG_MAX_64 31 537 #endif 538 #ifndef ZSTD_WINDOWLOG_MAX 539 #define ZSTD_WINDOWLOG_MAX \ 540 (sizeof(size_t)==4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64) 541 #endif 542 if (ZSTD_WINDOWLOG_MIN <= v && v <= ZSTD_WINDOWLOG_MAX) 543 params->zstd.windowLog = v;/* 10 .. 31 *//*(30 max for 32-bit)*/ 544 else 545 log_error(errh, __FILE__, __LINE__, 546 "invalid value for ZSTD_c_windowLog"); 547 continue; 548 } 549 #endif 550 #endif 551 #ifdef USE_ZLIB 552 if (buffer_eq_icase_slen(&du->key, 553 CONST_STR_LEN("gzip.level"))) { 554 if (1 <= v && v <= 9) 555 params->gzip.clevel = v; /* 1 .. 9 */ 556 else 557 log_error(errh, __FILE__, __LINE__, 558 "invalid value for gzip.level"); 559 continue; 560 } 561 if (buffer_eq_icase_slen(&du->key, 562 CONST_STR_LEN("gzip.windowBits"))) { 563 if (9 <= v && v <= 15) 564 params->gzip.windowBits = v; /* 9 .. 15 */ 565 else 566 log_error(errh, __FILE__, __LINE__, 567 "invalid value for gzip.windowBits"); 568 continue; 569 } 570 if (buffer_eq_icase_slen(&du->key, 571 CONST_STR_LEN("gzip.memLevel"))) { 572 if (1 <= v && v <= 9) 573 params->gzip.memLevel = v; /* 1 .. 9 */ 574 else 575 log_error(errh, __FILE__, __LINE__, 576 "invalid value for gzip.memLevel"); 577 continue; 578 } 579 if (buffer_eq_icase_slen(&du->key, 580 CONST_STR_LEN("gzip.strategy"))) { 581 /*(future: could check for string and then look for and translate 582 * Z_DEFAULT_STRATEGY Z_FILTERED Z_HUFFMAN_ONLY Z_RLE Z_FIXED)*/ 583 if (0 <= v && v <= 4) 584 params->gzip.strategy = v; /* 0 .. 4 */ 585 else 586 log_error(errh, __FILE__, __LINE__, 587 "invalid value for gzip.strategy"); 588 continue; 589 } 590 #endif 591 #ifdef USE_BZ2LIB 592 if (buffer_eq_icase_slen(&du->key, 593 CONST_STR_LEN("bzip2.blockSize100k"))) { 594 if (1 <= v && v <= 9) /*(bzip2 blockSize100k param)*/ 595 params->bzip2.clevel = v; /* 1 .. 9 */ 596 else 597 log_error(errh, __FILE__, __LINE__, 598 "invalid value for bzip2.blockSize100k"); 599 continue; 600 } 601 #endif 602 log_error(errh, __FILE__, __LINE__, 603 "unrecognized param: %s", du->key.ptr); 604 } 605 606 return params; 607 } 608 609 static uint16_t * mod_deflate_encodings_to_flags(const array *encodings) { 610 if (encodings->used) { 611 uint16_t * const x = ck_calloc(encodings->used+1, sizeof(short)); 612 int i = 0; 613 for (uint32_t j = 0; j < encodings->used; ++j) { 614 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 615 || defined(USE_ZSTD) 616 data_string *ds = (data_string *)encodings->data[j]; 617 #endif 618 #ifdef USE_ZLIB /* "gzip", "x-gzip" */ 619 if (NULL != strstr(ds->value.ptr, "gzip")) 620 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 621 | HTTP_ACCEPT_ENCODING_X_GZIP; 622 if (NULL != strstr(ds->value.ptr, "deflate")) 623 x[i++] = HTTP_ACCEPT_ENCODING_DEFLATE; 624 /* 625 if (NULL != strstr(ds->value.ptr, "compress")) 626 x[i++] = HTTP_ACCEPT_ENCODING_COMPRESS; 627 */ 628 #endif 629 #ifdef USE_BZ2LIB /* "bzip2", "x-bzip2" */ 630 if (NULL != strstr(ds->value.ptr, "bzip2")) 631 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 632 | HTTP_ACCEPT_ENCODING_X_BZIP2; 633 #endif 634 #ifdef USE_BROTLI /* "br" (also accepts "brotli") */ 635 if (NULL != strstr(ds->value.ptr, "br")) 636 x[i++] = HTTP_ACCEPT_ENCODING_BR; 637 #endif 638 #ifdef USE_ZSTD 639 if (NULL != strstr(ds->value.ptr, "zstd")) 640 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 641 #endif 642 } 643 x[i] = 0; /* end of list */ 644 return x; 645 } 646 else { 647 /* default encodings */ 648 uint16_t * const x = ck_calloc(4+1, sizeof(short)); 649 int i = 0; 650 #ifdef USE_ZSTD 651 x[i++] = HTTP_ACCEPT_ENCODING_ZSTD; 652 #endif 653 #ifdef USE_BROTLI 654 x[i++] = HTTP_ACCEPT_ENCODING_BR; 655 #endif 656 #ifdef USE_BZ2LIB 657 x[i++] = HTTP_ACCEPT_ENCODING_BZIP2 658 | HTTP_ACCEPT_ENCODING_X_BZIP2; 659 #endif 660 #ifdef USE_ZLIB 661 x[i++] = HTTP_ACCEPT_ENCODING_GZIP 662 | HTTP_ACCEPT_ENCODING_X_GZIP 663 | HTTP_ACCEPT_ENCODING_DEFLATE; 664 #endif 665 x[i] = 0; /* end of list */ 666 return x; 667 } 668 } 669 670 SETDEFAULTS_FUNC(mod_deflate_set_defaults) { 671 static const config_plugin_keys_t cpk[] = { 672 { CONST_STR_LEN("deflate.mimetypes"), 673 T_CONFIG_ARRAY_VLIST, 674 T_CONFIG_SCOPE_CONNECTION } 675 ,{ CONST_STR_LEN("deflate.allowed-encodings"), 676 T_CONFIG_ARRAY_VLIST, 677 T_CONFIG_SCOPE_CONNECTION } 678 ,{ CONST_STR_LEN("deflate.max-compress-size"), 679 T_CONFIG_INT, 680 T_CONFIG_SCOPE_CONNECTION } 681 ,{ CONST_STR_LEN("deflate.min-compress-size"), 682 T_CONFIG_SHORT, 683 T_CONFIG_SCOPE_CONNECTION } 684 ,{ CONST_STR_LEN("deflate.compression-level"), 685 T_CONFIG_SHORT, 686 T_CONFIG_SCOPE_CONNECTION } 687 ,{ CONST_STR_LEN("deflate.output-buffer-size"), 688 T_CONFIG_SHORT, 689 T_CONFIG_SCOPE_CONNECTION } 690 ,{ CONST_STR_LEN("deflate.work-block-size"), 691 T_CONFIG_SHORT, 692 T_CONFIG_SCOPE_CONNECTION } 693 ,{ CONST_STR_LEN("deflate.max-loadavg"), 694 T_CONFIG_STRING, 695 T_CONFIG_SCOPE_CONNECTION } 696 ,{ CONST_STR_LEN("deflate.cache-dir"), 697 T_CONFIG_STRING, 698 T_CONFIG_SCOPE_CONNECTION } 699 ,{ CONST_STR_LEN("compress.filetype"), 700 T_CONFIG_ARRAY_VLIST, 701 T_CONFIG_SCOPE_CONNECTION } 702 ,{ CONST_STR_LEN("compress.allowed-encodings"), 703 T_CONFIG_ARRAY_VLIST, 704 T_CONFIG_SCOPE_CONNECTION } 705 ,{ CONST_STR_LEN("compress.cache-dir"), 706 T_CONFIG_STRING, 707 T_CONFIG_SCOPE_CONNECTION } 708 ,{ CONST_STR_LEN("compress.max-filesize"), 709 T_CONFIG_INT, 710 T_CONFIG_SCOPE_CONNECTION } 711 ,{ CONST_STR_LEN("compress.max-loadavg"), 712 T_CONFIG_STRING, 713 T_CONFIG_SCOPE_CONNECTION } 714 ,{ CONST_STR_LEN("deflate.params"), 715 T_CONFIG_ARRAY_KVANY, 716 T_CONFIG_SCOPE_CONNECTION } 717 ,{ NULL, 0, 718 T_CONFIG_UNSET, 719 T_CONFIG_SCOPE_UNSET } 720 }; 721 722 plugin_data * const p = p_d; 723 if (!config_plugin_values_init(srv, p, cpk, "mod_deflate")) 724 return HANDLER_ERROR; 725 726 /* process and validate config directives 727 * (init i to 0 if global context; to 1 to skip empty global context) */ 728 for (int i = !p->cvlist[0].v.u2[1]; i < p->nconfig; ++i) { 729 config_plugin_value_t *cpv = p->cvlist + p->cvlist[i].v.u2[0]; 730 for (; -1 != cpv->k_id; ++cpv) { 731 switch (cpv->k_id) { 732 case 9: /* compress.filetype */ 733 log_error(srv->errh, __FILE__, __LINE__, 734 "DEPRECATED: %s replaced with deflate.mimetypes", 735 cpk[cpv->k_id].k); 736 cpv->k_id = 0; /* deflate.mimetypes */ 737 __attribute_fallthrough__ 738 case 0: /* deflate.mimetypes */ 739 /* mod_deflate matches mimetype as prefix of Content-Type 740 * so ignore '*' at end of mimetype for end-user flexibility 741 * in specifying trailing wildcard to grouping of mimetypes */ 742 for (uint32_t m = 0; m < cpv->v.a->used; ++m) { 743 buffer *mimetype=&((data_string *)cpv->v.a->data[m])->value; 744 size_t len = buffer_clen(mimetype); 745 if (len > 2 && mimetype->ptr[len-1] == '*') 746 buffer_truncate(mimetype, len-1); 747 if (buffer_eq_slen(mimetype, 748 CONST_STR_LEN("application/javascript"))) 749 buffer_copy_string_len(mimetype, "text/javascript", 15); 750 } 751 if (0 == cpv->v.a->used) cpv->v.a = NULL; 752 break; 753 case 10:/* compress.allowed-encodings */ 754 log_error(srv->errh, __FILE__, __LINE__, 755 "DEPRECATED: %s replaced with deflate.allowed-encodings", 756 cpk[cpv->k_id].k); 757 cpv->k_id = 1; /* deflate.allowed-encodings */ 758 __attribute_fallthrough__ 759 case 1: /* deflate.allowed-encodings */ 760 cpv->v.v = mod_deflate_encodings_to_flags(cpv->v.a); 761 cpv->vtype = T_CONFIG_LOCAL; 762 break; 763 case 12:/* compress.max-filesize */ 764 log_error(srv->errh, __FILE__, __LINE__, 765 "DEPRECATED: %s replaced with deflate.max-compress-size", 766 cpk[cpv->k_id].k); 767 cpv->k_id = 2; /* deflate.max-compress-size */ 768 __attribute_fallthrough__ 769 case 2: /* deflate.max-compress-size */ 770 case 3: /* deflate.min-compress-size */ 771 break; 772 case 4: /* deflate.compression-level */ 773 if ((cpv->v.shrt < 1 || cpv->v.shrt > 9) 774 && *(short *)&cpv->v.shrt != -1) { 775 log_error(srv->errh, __FILE__, __LINE__, 776 "compression-level must be between 1 and 9: %hu", 777 cpv->v.shrt); 778 return HANDLER_ERROR; 779 } 780 break; 781 case 5: /* deflate.output-buffer-size */ 782 case 6: /* deflate.work-block-size */ 783 break; 784 case 13:/* compress.max-loadavg */ 785 log_error(srv->errh, __FILE__, __LINE__, 786 "DEPRECATED: %s replaced with deflate.max-loadavg", 787 cpk[cpv->k_id].k); 788 cpv->k_id = 7; /* deflate.max-loadavg */ 789 __attribute_fallthrough__ 790 case 7: /* deflate.max-loadavg */ 791 cpv->v.d = (!buffer_is_blank(cpv->v.b)) 792 ? strtod(cpv->v.b->ptr, NULL) 793 : 0.0; 794 break; 795 case 11:/* compress.cache-dir */ 796 log_error(srv->errh, __FILE__, __LINE__, 797 "DEPRECATED: %s replaced with deflate.cache-dir", 798 cpk[cpv->k_id].k); 799 cpv->k_id = 8; /* deflate.cache-dir */ 800 __attribute_fallthrough__ 801 case 8: /* deflate.cache-dir */ 802 if (!buffer_is_blank(cpv->v.b)) { 803 buffer *b; 804 *(const buffer **)&b = cpv->v.b; 805 const uint32_t len = buffer_clen(b); 806 if (len > 0 && '/' == b->ptr[len-1]) 807 buffer_truncate(b, len-1); /*remove end slash*/ 808 struct stat st; 809 if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) { 810 log_perror(srv->errh, __FILE__, __LINE__, 811 "can't stat %s %s", cpk[cpv->k_id].k, b->ptr); 812 return HANDLER_ERROR; 813 } 814 } 815 else 816 cpv->v.b = NULL; 817 break; 818 #if 0 /*(handled further above)*/ 819 case 9: /* compress.filetype */ 820 case 10:/* compress.allowed-encodings */ 821 case 11:/* compress.cache-dir */ 822 case 12:/* compress.max-filesize */ 823 case 13:/* compress.max-loadavg */ 824 break; 825 #endif 826 case 14:/* deflate.params */ 827 cpv->v.v = mod_deflate_parse_params(cpv->v.a, srv->errh); 828 cpv->vtype = T_CONFIG_LOCAL; 829 break; 830 default:/* should not happen */ 831 break; 832 } 833 } 834 } 835 836 p->defaults.max_compress_size = 128*1024; /*(128 MB measured as num KB)*/ 837 p->defaults.min_compress_size = 256; 838 p->defaults.compression_level = -1; 839 p->defaults.output_buffer_size = 0; 840 p->defaults.work_block_size = 2048; 841 p->defaults.max_loadavg = 0.0; 842 p->defaults.sync_flush = 0; 843 844 /* initialize p->defaults from global config context */ 845 if (p->nconfig > 0 && p->cvlist->v.u2[1]) { 846 const config_plugin_value_t *cpv = p->cvlist + p->cvlist->v.u2[0]; 847 if (-1 != cpv->k_id) 848 mod_deflate_merge_config(&p->defaults, cpv); 849 } 850 851 return HANDLER_GO_ON; 852 } 853 854 855 #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ 856 || defined(USE_ZSTD) 857 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { 858 ssize_t wr; 859 do { 860 wr = write(hctx->cache_fd, out, len); 861 } while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR); 862 return (0 == len) ? 0 : -1; 863 } 864 865 static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) { 866 if (0 == len) return 0; 867 return (-1 == hctx->cache_fd) 868 ? http_chunk_append_mem(hctx->r, out, len) 869 : mod_deflate_cache_file_append(hctx, out, len); 870 } 871 #endif 872 873 874 #ifdef USE_ZLIB 875 876 static int stream_deflate_init(handler_ctx *hctx) { 877 z_stream * const z = &hctx->u.z; 878 z->zalloc = Z_NULL; 879 z->zfree = Z_NULL; 880 z->opaque = Z_NULL; 881 z->total_in = 0; 882 z->total_out = 0; 883 z->next_out = (unsigned char *)hctx->output->ptr; 884 z->avail_out = hctx->output->size; 885 886 const plugin_data * const p = hctx->plugin_data; 887 const encparms * const params = p->conf.params; 888 const int clevel = (NULL != params) 889 ? params->gzip.clevel 890 : p->conf.compression_level; 891 const int wbits = (NULL != params) 892 ? params->gzip.windowBits 893 : MAX_WBITS; 894 895 if (Z_OK != deflateInit2(z, 896 clevel > 0 ? clevel : Z_DEFAULT_COMPRESSION, 897 Z_DEFLATED, 898 (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 899 ? (wbits | 16) /*(0x10 flags gzip header, trailer)*/ 900 : wbits, 901 params ? params->gzip.memLevel : 8,/*default memLevel*/ 902 params ? params->gzip.strategy : Z_DEFAULT_STRATEGY)) { 903 return -1; 904 } 905 906 return 0; 907 } 908 909 static int stream_deflate_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 910 z_stream * const z = &(hctx->u.z); 911 size_t len; 912 913 /*(unknown whether or not linked zlib was built with ZLIB_CONST defined)*/ 914 *((const unsigned char **)&z->next_in) = start; 915 z->avail_in = st_size; 916 hctx->bytes_in += st_size; 917 918 /* compress data */ 919 do { 920 if (Z_OK != deflate(z, Z_NO_FLUSH)) return -1; 921 922 if (z->avail_out == 0 || z->avail_in > 0) { 923 len = hctx->output->size - z->avail_out; 924 hctx->bytes_out += len; 925 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 926 return -1; 927 z->next_out = (unsigned char *)hctx->output->ptr; 928 z->avail_out = hctx->output->size; 929 } 930 } while (z->avail_in > 0); 931 932 return 0; 933 } 934 935 static int stream_deflate_flush(handler_ctx * const hctx, int end) { 936 z_stream * const z = &(hctx->u.z); 937 const plugin_data *p = hctx->plugin_data; 938 size_t len; 939 int rc = 0; 940 int done; 941 942 /* compress data */ 943 do { 944 done = 1; 945 if (end) { 946 rc = deflate(z, Z_FINISH); 947 if (rc == Z_OK) { 948 done = 0; 949 } else if (rc != Z_STREAM_END) { 950 return -1; 951 } 952 } else { 953 if (p->conf.sync_flush) { 954 rc = deflate(z, Z_SYNC_FLUSH); 955 if (rc != Z_OK) return -1; 956 } else if (z->avail_in > 0) { 957 rc = deflate(z, Z_NO_FLUSH); 958 if (rc != Z_OK) return -1; 959 } 960 } 961 962 len = hctx->output->size - z->avail_out; 963 if (z->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 964 hctx->bytes_out += len; 965 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 966 return -1; 967 z->next_out = (unsigned char *)hctx->output->ptr; 968 z->avail_out = hctx->output->size; 969 } 970 } while (z->avail_in != 0 || !done); 971 972 return 0; 973 } 974 975 static int stream_deflate_end(handler_ctx *hctx) { 976 z_stream * const z = &(hctx->u.z); 977 int rc = deflateEnd(z); 978 if (Z_OK == rc || Z_DATA_ERROR == rc) return 0; 979 980 if (z->msg != NULL) { 981 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 982 "deflateEnd error ret=%d, msg=%s", rc, z->msg); 983 } else { 984 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 985 "deflateEnd error ret=%d", rc); 986 } 987 return -1; 988 } 989 990 #endif 991 992 993 #ifdef USE_BZ2LIB 994 995 static int stream_bzip2_init(handler_ctx *hctx) { 996 bz_stream * const bz = &hctx->u.bz; 997 bz->bzalloc = NULL; 998 bz->bzfree = NULL; 999 bz->opaque = NULL; 1000 bz->total_in_lo32 = 0; 1001 bz->total_in_hi32 = 0; 1002 bz->total_out_lo32 = 0; 1003 bz->total_out_hi32 = 0; 1004 bz->next_out = hctx->output->ptr; 1005 bz->avail_out = hctx->output->size; 1006 1007 const plugin_data * const p = hctx->plugin_data; 1008 const encparms * const params = p->conf.params; 1009 const int clevel = (NULL != params) 1010 ? params->bzip2.clevel 1011 : p->conf.compression_level; 1012 1013 if (BZ_OK != BZ2_bzCompressInit(bz, 1014 clevel > 0 1015 ? p->conf.compression_level 1016 : 9, /* blocksize = 900k */ 1017 0, /* verbosity */ 1018 0)) { /* workFactor: default */ 1019 return -1; 1020 } 1021 1022 return 0; 1023 } 1024 1025 static int stream_bzip2_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1026 bz_stream * const bz = &(hctx->u.bz); 1027 size_t len; 1028 1029 bz->next_in = (char *)start; 1030 bz->avail_in = st_size; 1031 hctx->bytes_in += st_size; 1032 1033 /* compress data */ 1034 do { 1035 if (BZ_RUN_OK != BZ2_bzCompress(bz, BZ_RUN)) return -1; 1036 1037 if (bz->avail_out == 0 || bz->avail_in > 0) { 1038 len = hctx->output->size - bz->avail_out; 1039 hctx->bytes_out += len; 1040 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1041 return -1; 1042 bz->next_out = hctx->output->ptr; 1043 bz->avail_out = hctx->output->size; 1044 } 1045 } while (bz->avail_in > 0); 1046 1047 return 0; 1048 } 1049 1050 static int stream_bzip2_flush(handler_ctx * const hctx, int end) { 1051 bz_stream * const bz = &(hctx->u.bz); 1052 const plugin_data *p = hctx->plugin_data; 1053 size_t len; 1054 int rc; 1055 int done; 1056 1057 /* compress data */ 1058 do { 1059 done = 1; 1060 if (end) { 1061 rc = BZ2_bzCompress(bz, BZ_FINISH); 1062 if (rc == BZ_FINISH_OK) { 1063 done = 0; 1064 } else if (rc != BZ_STREAM_END) { 1065 return -1; 1066 } 1067 } else if (bz->avail_in > 0) { 1068 /* p->conf.sync_flush not implemented here, 1069 * which would loop on BZ_FLUSH while BZ_FLUSH_OK 1070 * until BZ_RUN_OK returned */ 1071 rc = BZ2_bzCompress(bz, BZ_RUN); 1072 if (rc != BZ_RUN_OK) { 1073 return -1; 1074 } 1075 } 1076 1077 len = hctx->output->size - bz->avail_out; 1078 if (bz->avail_out == 0 || (len > 0 && (end || p->conf.sync_flush))) { 1079 hctx->bytes_out += len; 1080 if (0 != stream_http_chunk_append_mem(hctx, hctx->output->ptr, len)) 1081 return -1; 1082 bz->next_out = hctx->output->ptr; 1083 bz->avail_out = hctx->output->size; 1084 } 1085 } while (bz->avail_in != 0 || !done); 1086 1087 return 0; 1088 } 1089 1090 static int stream_bzip2_end(handler_ctx *hctx) { 1091 bz_stream * const bz = &(hctx->u.bz); 1092 int rc = BZ2_bzCompressEnd(bz); 1093 if (BZ_OK == rc || BZ_DATA_ERROR == rc) return 0; 1094 1095 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 1096 "BZ2_bzCompressEnd error ret=%d", rc); 1097 return -1; 1098 } 1099 1100 #endif 1101 1102 1103 #ifdef USE_BROTLI 1104 1105 static int stream_br_init(handler_ctx *hctx) { 1106 BrotliEncoderState * const br = hctx->u.br = 1107 BrotliEncoderCreateInstance(NULL, NULL, NULL); 1108 if (NULL == br) return -1; 1109 1110 /*(note: we ignore any errors while tuning parameters here)*/ 1111 const plugin_data * const p = hctx->plugin_data; 1112 const encparms * const params = p->conf.params; 1113 const uint32_t quality = (NULL != params) 1114 ? params->brotli.quality 1115 : (p->conf.compression_level >= 0) /* 0 .. 11 are valid values */ 1116 ? (uint32_t)p->conf.compression_level 1117 : 5; 1118 /* BROTLI_DEFAULT_QUALITY is 11 and can be *very* time-consuming */ 1119 if (quality != BROTLI_DEFAULT_QUALITY) 1120 BrotliEncoderSetParameter(br, BROTLI_PARAM_QUALITY, quality); 1121 1122 if (params && params->brotli.window != BROTLI_DEFAULT_WINDOW) 1123 BrotliEncoderSetParameter(br, BROTLI_PARAM_LGWIN,params->brotli.window); 1124 1125 const buffer *vb; 1126 if (params && params->brotli.mode != BROTLI_MODE_GENERIC) 1127 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, params->brotli.mode); 1128 else if ((vb = http_header_response_get(hctx->r, HTTP_HEADER_CONTENT_TYPE, 1129 CONST_STR_LEN("Content-Type")))) { 1130 /* BROTLI_MODE_GENERIC vs BROTLI_MODE_TEXT or BROTLI_MODE_FONT */ 1131 const uint32_t len = buffer_clen(vb); 1132 if (0 == strncmp(vb->ptr, "text/", sizeof("text/")-1) 1133 || (0 == strncmp(vb->ptr, "application/", sizeof("application/")-1) 1134 && (0 == strncmp(vb->ptr+12,"javascript",sizeof("javascript")-1) 1135 || 0 == strncmp(vb->ptr+12,"json", sizeof("json")-1) 1136 || 0 == strncmp(vb->ptr+12,"xml", sizeof("xml")-1))) 1137 || (len > 4 1138 && (0 == strncmp(vb->ptr+len-5, "+json", sizeof("+json")-1) 1139 || 0 == strncmp(vb->ptr+len-4, "+xml", sizeof("+xml")-1)))) 1140 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_TEXT); 1141 else if (0 == strncmp(vb->ptr, "font/", sizeof("font/")-1)) 1142 BrotliEncoderSetParameter(br, BROTLI_PARAM_MODE, BROTLI_MODE_FONT); 1143 } 1144 1145 return 0; 1146 } 1147 1148 static int stream_br_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1149 const uint8_t *in = (uint8_t *)start; 1150 BrotliEncoderState * const br = hctx->u.br; 1151 hctx->bytes_in += st_size; 1152 while (st_size || BrotliEncoderHasMoreOutput(br)) { 1153 size_t insz = ((off_t)((~(uint32_t)0) >> 1) > st_size) 1154 ? (size_t)st_size 1155 : ((~(uint32_t)0) >> 1); 1156 size_t outsz = 0; 1157 BrotliEncoderCompressStream(br, BROTLI_OPERATION_PROCESS, 1158 &insz, &in, &outsz, NULL, NULL); 1159 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1160 st_size -= (st_size - (off_t)insz); 1161 if (outsz) { 1162 hctx->bytes_out += (off_t)outsz; 1163 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1164 return -1; 1165 } 1166 } 1167 return 0; 1168 } 1169 1170 static int stream_br_flush(handler_ctx * const hctx, int end) { 1171 const int brmode = end ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; 1172 BrotliEncoderState * const br = hctx->u.br; 1173 do { 1174 size_t insz = 0; 1175 size_t outsz = 0; 1176 BrotliEncoderCompressStream(br, brmode, 1177 &insz, NULL, &outsz, NULL, NULL); 1178 const uint8_t *out = BrotliEncoderTakeOutput(br, &outsz); 1179 if (outsz) { 1180 hctx->bytes_out += (off_t)outsz; 1181 if (0 != stream_http_chunk_append_mem(hctx, (char *)out, outsz)) 1182 return -1; 1183 } 1184 } while (BrotliEncoderHasMoreOutput(br)); 1185 return 0; 1186 } 1187 1188 static int stream_br_end(handler_ctx *hctx) { 1189 BrotliEncoderState * const br = hctx->u.br; 1190 BrotliEncoderDestroyInstance(br); 1191 return 0; 1192 } 1193 1194 #endif 1195 1196 1197 #ifdef USE_ZSTD 1198 1199 static int stream_zstd_init(handler_ctx *hctx) { 1200 ZSTD_CStream * const cctx = hctx->u.cctx = ZSTD_createCStream(); 1201 if (NULL == cctx) return -1; 1202 hctx->output->used = 0; 1203 1204 /*(note: we ignore any errors while tuning parameters here)*/ 1205 const plugin_data * const p = hctx->plugin_data; 1206 const encparms * const params = p->conf.params; 1207 if (params) { 1208 if (params->zstd.clevel && params->zstd.clevel != ZSTD_CLEVEL_DEFAULT) { 1209 const int level = params->zstd.clevel; 1210 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1211 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level); 1212 #else 1213 ZSTD_initCStream(cctx, level); 1214 #endif 1215 } 1216 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1217 if (params->zstd.strategy) 1218 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, 1219 params->zstd.strategy); 1220 if (params->zstd.windowLog) 1221 ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 1222 params->zstd.windowLog); 1223 #endif 1224 } 1225 else if (p->conf.compression_level >= 0) { /* -1 here is "unset" */ 1226 int level = p->conf.compression_level; 1227 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1228 ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, level); 1229 #else 1230 ZSTD_initCStream(cctx, level); 1231 #endif 1232 } 1233 return 0; 1234 } 1235 1236 static int stream_zstd_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1237 ZSTD_CStream * const cctx = hctx->u.cctx; 1238 ZSTD_inBuffer zib = { start, (size_t)st_size, 0 }; 1239 ZSTD_outBuffer zob = { hctx->output->ptr, 1240 hctx->output->size, 1241 hctx->output->used }; 1242 hctx->output->used = 0; 1243 hctx->bytes_in += st_size; 1244 while (zib.pos < zib.size) { 1245 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1246 const size_t rv = ZSTD_compressStream2(cctx,&zob,&zib,ZSTD_e_continue); 1247 #else 1248 const size_t rv = ZSTD_compressStream(cctx, &zob, &zib); 1249 #endif 1250 if (ZSTD_isError(rv)) return -1; 1251 if (zib.pos == zib.size) break; /* defer flush */ 1252 hctx->bytes_out += (off_t)zob.pos; 1253 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1254 return -1; 1255 zob.pos = 0; 1256 } 1257 hctx->output->used = (uint32_t)zob.pos; 1258 return 0; 1259 } 1260 1261 static int stream_zstd_flush(handler_ctx * const hctx, int end) { 1262 ZSTD_CStream * const cctx = hctx->u.cctx; 1263 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1264 const ZSTD_EndDirective endOp = end ? ZSTD_e_end : ZSTD_e_flush; 1265 ZSTD_inBuffer zib = { NULL, 0, 0 }; 1266 #endif 1267 ZSTD_outBuffer zob = { hctx->output->ptr, 1268 hctx->output->size, 1269 hctx->output->used }; 1270 size_t rv; 1271 do { 1272 #if ZSTD_VERSION_NUMBER >= 10000+400+0 /* v1.4.0 */ 1273 rv = ZSTD_compressStream2(cctx, &zob, &zib, endOp); 1274 #else 1275 rv = end 1276 ? ZSTD_endStream(cctx, &zob) 1277 : ZSTD_flushStream(cctx, &zob); 1278 #endif 1279 if (ZSTD_isError(rv)) return -1; 1280 hctx->bytes_out += (off_t)zob.pos; 1281 if (0 != stream_http_chunk_append_mem(hctx, zob.dst, zob.pos)) 1282 return -1; 1283 zob.pos = 0; 1284 } while (0 != rv); 1285 return 0; 1286 } 1287 1288 static int stream_zstd_end(handler_ctx *hctx) { 1289 ZSTD_CStream * const cctx = hctx->u.cctx; 1290 ZSTD_freeCStream(cctx); 1291 return 0; 1292 } 1293 1294 #endif 1295 1296 1297 static int mod_deflate_stream_init(handler_ctx *hctx) { 1298 switch(hctx->compression_type) { 1299 #ifdef USE_ZLIB 1300 case HTTP_ACCEPT_ENCODING_GZIP: 1301 case HTTP_ACCEPT_ENCODING_DEFLATE: 1302 return stream_deflate_init(hctx); 1303 #endif 1304 #ifdef USE_BZ2LIB 1305 case HTTP_ACCEPT_ENCODING_BZIP2: 1306 return stream_bzip2_init(hctx); 1307 #endif 1308 #ifdef USE_BROTLI 1309 case HTTP_ACCEPT_ENCODING_BR: 1310 return stream_br_init(hctx); 1311 #endif 1312 #ifdef USE_ZSTD 1313 case HTTP_ACCEPT_ENCODING_ZSTD: 1314 return stream_zstd_init(hctx); 1315 #endif 1316 default: 1317 return -1; 1318 } 1319 } 1320 1321 static int mod_deflate_compress(handler_ctx * const hctx, const unsigned char * const start, off_t st_size) { 1322 if (0 == st_size) return 0; 1323 switch(hctx->compression_type) { 1324 #ifdef USE_ZLIB 1325 case HTTP_ACCEPT_ENCODING_GZIP: 1326 case HTTP_ACCEPT_ENCODING_DEFLATE: 1327 return stream_deflate_compress(hctx, start, st_size); 1328 #endif 1329 #ifdef USE_BZ2LIB 1330 case HTTP_ACCEPT_ENCODING_BZIP2: 1331 return stream_bzip2_compress(hctx, start, st_size); 1332 #endif 1333 #ifdef USE_BROTLI 1334 case HTTP_ACCEPT_ENCODING_BR: 1335 return stream_br_compress(hctx, start, st_size); 1336 #endif 1337 #ifdef USE_ZSTD 1338 case HTTP_ACCEPT_ENCODING_ZSTD: 1339 return stream_zstd_compress(hctx, start, st_size); 1340 #endif 1341 default: 1342 UNUSED(start); 1343 return -1; 1344 } 1345 } 1346 1347 static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { 1348 if (0 == hctx->bytes_in) return 0; 1349 switch(hctx->compression_type) { 1350 #ifdef USE_ZLIB 1351 case HTTP_ACCEPT_ENCODING_GZIP: 1352 case HTTP_ACCEPT_ENCODING_DEFLATE: 1353 return stream_deflate_flush(hctx, end); 1354 #endif 1355 #ifdef USE_BZ2LIB 1356 case HTTP_ACCEPT_ENCODING_BZIP2: 1357 return stream_bzip2_flush(hctx, end); 1358 #endif 1359 #ifdef USE_BROTLI 1360 case HTTP_ACCEPT_ENCODING_BR: 1361 return stream_br_flush(hctx, end); 1362 #endif 1363 #ifdef USE_ZSTD 1364 case HTTP_ACCEPT_ENCODING_ZSTD: 1365 return stream_zstd_flush(hctx, end); 1366 #endif 1367 default: 1368 UNUSED(end); 1369 return -1; 1370 } 1371 } 1372 1373 static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) { 1374 /* store compression ratio in environment 1375 * for possible logging by mod_accesslog 1376 * (late in response handling, so not seen by most other modules) */ 1377 /*(should be called only at end of successful response compression)*/ 1378 if (0 == bytes_in) return; 1379 buffer_append_int( 1380 http_header_env_set_ptr(r, CONST_STR_LEN("ratio")), 1381 bytes_out * 100 / bytes_in); 1382 } 1383 1384 static int mod_deflate_stream_end(handler_ctx *hctx) { 1385 switch(hctx->compression_type) { 1386 #ifdef USE_ZLIB 1387 case HTTP_ACCEPT_ENCODING_GZIP: 1388 case HTTP_ACCEPT_ENCODING_DEFLATE: 1389 return stream_deflate_end(hctx); 1390 #endif 1391 #ifdef USE_BZ2LIB 1392 case HTTP_ACCEPT_ENCODING_BZIP2: 1393 return stream_bzip2_end(hctx); 1394 #endif 1395 #ifdef USE_BROTLI 1396 case HTTP_ACCEPT_ENCODING_BR: 1397 return stream_br_end(hctx); 1398 #endif 1399 #ifdef USE_ZSTD 1400 case HTTP_ACCEPT_ENCODING_ZSTD: 1401 return stream_zstd_end(hctx); 1402 #endif 1403 default: 1404 return -1; 1405 } 1406 } 1407 1408 static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) { 1409 int rc = mod_deflate_stream_end(hctx); 1410 1411 #if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */ 1412 if (0 == rc && hctx->bytes_in < hctx->bytes_out) 1413 log_error(r->conf.errh, __FILE__, __LINE__, 1414 "uri %s in=%lld smaller than out=%lld", r->target.ptr, 1415 (long long)hctx->bytes_in, (long long)hctx->bytes_out); 1416 #endif 1417 1418 handler_ctx_free(hctx); 1419 return rc; 1420 } 1421 1422 1423 #ifdef HAVE_LIBDEFLATE 1424 #include <libdeflate.h> 1425 1426 __attribute_cold__ 1427 __attribute_noinline__ 1428 static int mod_deflate_using_libdeflate_err (handler_ctx * const hctx, buffer * const fn, const int fd) 1429 { 1430 if (-1 == fd) { 1431 } 1432 else if (fd == hctx->cache_fd) { 1433 if (0 != ftruncate(fd, 0)) 1434 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "ftruncate"); 1435 if (0 != lseek(fd, 0, SEEK_SET)) 1436 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "lseek"); 1437 } 1438 else { 1439 if (0 != unlink(fn->ptr)) 1440 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "unlink"); 1441 close(fd); 1442 } 1443 buffer_clear(fn); /*(&p->tmp_buf)*/ 1444 return 0; 1445 } 1446 1447 1448 static int mod_deflate_using_libdeflate_sm (handler_ctx * const hctx, const plugin_data * const p) 1449 { 1450 const encparms * const params = p->conf.params; 1451 const int clevel = (NULL != params) 1452 ? params->gzip.clevel 1453 : p->conf.compression_level; 1454 struct libdeflate_compressor * const compressor = 1455 libdeflate_alloc_compressor(clevel > 0 ? clevel : 6); 1456 /* Z_DEFAULT_COMPRESSION -1 not supported */ 1457 if (NULL == compressor) 1458 return 0; 1459 1460 char * const in = hctx->r->write_queue.first->mem->ptr; 1461 const size_t in_nbytes = (size_t)hctx->bytes_in; 1462 buffer * const addrb = hctx->output; /*(&p->tmp_buf)*/ 1463 size_t sz = buffer_string_space(addrb)+1; 1464 sz = (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 1465 ? libdeflate_gzip_compress(compressor, in, in_nbytes, addrb->ptr, sz) 1466 : libdeflate_zlib_compress(compressor, in, in_nbytes, addrb->ptr, sz); 1467 libdeflate_free_compressor(compressor); 1468 1469 if (0 == sz) { 1470 buffer_clear(addrb); 1471 return 0; 1472 } 1473 1474 chunkqueue_reset(&hctx->r->write_queue); 1475 hctx->bytes_out = (off_t)sz; 1476 if (0 != stream_http_chunk_append_mem(hctx, addrb->ptr, sz)) 1477 return mod_deflate_using_libdeflate_err(hctx, addrb, hctx->cache_fd); 1478 1479 buffer * const vb = 1480 http_header_response_set_ptr(hctx->r, HTTP_HEADER_CONTENT_LENGTH, 1481 CONST_STR_LEN("Content-Length")); 1482 buffer_append_int(vb, hctx->bytes_out); 1483 return 1; 1484 } 1485 1486 1487 #ifdef HAVE_MMAP 1488 #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) 1489 1490 struct mod_deflate_setjmp_params { 1491 struct libdeflate_compressor *compressor; 1492 void *out; 1493 size_t outsz; 1494 }; 1495 1496 static off_t mod_deflate_using_libdeflate_setjmp_cb (void *dst, const void *src, off_t len) 1497 { 1498 const struct mod_deflate_setjmp_params * const params = dst; 1499 const handler_ctx * const hctx = src; 1500 const chunk * const c = hctx->r->write_queue.first; 1501 const char *in = chunk_file_view_dptr(c->file.view, c->offset); 1502 return (off_t)((hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP) 1503 ? libdeflate_gzip_compress(params->compressor, in, (size_t)len, 1504 params->out, params->outsz) 1505 : libdeflate_zlib_compress(params->compressor, in, (size_t)len, 1506 params->out, params->outsz)); 1507 } 1508 1509 1510 static int mod_deflate_using_libdeflate (handler_ctx * const hctx, const plugin_data * const p) 1511 { 1512 buffer * const fn = hctx->output; /*(&p->tmp_buf)*/ 1513 int fd = hctx->cache_fd; 1514 if (-1 == fd) { 1515 chunkqueue * const cq = &hctx->r->write_queue; 1516 if (cq->tempdirs && cq->tempdir_idx < cq->tempdirs->used) { 1517 data_string *ds =(data_string *)cq->tempdirs->data[cq->tempdir_idx]; 1518 buffer_copy_string_len(fn, BUF_PTR_LEN(&ds->value)); 1519 } 1520 else 1521 buffer_copy_string_len(fn, CONST_STR_LEN("/var/tmp")); 1522 buffer_append_path_len(fn, CONST_STR_LEN("lighttpd-XXXXXX")); 1523 fd = fdevent_mkostemp(fn->ptr, 0); 1524 if (-1 == fd) return 0; 1525 } 1526 1527 const size_t sz = 1528 libdeflate_zlib_compress_bound(NULL, (size_t)hctx->bytes_in); 1529 /*(XXX: consider trying posix_fallocate() first, 1530 * with fallback to ftrunctate() if EOPNOTSUPP)*/ 1531 if (0 != ftruncate(fd, (off_t)sz)) { 1532 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "ftruncate"); 1533 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1534 } 1535 1536 /*void *addr = mmap(NULL, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);*/ 1537 void * const addr = mmap(NULL, sz, PROT_WRITE, MAP_SHARED, fd, 0); 1538 if (MAP_FAILED == addr) { 1539 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "mmap"); 1540 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1541 } 1542 1543 const encparms * const params = p->conf.params; 1544 const int clevel = (NULL != params) 1545 ? params->gzip.clevel 1546 : p->conf.compression_level; 1547 struct libdeflate_compressor * const compressor = 1548 libdeflate_alloc_compressor(clevel > 0 ? clevel : 6); 1549 /* Z_DEFAULT_COMPRESSION -1 not supported */ 1550 if (NULL != compressor) { 1551 struct mod_deflate_setjmp_params outparams = { compressor, addr, sz }; 1552 hctx->bytes_out = 1553 sys_setjmp_eval3(mod_deflate_using_libdeflate_setjmp_cb, 1554 &outparams, hctx, hctx->bytes_in); 1555 libdeflate_free_compressor(compressor); 1556 } 1557 1558 /*(XXX: we theoretically could assign mmap to FILE_CHUNK in output 1559 * r->write_queue, for potential use by TLS modules, or if not using 1560 * sendfile in network_write.c, but we do not (easily) know if either 1561 * are configured or in use for this request. Might consider heuristic: 1562 * (*srv->srvconf.network_backend->ptr == 'w') (writev or write), or 1563 * to check con->is_ssl_sock.) These files would be safe to pass to 1564 * TLS modules to read from mmap without catching SIGBUS since these files 1565 * are created by lighttpd, either in deflate cache or as temporary file */ 1566 if (0 != munmap(addr, sz)) 1567 log_perror(hctx->r->conf.errh, __FILE__, __LINE__, "munmap"); 1568 1569 if (0 == hctx->bytes_out) { 1570 const chunk * const c = hctx->r->write_queue.first; 1571 log_error(hctx->r->conf.errh, __FILE__, __LINE__, 1572 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1573 } 1574 else if (0 != ftruncate(fd, hctx->bytes_out)) 1575 hctx->bytes_out = 0; 1576 1577 if (0 == hctx->bytes_out) 1578 return mod_deflate_using_libdeflate_err(hctx, fn, fd); 1579 1580 if (fd != hctx->cache_fd) { 1581 chunkqueue * const cq = &hctx->r->write_queue; 1582 chunkqueue_reset(cq); 1583 http_chunk_append_file_fd_range(hctx->r, fn, fd, 0, hctx->bytes_out); 1584 cq->last->file.is_temp = 1; 1585 } 1586 1587 buffer * const vb = 1588 http_header_response_set_ptr(hctx->r, HTTP_HEADER_CONTENT_LENGTH, 1589 CONST_STR_LEN("Content-Length")); 1590 buffer_append_int(vb, hctx->bytes_out); 1591 return 1; 1592 } 1593 1594 #endif /* defined(_LP64) || defined(__LP64__) || defined(_WIN64) */ 1595 #endif /* HAVE_MMAP */ 1596 1597 #endif /* HAVE_LIBDEFLATE */ 1598 1599 1600 static off_t mod_deflate_file_chunk_no_mmap(request_st * const r, handler_ctx * const hctx, const chunk * const c, off_t n) 1601 { 1602 const off_t insz = n; 1603 const size_t psz = (n < 2*1024*1024) ? (size_t)n : 2*1024*1024; 1604 char * const p = malloc(psz); 1605 if (NULL == p) { 1606 log_perror(r->conf.errh, __FILE__, __LINE__, "malloc"); 1607 return -1; 1608 } 1609 1610 ssize_t rd = 0; 1611 for (n = 0; n < insz; n += rd) { 1612 rd = chunk_file_pread(c->file.fd, p, (size_t)psz, c->offset+n); 1613 if (__builtin_expect( (rd > 0), 1)) { 1614 if (0 == mod_deflate_compress(hctx, (unsigned char *)p, rd)) 1615 continue; 1616 /*(else error trace printed upon return)*/ 1617 } 1618 else if (-1 == rd) 1619 log_perror(r->conf.errh, __FILE__, __LINE__, 1620 "reading %s failed", c->mem->ptr); 1621 else /*(0 == rd)*/ 1622 log_error(r->conf.errh, __FILE__, __LINE__, 1623 "file truncated %s", c->mem->ptr); 1624 n = -1; 1625 break; 1626 } 1627 1628 free(p); 1629 return n; 1630 } 1631 1632 1633 #ifdef ENABLE_MMAP 1634 1635 static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t len) 1636 { 1637 return mod_deflate_compress(dst, (const unsigned char *)src, len); 1638 } 1639 1640 1641 static off_t mod_deflate_file_chunk_mmap(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n) 1642 { 1643 /* n is length of entire file since server blocks while compressing 1644 * (mod_deflate is not recommended for large files; 1645 * mod_deflate default upper limit is 128MB; deflate.max-compress-size) */ 1646 1647 const chunk_file_view * const restrict cfv = (!c->file.is_temp) 1648 ? chunkqueue_chunk_file_view(c, n, r->conf.errh) 1649 : NULL; 1650 if (NULL == cfv) 1651 return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); 1652 1653 const char * const p = chunk_file_view_dptr(cfv, c->offset); 1654 off_t len = chunk_file_view_dlen(cfv, c->offset); 1655 if (len > n) len = n; 1656 off_t rc = sys_setjmp_eval3(mod_deflate_file_chunk_setjmp_cb, hctx, p, len); 1657 if (__builtin_expect( (rc < 0), 0)) { 1658 if (errno == EFAULT) 1659 log_error(r->conf.errh, __FILE__, __LINE__, 1660 "SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd); 1661 else 1662 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1663 len = -1; /*return -1;*/ 1664 } 1665 return len; 1666 } 1667 1668 #endif 1669 1670 1671 static off_t mod_deflate_file_chunk(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n) { 1672 if (-1 == c->file.fd) { /* open the file if not already open */ 1673 if (-1 == (c->file.fd = fdevent_open_cloexec(c->mem->ptr, r->conf.follow_symlink, O_RDONLY, 0))) { 1674 log_perror(r->conf.errh, __FILE__, __LINE__, "open failed %s", c->mem->ptr); 1675 return -1; 1676 } 1677 } 1678 #ifdef ENABLE_MMAP 1679 return mod_deflate_file_chunk_mmap(r, hctx, c, n); 1680 #else 1681 return mod_deflate_file_chunk_no_mmap(r, hctx, c, n); 1682 #endif 1683 } 1684 1685 1686 static handler_t deflate_compress_response(request_st * const r, handler_ctx * const hctx) { 1687 off_t len, max; 1688 int close_stream; 1689 1690 /* move all chunk from write_queue into our in_queue, then adjust 1691 * counters since r->write_queue is reused for compressed output */ 1692 chunkqueue * const cq = &r->write_queue; 1693 len = chunkqueue_length(cq); 1694 chunkqueue_remove_finished_chunks(cq); 1695 chunkqueue_append_chunkqueue(&hctx->in_queue, cq); 1696 cq->bytes_in -= len; 1697 cq->bytes_out -= len; 1698 1699 max = chunkqueue_length(&hctx->in_queue); 1700 #if 0 1701 /* calculate max bytes to compress for this call */ 1702 if (p->conf.sync_flush && max > (len = p->conf.work_block_size << 10)) { 1703 max = len; 1704 } 1705 #endif 1706 1707 /* Compress chunks from in_queue into chunks for write_queue */ 1708 while (max) { 1709 chunk *c = hctx->in_queue.first; 1710 1711 switch(c->type) { 1712 case MEM_CHUNK: 1713 len = buffer_clen(c->mem) - c->offset; 1714 if (len > max) len = max; 1715 if (mod_deflate_compress(hctx, (unsigned char *)c->mem->ptr+c->offset, len) < 0) { 1716 log_error(r->conf.errh, __FILE__, __LINE__, "compress failed."); 1717 return HANDLER_ERROR; 1718 } 1719 break; 1720 case FILE_CHUNK: 1721 len = c->file.length - c->offset; 1722 if (len > max) len = max; 1723 if ((len = mod_deflate_file_chunk(r, hctx, c, len)) < 0) { 1724 log_error(r->conf.errh, __FILE__, __LINE__, 1725 "compress file chunk failed %s", c->mem->ptr); 1726 return HANDLER_ERROR; 1727 } 1728 break; 1729 default: 1730 log_error(r->conf.errh, __FILE__, __LINE__, "%d type not known", c->type); 1731 return HANDLER_ERROR; 1732 } 1733 1734 max -= len; 1735 chunkqueue_mark_written(&hctx->in_queue, len); 1736 } 1737 1738 /*(currently should always be true)*/ 1739 /*(current implementation requires response be complete)*/ 1740 close_stream = (r->resp_body_finished 1741 && chunkqueue_is_empty(&hctx->in_queue)); 1742 if (mod_deflate_stream_flush(hctx, close_stream) < 0) { 1743 log_error(r->conf.errh, __FILE__, __LINE__, "flush error"); 1744 return HANDLER_ERROR; 1745 } 1746 1747 return close_stream ? HANDLER_FINISHED : HANDLER_GO_ON; 1748 } 1749 1750 1751 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { 1752 /* get client side support encodings */ 1753 int accept_encoding = 0; 1754 #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \ 1755 && !defined(USE_ZSTD) 1756 UNUSED(value); 1757 UNUSED(label); 1758 #else 1759 for (; *value; ++value) { 1760 const char *v; 1761 while (*value == ' ' || *value == ',') ++value; 1762 v = value; 1763 while (*value!=' ' && *value!=',' && *value!=';' && *value!='\0') 1764 ++value; 1765 switch (value - v) { 1766 case 2: 1767 #ifdef USE_BROTLI 1768 if (0 == memcmp(v, "br", 2)) 1769 accept_encoding |= HTTP_ACCEPT_ENCODING_BR; 1770 #endif 1771 break; 1772 case 4: 1773 #ifdef USE_ZLIB 1774 if (0 == memcmp(v, "gzip", 4)) 1775 accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; 1776 #endif 1777 #ifdef USE_ZSTD 1778 #ifdef USE_ZLIB 1779 else 1780 #endif 1781 if (0 == memcmp(v, "zstd", 4)) 1782 accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD; 1783 #endif 1784 break; 1785 case 5: 1786 #ifdef USE_BZ2LIB 1787 if (0 == memcmp(v, "bzip2", 5)) 1788 accept_encoding |= HTTP_ACCEPT_ENCODING_BZIP2; 1789 #endif 1790 break; 1791 case 6: 1792 #ifdef USE_ZLIB 1793 if (0 == memcmp(v, "x-gzip", 6)) 1794 accept_encoding |= HTTP_ACCEPT_ENCODING_X_GZIP; 1795 #endif 1796 break; 1797 case 7: 1798 #ifdef USE_ZLIB 1799 if (0 == memcmp(v, "deflate", 7)) 1800 accept_encoding |= HTTP_ACCEPT_ENCODING_DEFLATE; 1801 #endif 1802 #ifdef USE_BZ2LIB 1803 if (0 == memcmp(v, "x-bzip2", 7)) 1804 accept_encoding |= HTTP_ACCEPT_ENCODING_X_BZIP2; 1805 #endif 1806 break; 1807 #if 0 1808 case 8: 1809 if (0 == memcmp(v, "identity", 8)) 1810 accept_encoding |= HTTP_ACCEPT_ENCODING_IDENTITY; 1811 else if (0 == memcmp(v, "compress", 8)) 1812 accept_encoding |= HTTP_ACCEPT_ENCODING_COMPRESS; 1813 break; 1814 #endif 1815 default: 1816 break; 1817 } 1818 if (*value == ';') { 1819 while (*value != ',' && *value != '\0') ++value; 1820 } 1821 if (*value == '\0') break; 1822 } 1823 #endif 1824 1825 /* select best matching encoding */ 1826 const uint16_t *x = p->conf.allowed_encodings; 1827 if (NULL == x) return 0; 1828 while (*x && !(*x & accept_encoding)) ++x; 1829 accept_encoding &= *x; 1830 #ifdef USE_ZSTD 1831 if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) { 1832 *label = "zstd"; 1833 return HTTP_ACCEPT_ENCODING_ZSTD; 1834 } else 1835 #endif 1836 #ifdef USE_BROTLI 1837 if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { 1838 *label = "br"; 1839 return HTTP_ACCEPT_ENCODING_BR; 1840 } else 1841 #endif 1842 #ifdef USE_BZ2LIB 1843 if (accept_encoding & HTTP_ACCEPT_ENCODING_BZIP2) { 1844 *label = "bzip2"; 1845 return HTTP_ACCEPT_ENCODING_BZIP2; 1846 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_BZIP2) { 1847 *label = "x-bzip2"; 1848 return HTTP_ACCEPT_ENCODING_BZIP2; 1849 } else 1850 #endif 1851 #ifdef USE_ZLIB 1852 if (accept_encoding & HTTP_ACCEPT_ENCODING_GZIP) { 1853 *label = "gzip"; 1854 return HTTP_ACCEPT_ENCODING_GZIP; 1855 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_X_GZIP) { 1856 *label = "x-gzip"; 1857 return HTTP_ACCEPT_ENCODING_GZIP; 1858 } else if (accept_encoding & HTTP_ACCEPT_ENCODING_DEFLATE) { 1859 *label = "deflate"; 1860 return HTTP_ACCEPT_ENCODING_DEFLATE; 1861 } else 1862 #endif 1863 if (0 == accept_encoding) { 1864 return 0; 1865 } else { 1866 return 0; 1867 } 1868 } 1869 1870 REQUEST_FUNC(mod_deflate_handle_response_start) { 1871 plugin_data *p = p_d; 1872 const buffer *vbro; 1873 buffer *vb; 1874 handler_ctx *hctx; 1875 const char *label; 1876 off_t len; 1877 uint32_t etaglen; 1878 int compression_type; 1879 handler_t rc; 1880 int had_vary = 0; 1881 1882 /*(current implementation requires response be complete)*/ 1883 if (!r->resp_body_finished) return HANDLER_GO_ON; 1884 if (r->http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON; 1885 if (light_btst(r->resp_htags, HTTP_HEADER_TRANSFER_ENCODING)) 1886 return HANDLER_GO_ON; 1887 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_ENCODING)) 1888 return HANDLER_GO_ON; 1889 1890 /* disable compression for some http status types. */ 1891 switch(r->http_status) { 1892 case 100: 1893 case 101: 1894 case 204: 1895 case 205: 1896 case 304: 1897 /* disable compression as we have no response entity */ 1898 return HANDLER_GO_ON; 1899 default: 1900 break; 1901 } 1902 1903 mod_deflate_patch_config(r, p); 1904 1905 /* check if deflate configured for any mimetypes */ 1906 if (NULL == p->conf.mimetypes) return HANDLER_GO_ON; 1907 1908 /* check if size of response is below min-compress-size or exceeds max*/ 1909 /* (r->resp_body_finished checked at top of routine) */ 1910 len = chunkqueue_length(&r->write_queue); 1911 if (len <= (off_t)p->conf.min_compress_size) return HANDLER_GO_ON; 1912 if (p->conf.max_compress_size /*(max_compress_size in KB)*/ 1913 && len > ((off_t)p->conf.max_compress_size << 10)) { 1914 return HANDLER_GO_ON; 1915 } 1916 1917 /* Check Accept-Encoding for supported encoding. */ 1918 vbro = http_header_request_get(r, HTTP_HEADER_ACCEPT_ENCODING, CONST_STR_LEN("Accept-Encoding")); 1919 if (NULL == vbro) return HANDLER_GO_ON; 1920 1921 /* find matching encodings */ 1922 compression_type = mod_deflate_choose_encoding(vbro->ptr, p, &label); 1923 if (!compression_type) return HANDLER_GO_ON; 1924 1925 /* Check mimetype in response header "Content-Type" */ 1926 if (NULL != (vbro = http_header_response_get(r, HTTP_HEADER_CONTENT_TYPE, CONST_STR_LEN("Content-Type")))) { 1927 if (NULL == array_match_value_prefix(p->conf.mimetypes, vbro)) return HANDLER_GO_ON; 1928 } else { 1929 /* If no Content-Type set, compress only if first p->conf.mimetypes value is "" */ 1930 data_string *mimetype = (data_string *)p->conf.mimetypes->data[0]; 1931 if (!buffer_is_blank(&mimetype->value)) return HANDLER_GO_ON; 1932 } 1933 1934 /* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */ 1935 if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) { 1936 had_vary = 1; 1937 if (!http_header_str_contains_token(BUF_PTR_LEN(vb), 1938 CONST_STR_LEN("Accept-Encoding"))) 1939 buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding")); 1940 } else { 1941 http_header_response_append(r, HTTP_HEADER_VARY, 1942 CONST_STR_LEN("Vary"), 1943 CONST_STR_LEN("Accept-Encoding")); 1944 } 1945 1946 /* check ETag as is done in http_response_handle_cachable() 1947 * (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */ 1948 vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag")); 1949 etaglen = vb ? buffer_clen(vb) : 0; 1950 if (etaglen && light_btst(r->rqst_htags, HTTP_HEADER_IF_NONE_MATCH)) { 1951 const buffer *if_none_match = http_header_request_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match")); 1952 if ( r->http_status < 300 /*(want 2xx only)*/ 1953 && NULL != if_none_match 1954 && 0 == strncmp(if_none_match->ptr, vb->ptr, etaglen-1) 1955 && if_none_match->ptr[etaglen-1] == '-' 1956 && 0 == strncmp(if_none_match->ptr+etaglen, label, strlen(label))) { 1957 1958 if (http_method_get_head_query(r->http_method)) { 1959 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1960 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1961 buffer_append_string(vb, label); 1962 buffer_append_char(vb, '"'); 1963 r->http_status = 304; 1964 } else { 1965 r->http_status = 412; 1966 } 1967 1968 /* response_start hook occurs after error docs have been handled. 1969 * For now, send back empty response body. 1970 * In the future, might extract the error doc code so that it 1971 * might be run again if response_start hooks return with 1972 * changed http_status and r->handler_module = NULL */ 1973 /* clear content length even if 304 since compressed length unknown */ 1974 http_response_body_clear(r, 0); 1975 1976 r->resp_body_finished = 1; 1977 r->handler_module = NULL; 1978 return HANDLER_GO_ON; 1979 } 1980 } 1981 1982 if (0.0 < p->conf.max_loadavg && p->conf.max_loadavg < r->con->srv->loadavg[0]) { 1983 return HANDLER_GO_ON; 1984 } 1985 1986 /* update ETag, if ETag response header is set */ 1987 if (etaglen) { 1988 /* modify ETag response header in-place to remove '"' and append '-label"' */ 1989 vb->ptr[etaglen-1] = '-'; /*(overwrite end '"')*/ 1990 buffer_append_string(vb, label); 1991 buffer_append_char(vb, '"'); 1992 } 1993 1994 /* set Content-Encoding to show selected compression type */ 1995 http_header_response_set(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding"), label, strlen(label)); 1996 1997 /* clear Content-Length and r->write_queue if HTTP HEAD request 1998 * (alternatively, could return original Content-Length with HEAD 1999 * request if ETag not modified and Content-Encoding not added) 2000 * (see top of this func where short-circuit is done on HTTP HEAD) */ 2001 if (HTTP_METHOD_HEAD == r->http_method) { 2002 /* ensure that uncompressed Content-Length is not sent in HEAD response */ 2003 http_response_body_clear(r, 0); 2004 r->resp_body_finished = 1; 2005 return HANDLER_GO_ON; 2006 } 2007 2008 /* restrict items eligible for cache of compressed responses 2009 * (This module does not aim to be a full caching proxy) 2010 * response must be complete (not streaming response) 2011 * must not have prior Vary response header (before Accept-Encoding added) 2012 * must have ETag 2013 * must be file 2014 * must be single FILE_CHUNK in chunkqueue 2015 * must not be chunkqueue temporary file 2016 * must be whole file, not partial content 2017 * must not be HTTP status 206 Partial Content 2018 * must not have Cache-Control 'private' or 'no-store' 2019 * Note: small files (< 32k (see http_chunk.c)) will have been read into 2020 * memory (if streaming HTTP/1.1 chunked response) and will end up 2021 * getting stream-compressed rather than cached on disk as compressed 2022 * file 2023 */ 2024 buffer *tb = NULL; 2025 if (p->conf.cache_dir 2026 && !had_vary 2027 && etaglen > 2 2028 && r->resp_body_finished 2029 && r->write_queue.first == r->write_queue.last 2030 && r->write_queue.first->type == FILE_CHUNK 2031 && r->write_queue.first->offset == 0 2032 && !r->write_queue.first->file.is_temp 2033 && r->http_status != 206 2034 && (!(vbro = http_header_response_get(r, HTTP_HEADER_CACHE_CONTROL, 2035 CONST_STR_LEN("Cache-Control"))) 2036 || (!http_header_str_contains_token(BUF_PTR_LEN(vbro), 2037 CONST_STR_LEN("private")) 2038 && !http_header_str_contains_token(BUF_PTR_LEN(vbro), 2039 CONST_STR_LEN("no-store")))) 2040 ) { 2041 tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb); 2042 /*(checked earlier and skipped if Transfer-Encoding had been set)*/ 2043 stat_cache_entry *sce = stat_cache_get_entry_open(tb, 1); 2044 if (NULL != sce) { 2045 chunkqueue_reset(&r->write_queue); 2046 if (sce->fd < 0 || 0 != http_chunk_append_file_ref(r, sce)) 2047 return HANDLER_ERROR; 2048 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) 2049 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, 2050 CONST_STR_LEN("Content-Length")); 2051 mod_deflate_note_ratio(r, sce->st.st_size, len); 2052 return HANDLER_GO_ON; 2053 } 2054 /* sanity check that response was whole file; 2055 * (racy since using stat_cache, but cache file only if match) */ 2056 sce = stat_cache_get_entry(r->write_queue.first->mem); 2057 if (NULL == sce || sce->st.st_size != len) 2058 tb = NULL; 2059 else if (0 != mkdir_for_file(tb->ptr)) 2060 tb = NULL; 2061 } 2062 2063 /* enable compression */ 2064 p->conf.sync_flush = 2065 ((r->conf.stream_response_body 2066 & (FDEVENT_STREAM_RESPONSE | FDEVENT_STREAM_RESPONSE_BUFMIN)) 2067 && 0 == p->conf.output_buffer_size); 2068 hctx = handler_ctx_init(); 2069 hctx->plugin_data = p; 2070 hctx->compression_type = compression_type; 2071 hctx->r = r; 2072 /* setup output buffer */ 2073 buffer_clear(&p->tmp_buf); 2074 hctx->output = &p->tmp_buf; 2075 /* open cache file if caching compressed file */ 2076 if (tb) mod_deflate_cache_file_open(hctx, tb); 2077 2078 #ifdef HAVE_LIBDEFLATE 2079 chunk * const c = r->write_queue.first; /*(invalid after compression)*/ 2080 #ifdef HAVE_MMAP 2081 #if defined(_LP64) || defined(__LP64__) || defined(_WIN64) 2082 /* optimization to compress single file in one-shot to writeable mmap */ 2083 /*(future: might extend to other compression types)*/ 2084 /*(chunkqueue_chunk_file_view() current min size for mmap is 128k)*/ 2085 if (len > 131072 /* XXX: TBD what should min size be for optimization?*/ 2086 && (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP 2087 || hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE) 2088 && c == r->write_queue.last 2089 && c->type == FILE_CHUNK 2090 && chunkqueue_chunk_file_view(c, len, r->conf.errh) 2091 && chunk_file_view_dlen(c->file.view, c->offset) >= len) { /*(cfv)*/ 2092 rc = HANDLER_GO_ON; 2093 hctx->bytes_in = len; 2094 if (mod_deflate_using_libdeflate(hctx, p)) { 2095 if (NULL == tb || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) 2096 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2097 else 2098 rc = HANDLER_ERROR; 2099 handler_ctx_free(hctx); 2100 return rc; 2101 } 2102 hctx->bytes_in = hctx->bytes_out = 0; 2103 } 2104 else 2105 #endif 2106 #endif 2107 if (len <= 65536 /*(p->tmp_buf is at least 64k)*/ 2108 && (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP 2109 || hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE) 2110 && c == r->write_queue.last 2111 && c->type == MEM_CHUNK) { 2112 /*(skip if FILE_CHUNK; not worth mmap/munmap overhead on small file)*/ 2113 rc = HANDLER_GO_ON; 2114 hctx->bytes_in = len; 2115 if (mod_deflate_using_libdeflate_sm(hctx, p)) { 2116 if (NULL == tb || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) 2117 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2118 else 2119 rc = HANDLER_ERROR; 2120 handler_ctx_free(hctx); 2121 return rc; 2122 } 2123 hctx->bytes_in = hctx->bytes_out = 0; 2124 } 2125 #endif /* HAVE_LIBDEFLATE */ 2126 2127 if (0 != mod_deflate_stream_init(hctx)) { 2128 /*(should not happen unless ENOMEM)*/ 2129 handler_ctx_free(hctx); 2130 log_error(r->conf.errh, __FILE__, __LINE__, 2131 "Failed to initialize compression %s", label); 2132 /* restore prior Etag and unset Content-Encoding */ 2133 if (etaglen) { 2134 vb->ptr[etaglen-1] = '"'; /*(overwrite '-')*/ 2135 buffer_truncate(vb, etaglen); 2136 } 2137 http_header_response_unset(r, HTTP_HEADER_CONTENT_ENCODING, CONST_STR_LEN("Content-Encoding")); 2138 return HANDLER_GO_ON; 2139 } 2140 2141 #ifdef USE_BROTLI 2142 if (r->resp_body_finished 2143 && (hctx->compression_type & HTTP_ACCEPT_ENCODING_BR) 2144 && (len >> 1) < (off_t)((~(uint32_t)0u) >> 1)) 2145 BrotliEncoderSetParameter(hctx->u.br, BROTLI_PARAM_SIZE_HINT, (uint32_t)len); 2146 #endif 2147 2148 if (light_btst(r->resp_htags, HTTP_HEADER_CONTENT_LENGTH)) { 2149 http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length")); 2150 } 2151 r->plugin_ctx[p->id] = hctx; 2152 2153 rc = deflate_compress_response(r, hctx); 2154 if (HANDLER_GO_ON == rc) return HANDLER_GO_ON; 2155 if (HANDLER_FINISHED == rc) { 2156 #ifdef __COVERITY__ 2157 /* coverity misses if hctx->cache_fd is not -1, then tb is not NULL */ 2158 force_assert(-1 == hctx->cache_fd || NULL != tb); 2159 #endif 2160 if (-1 == hctx->cache_fd 2161 || 0 == mod_deflate_cache_file_finish(r, hctx, tb)) { 2162 mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in); 2163 rc = HANDLER_GO_ON; 2164 } 2165 else 2166 rc = HANDLER_ERROR; 2167 } 2168 r->plugin_ctx[p->id] = NULL; 2169 if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR; 2170 return rc; 2171 } 2172 2173 static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) { 2174 plugin_data *p = p_d; 2175 handler_ctx *hctx = r->plugin_ctx[p->id]; 2176 2177 if (NULL != hctx) { 2178 r->plugin_ctx[p->id] = NULL; 2179 deflate_compress_cleanup(r, hctx); 2180 } 2181 2182 return HANDLER_GO_ON; 2183 } 2184 2185 2186 __attribute_cold__ 2187 int mod_deflate_plugin_init(plugin *p); 2188 int mod_deflate_plugin_init(plugin *p) { 2189 p->version = LIGHTTPD_VERSION_ID; 2190 p->name = "deflate"; 2191 2192 p->init = mod_deflate_init; 2193 p->cleanup = mod_deflate_free; 2194 p->set_defaults = mod_deflate_set_defaults; 2195 p->handle_request_reset = mod_deflate_cleanup; 2196 p->handle_response_start = mod_deflate_handle_response_start; 2197 2198 return 0; 2199 } 2200