1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_http.h>
11 #include <ngx_md5.h>
12
13
14 static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r,
15 ngx_http_cache_t *c);
16 static void ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev);
17 static void ngx_http_file_cache_lock_wait(ngx_http_request_t *r,
18 ngx_http_cache_t *c);
19 static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
20 ngx_http_cache_t *c);
21 static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
22 ngx_http_cache_t *c);
23 #if (NGX_HAVE_FILE_AIO)
24 static void ngx_http_cache_aio_event_handler(ngx_event_t *ev);
25 #endif
26 #if (NGX_THREADS)
27 static ngx_int_t ngx_http_cache_thread_handler(ngx_thread_task_t *task,
28 ngx_file_t *file);
29 static void ngx_http_cache_thread_event_handler(ngx_event_t *ev);
30 #endif
31 static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache,
32 ngx_http_cache_t *c);
33 static ngx_int_t ngx_http_file_cache_name(ngx_http_request_t *r,
34 ngx_path_t *path);
35 static ngx_http_file_cache_node_t *
36 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key);
37 static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp,
38 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
39 static void ngx_http_file_cache_vary(ngx_http_request_t *r, u_char *vary,
40 size_t len, u_char *hash);
41 static void ngx_http_file_cache_vary_header(ngx_http_request_t *r,
42 ngx_md5_t *md5, ngx_str_t *name);
43 static ngx_int_t ngx_http_file_cache_reopen(ngx_http_request_t *r,
44 ngx_http_cache_t *c);
45 static ngx_int_t ngx_http_file_cache_update_variant(ngx_http_request_t *r,
46 ngx_http_cache_t *c);
47 static void ngx_http_file_cache_cleanup(void *data);
48 static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache);
49 static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache);
50 static void ngx_http_file_cache_delete(ngx_http_file_cache_t *cache,
51 ngx_queue_t *q, u_char *name);
52 static void ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t *cache);
53 static ngx_int_t ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx,
54 ngx_str_t *path);
55 static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx,
56 ngx_str_t *path);
57 static ngx_int_t ngx_http_file_cache_manage_directory(ngx_tree_ctx_t *ctx,
58 ngx_str_t *path);
59 static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx,
60 ngx_str_t *path);
61 static ngx_int_t ngx_http_file_cache_add(ngx_http_file_cache_t *cache,
62 ngx_http_cache_t *c);
63 static ngx_int_t ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx,
64 ngx_str_t *path);
65 static void ngx_http_file_cache_set_watermark(ngx_http_file_cache_t *cache);
66
67
68 ngx_str_t ngx_http_cache_status[] = {
69 ngx_string("MISS"),
70 ngx_string("BYPASS"),
71 ngx_string("EXPIRED"),
72 ngx_string("STALE"),
73 ngx_string("UPDATING"),
74 ngx_string("REVALIDATED"),
75 ngx_string("HIT")
76 };
77
78
79 static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' };
80
81
82 static ngx_int_t
ngx_http_file_cache_init(ngx_shm_zone_t * shm_zone,void * data)83 ngx_http_file_cache_init(ngx_shm_zone_t *shm_zone, void *data)
84 {
85 ngx_http_file_cache_t *ocache = data;
86
87 size_t len;
88 ngx_uint_t n;
89 ngx_http_file_cache_t *cache;
90
91 cache = shm_zone->data;
92
93 if (ocache) {
94 if (ngx_strcmp(cache->path->name.data, ocache->path->name.data) != 0) {
95 ngx_log_error(NGX_LOG_EMERG, shm_zone->shm.log, 0,
96 "cache \"%V\" uses the \"%V\" cache path "
97 "while previously it used the \"%V\" cache path",
98 &shm_zone->shm.name, &cache->path->name,
99 &ocache->path->name);
100
101 return NGX_ERROR;
102 }
103
104 for (n = 0; n < NGX_MAX_PATH_LEVEL; n++) {
105 if (cache->path->level[n] != ocache->path->level[n]) {
106 ngx_log_error(NGX_LOG_EMERG, shm_zone->shm.log, 0,
107 "cache \"%V\" had previously different levels",
108 &shm_zone->shm.name);
109 return NGX_ERROR;
110 }
111 }
112
113 cache->sh = ocache->sh;
114
115 cache->shpool = ocache->shpool;
116 cache->bsize = ocache->bsize;
117
118 cache->max_size /= cache->bsize;
119
120 if (!cache->sh->cold || cache->sh->loading) {
121 cache->path->loader = NULL;
122 }
123
124 return NGX_OK;
125 }
126
127 cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr;
128
129 if (shm_zone->shm.exists) {
130 cache->sh = cache->shpool->data;
131 cache->bsize = ngx_fs_bsize(cache->path->name.data);
132 cache->max_size /= cache->bsize;
133
134 return NGX_OK;
135 }
136
137 cache->sh = ngx_slab_alloc(cache->shpool, sizeof(ngx_http_file_cache_sh_t));
138 if (cache->sh == NULL) {
139 return NGX_ERROR;
140 }
141
142 cache->shpool->data = cache->sh;
143
144 ngx_rbtree_init(&cache->sh->rbtree, &cache->sh->sentinel,
145 ngx_http_file_cache_rbtree_insert_value);
146
147 ngx_queue_init(&cache->sh->queue);
148
149 cache->sh->cold = 1;
150 cache->sh->loading = 0;
151 cache->sh->size = 0;
152 cache->sh->count = 0;
153 cache->sh->watermark = (ngx_uint_t) -1;
154
155 cache->bsize = ngx_fs_bsize(cache->path->name.data);
156
157 cache->max_size /= cache->bsize;
158
159 len = sizeof(" in cache keys zone \"\"") + shm_zone->shm.name.len;
160
161 cache->shpool->log_ctx = ngx_slab_alloc(cache->shpool, len);
162 if (cache->shpool->log_ctx == NULL) {
163 return NGX_ERROR;
164 }
165
166 ngx_sprintf(cache->shpool->log_ctx, " in cache keys zone \"%V\"%Z",
167 &shm_zone->shm.name);
168
169 cache->shpool->log_nomem = 0;
170
171 return NGX_OK;
172 }
173
174
175 ngx_int_t
ngx_http_file_cache_new(ngx_http_request_t * r)176 ngx_http_file_cache_new(ngx_http_request_t *r)
177 {
178 ngx_http_cache_t *c;
179
180 c = ngx_pcalloc(r->pool, sizeof(ngx_http_cache_t));
181 if (c == NULL) {
182 return NGX_ERROR;
183 }
184
185 if (ngx_array_init(&c->keys, r->pool, 4, sizeof(ngx_str_t)) != NGX_OK) {
186 return NGX_ERROR;
187 }
188
189 r->cache = c;
190 c->file.log = r->connection->log;
191 c->file.fd = NGX_INVALID_FILE;
192
193 return NGX_OK;
194 }
195
196
197 ngx_int_t
ngx_http_file_cache_create(ngx_http_request_t * r)198 ngx_http_file_cache_create(ngx_http_request_t *r)
199 {
200 ngx_http_cache_t *c;
201 ngx_pool_cleanup_t *cln;
202 ngx_http_file_cache_t *cache;
203
204 c = r->cache;
205 cache = c->file_cache;
206
207 cln = ngx_pool_cleanup_add(r->pool, 0);
208 if (cln == NULL) {
209 return NGX_ERROR;
210 }
211
212 cln->handler = ngx_http_file_cache_cleanup;
213 cln->data = c;
214
215 if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) {
216 return NGX_ERROR;
217 }
218
219 if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) {
220 return NGX_ERROR;
221 }
222
223 return NGX_OK;
224 }
225
226
227 void
ngx_http_file_cache_create_key(ngx_http_request_t * r)228 ngx_http_file_cache_create_key(ngx_http_request_t *r)
229 {
230 size_t len;
231 ngx_str_t *key;
232 ngx_uint_t i;
233 ngx_md5_t md5;
234 ngx_http_cache_t *c;
235
236 c = r->cache;
237
238 len = 0;
239
240 ngx_crc32_init(c->crc32);
241 ngx_md5_init(&md5);
242
243 key = c->keys.elts;
244 for (i = 0; i < c->keys.nelts; i++) {
245 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
246 "http cache key: \"%V\"", &key[i]);
247
248 len += key[i].len;
249
250 ngx_crc32_update(&c->crc32, key[i].data, key[i].len);
251 ngx_md5_update(&md5, key[i].data, key[i].len);
252 }
253
254 c->header_start = sizeof(ngx_http_file_cache_header_t)
255 + sizeof(ngx_http_file_cache_key) + len + 1;
256
257 ngx_crc32_final(c->crc32);
258 ngx_md5_final(c->key, &md5);
259
260 ngx_memcpy(c->main, c->key, NGX_HTTP_CACHE_KEY_LEN);
261 }
262
263
264 ngx_int_t
ngx_http_file_cache_open(ngx_http_request_t * r)265 ngx_http_file_cache_open(ngx_http_request_t *r)
266 {
267 ngx_int_t rc, rv;
268 ngx_uint_t test;
269 ngx_http_cache_t *c;
270 ngx_pool_cleanup_t *cln;
271 ngx_open_file_info_t of;
272 ngx_http_file_cache_t *cache;
273 ngx_http_core_loc_conf_t *clcf;
274
275 c = r->cache;
276
277 if (c->waiting) {
278 return NGX_AGAIN;
279 }
280
281 if (c->reading) {
282 return ngx_http_file_cache_read(r, c);
283 }
284
285 cache = c->file_cache;
286
287 if (c->node == NULL) {
288 cln = ngx_pool_cleanup_add(r->pool, 0);
289 if (cln == NULL) {
290 return NGX_ERROR;
291 }
292
293 cln->handler = ngx_http_file_cache_cleanup;
294 cln->data = c;
295 }
296
297 rc = ngx_http_file_cache_exists(cache, c);
298
299 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
300 "http file cache exists: %i e:%d", rc, c->exists);
301
302 if (rc == NGX_ERROR) {
303 return rc;
304 }
305
306 if (rc == NGX_AGAIN) {
307 return NGX_HTTP_CACHE_SCARCE;
308 }
309
310 if (rc == NGX_OK) {
311
312 if (c->error) {
313 return c->error;
314 }
315
316 c->temp_file = 1;
317 test = c->exists ? 1 : 0;
318 rv = NGX_DECLINED;
319
320 } else { /* rc == NGX_DECLINED */
321
322 test = cache->sh->cold ? 1 : 0;
323
324 if (c->min_uses > 1) {
325
326 if (!test) {
327 return NGX_HTTP_CACHE_SCARCE;
328 }
329
330 rv = NGX_HTTP_CACHE_SCARCE;
331
332 } else {
333 c->temp_file = 1;
334 rv = NGX_DECLINED;
335 }
336 }
337
338 if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) {
339 return NGX_ERROR;
340 }
341
342 if (!test) {
343 goto done;
344 }
345
346 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
347
348 ngx_memzero(&of, sizeof(ngx_open_file_info_t));
349
350 of.uniq = c->uniq;
351 of.valid = clcf->open_file_cache_valid;
352 of.min_uses = clcf->open_file_cache_min_uses;
353 of.events = clcf->open_file_cache_events;
354 of.directio = NGX_OPEN_FILE_DIRECTIO_OFF;
355 of.read_ahead = clcf->read_ahead;
356
357 if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool)
358 != NGX_OK)
359 {
360 switch (of.err) {
361
362 case 0:
363 return NGX_ERROR;
364
365 case NGX_ENOENT:
366 case NGX_ENOTDIR:
367 goto done;
368
369 default:
370 ngx_log_error(NGX_LOG_CRIT, r->connection->log, of.err,
371 ngx_open_file_n " \"%s\" failed", c->file.name.data);
372 return NGX_ERROR;
373 }
374 }
375
376 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
377 "http file cache fd: %d", of.fd);
378
379 c->file.fd = of.fd;
380 c->file.log = r->connection->log;
381 c->uniq = of.uniq;
382 c->length = of.size;
383 c->fs_size = (of.fs_size + cache->bsize - 1) / cache->bsize;
384
385 c->buf = ngx_create_temp_buf(r->pool, c->body_start);
386 if (c->buf == NULL) {
387 return NGX_ERROR;
388 }
389
390 return ngx_http_file_cache_read(r, c);
391
392 done:
393
394 if (rv == NGX_DECLINED) {
395 return ngx_http_file_cache_lock(r, c);
396 }
397
398 return rv;
399 }
400
401
402 static ngx_int_t
ngx_http_file_cache_lock(ngx_http_request_t * r,ngx_http_cache_t * c)403 ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c)
404 {
405 ngx_msec_t now, timer;
406 ngx_http_file_cache_t *cache;
407
408 if (!c->lock) {
409 return NGX_DECLINED;
410 }
411
412 now = ngx_current_msec;
413
414 cache = c->file_cache;
415
416 ngx_shmtx_lock(&cache->shpool->mutex);
417
418 timer = c->node->lock_time - now;
419
420 if (!c->node->updating || (ngx_msec_int_t) timer <= 0) {
421 c->node->updating = 1;
422 c->node->lock_time = now + c->lock_age;
423 c->updating = 1;
424 c->lock_time = c->node->lock_time;
425 }
426
427 ngx_shmtx_unlock(&cache->shpool->mutex);
428
429 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
430 "http file cache lock u:%d wt:%M",
431 c->updating, c->wait_time);
432
433 if (c->updating) {
434 return NGX_DECLINED;
435 }
436
437 if (c->lock_timeout == 0) {
438 return NGX_HTTP_CACHE_SCARCE;
439 }
440
441 c->waiting = 1;
442
443 if (c->wait_time == 0) {
444 c->wait_time = now + c->lock_timeout;
445
446 c->wait_event.handler = ngx_http_file_cache_lock_wait_handler;
447 c->wait_event.data = r;
448 c->wait_event.log = r->connection->log;
449 }
450
451 timer = c->wait_time - now;
452
453 ngx_add_timer(&c->wait_event, (timer > 500) ? 500 : timer);
454
455 r->main->blocked++;
456
457 return NGX_AGAIN;
458 }
459
460
461 static void
ngx_http_file_cache_lock_wait_handler(ngx_event_t * ev)462 ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev)
463 {
464 ngx_connection_t *c;
465 ngx_http_request_t *r;
466
467 r = ev->data;
468 c = r->connection;
469
470 ngx_http_set_log_request(c->log, r);
471
472 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
473 "http file cache wait: \"%V?%V\"", &r->uri, &r->args);
474
475 ngx_http_file_cache_lock_wait(r, r->cache);
476
477 ngx_http_run_posted_requests(c);
478 }
479
480
481 static void
ngx_http_file_cache_lock_wait(ngx_http_request_t * r,ngx_http_cache_t * c)482 ngx_http_file_cache_lock_wait(ngx_http_request_t *r, ngx_http_cache_t *c)
483 {
484 ngx_uint_t wait;
485 ngx_msec_t now, timer;
486 ngx_http_file_cache_t *cache;
487
488 now = ngx_current_msec;
489
490 timer = c->wait_time - now;
491
492 if ((ngx_msec_int_t) timer <= 0) {
493 ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
494 "cache lock timeout");
495 c->lock_timeout = 0;
496 goto wakeup;
497 }
498
499 cache = c->file_cache;
500 wait = 0;
501
502 ngx_shmtx_lock(&cache->shpool->mutex);
503
504 timer = c->node->lock_time - now;
505
506 if (c->node->updating && (ngx_msec_int_t) timer > 0) {
507 wait = 1;
508 }
509
510 ngx_shmtx_unlock(&cache->shpool->mutex);
511
512 if (wait) {
513 ngx_add_timer(&c->wait_event, (timer > 500) ? 500 : timer);
514 return;
515 }
516
517 wakeup:
518
519 c->waiting = 0;
520 r->main->blocked--;
521 r->write_event_handler(r);
522 }
523
524
525 static ngx_int_t
ngx_http_file_cache_read(ngx_http_request_t * r,ngx_http_cache_t * c)526 ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c)
527 {
528 u_char *p;
529 time_t now;
530 ssize_t n;
531 ngx_str_t *key;
532 ngx_int_t rc;
533 ngx_uint_t i;
534 ngx_http_file_cache_t *cache;
535 ngx_http_file_cache_header_t *h;
536
537 n = ngx_http_file_cache_aio_read(r, c);
538
539 if (n < 0) {
540 return n;
541 }
542
543 if ((size_t) n < c->header_start) {
544 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
545 "cache file \"%s\" is too small", c->file.name.data);
546 return NGX_DECLINED;
547 }
548
549 h = (ngx_http_file_cache_header_t *) c->buf->pos;
550
551 if (h->version != NGX_HTTP_CACHE_VERSION) {
552 ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
553 "cache file \"%s\" version mismatch", c->file.name.data);
554 return NGX_DECLINED;
555 }
556
557 if (h->crc32 != c->crc32 || (size_t) h->header_start != c->header_start) {
558 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
559 "cache file \"%s\" has md5 collision", c->file.name.data);
560 return NGX_DECLINED;
561 }
562
563 p = c->buf->pos + sizeof(ngx_http_file_cache_header_t)
564 + sizeof(ngx_http_file_cache_key);
565
566 key = c->keys.elts;
567 for (i = 0; i < c->keys.nelts; i++) {
568 if (ngx_memcmp(p, key[i].data, key[i].len) != 0) {
569 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
570 "cache file \"%s\" has md5 collision",
571 c->file.name.data);
572 return NGX_DECLINED;
573 }
574
575 p += key[i].len;
576 }
577
578 if ((size_t) h->body_start > c->body_start) {
579 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
580 "cache file \"%s\" has too long header",
581 c->file.name.data);
582 return NGX_DECLINED;
583 }
584
585 if (h->vary_len > NGX_HTTP_CACHE_VARY_LEN) {
586 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
587 "cache file \"%s\" has incorrect vary length",
588 c->file.name.data);
589 return NGX_DECLINED;
590 }
591
592 if (h->vary_len) {
593 ngx_http_file_cache_vary(r, h->vary, h->vary_len, c->variant);
594
595 if (ngx_memcmp(c->variant, h->variant, NGX_HTTP_CACHE_KEY_LEN) != 0) {
596 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
597 "http file cache vary mismatch");
598 return ngx_http_file_cache_reopen(r, c);
599 }
600 }
601
602 c->buf->last += n;
603
604 c->valid_sec = h->valid_sec;
605 c->updating_sec = h->updating_sec;
606 c->error_sec = h->error_sec;
607 c->last_modified = h->last_modified;
608 c->date = h->date;
609 c->valid_msec = h->valid_msec;
610 c->body_start = h->body_start;
611 c->etag.len = h->etag_len;
612 c->etag.data = h->etag;
613
614 r->cached = 1;
615
616 cache = c->file_cache;
617
618 if (cache->sh->cold) {
619
620 ngx_shmtx_lock(&cache->shpool->mutex);
621
622 if (!c->node->exists) {
623 c->node->uses = 1;
624 c->node->body_start = c->body_start;
625 c->node->exists = 1;
626 c->node->uniq = c->uniq;
627 c->node->fs_size = c->fs_size;
628
629 cache->sh->size += c->fs_size;
630 }
631
632 ngx_shmtx_unlock(&cache->shpool->mutex);
633 }
634
635 now = ngx_time();
636
637 if (c->valid_sec < now) {
638 c->stale_updating = c->valid_sec + c->updating_sec >= now;
639 c->stale_error = c->valid_sec + c->error_sec >= now;
640
641 ngx_shmtx_lock(&cache->shpool->mutex);
642
643 if (c->node->updating) {
644 rc = NGX_HTTP_CACHE_UPDATING;
645
646 } else {
647 c->node->updating = 1;
648 c->updating = 1;
649 c->lock_time = c->node->lock_time;
650 rc = NGX_HTTP_CACHE_STALE;
651 }
652
653 ngx_shmtx_unlock(&cache->shpool->mutex);
654
655 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
656 "http file cache expired: %i %T %T",
657 rc, c->valid_sec, now);
658
659 return rc;
660 }
661
662 return NGX_OK;
663 }
664
665
666 static ssize_t
ngx_http_file_cache_aio_read(ngx_http_request_t * r,ngx_http_cache_t * c)667 ngx_http_file_cache_aio_read(ngx_http_request_t *r, ngx_http_cache_t *c)
668 {
669 #if (NGX_HAVE_FILE_AIO || NGX_THREADS)
670 ssize_t n;
671 ngx_http_core_loc_conf_t *clcf;
672
673 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
674 #endif
675
676 #if (NGX_HAVE_FILE_AIO)
677
678 if (clcf->aio == NGX_HTTP_AIO_ON && ngx_file_aio) {
679 n = ngx_file_aio_read(&c->file, c->buf->pos, c->body_start, 0, r->pool);
680
681 if (n != NGX_AGAIN) {
682 c->reading = 0;
683 return n;
684 }
685
686 c->reading = 1;
687
688 c->file.aio->data = r;
689 c->file.aio->handler = ngx_http_cache_aio_event_handler;
690
691 r->main->blocked++;
692 r->aio = 1;
693
694 return NGX_AGAIN;
695 }
696
697 #endif
698
699 #if (NGX_THREADS)
700
701 if (clcf->aio == NGX_HTTP_AIO_THREADS) {
702 c->file.thread_task = c->thread_task;
703 c->file.thread_handler = ngx_http_cache_thread_handler;
704 c->file.thread_ctx = r;
705
706 n = ngx_thread_read(&c->file, c->buf->pos, c->body_start, 0, r->pool);
707
708 c->thread_task = c->file.thread_task;
709 c->reading = (n == NGX_AGAIN);
710
711 return n;
712 }
713
714 #endif
715
716 return ngx_read_file(&c->file, c->buf->pos, c->body_start, 0);
717 }
718
719
720 #if (NGX_HAVE_FILE_AIO)
721
722 static void
ngx_http_cache_aio_event_handler(ngx_event_t * ev)723 ngx_http_cache_aio_event_handler(ngx_event_t *ev)
724 {
725 ngx_event_aio_t *aio;
726 ngx_connection_t *c;
727 ngx_http_request_t *r;
728
729 aio = ev->data;
730 r = aio->data;
731 c = r->connection;
732
733 ngx_http_set_log_request(c->log, r);
734
735 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
736 "http file cache aio: \"%V?%V\"", &r->uri, &r->args);
737
738 r->main->blocked--;
739 r->aio = 0;
740
741 r->write_event_handler(r);
742
743 ngx_http_run_posted_requests(c);
744 }
745
746 #endif
747
748
749 #if (NGX_THREADS)
750
751 static ngx_int_t
ngx_http_cache_thread_handler(ngx_thread_task_t * task,ngx_file_t * file)752 ngx_http_cache_thread_handler(ngx_thread_task_t *task, ngx_file_t *file)
753 {
754 ngx_str_t name;
755 ngx_thread_pool_t *tp;
756 ngx_http_request_t *r;
757 ngx_http_core_loc_conf_t *clcf;
758
759 r = file->thread_ctx;
760
761 clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
762 tp = clcf->thread_pool;
763
764 if (tp == NULL) {
765 if (ngx_http_complex_value(r, clcf->thread_pool_value, &name)
766 != NGX_OK)
767 {
768 return NGX_ERROR;
769 }
770
771 tp = ngx_thread_pool_get((ngx_cycle_t *) ngx_cycle, &name);
772
773 if (tp == NULL) {
774 ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
775 "thread pool \"%V\" not found", &name);
776 return NGX_ERROR;
777 }
778 }
779
780 task->event.data = r;
781 task->event.handler = ngx_http_cache_thread_event_handler;
782
783 if (ngx_thread_task_post(tp, task) != NGX_OK) {
784 return NGX_ERROR;
785 }
786
787 r->main->blocked++;
788 r->aio = 1;
789
790 return NGX_OK;
791 }
792
793
794 static void
ngx_http_cache_thread_event_handler(ngx_event_t * ev)795 ngx_http_cache_thread_event_handler(ngx_event_t *ev)
796 {
797 ngx_connection_t *c;
798 ngx_http_request_t *r;
799
800 r = ev->data;
801 c = r->connection;
802
803 ngx_http_set_log_request(c->log, r);
804
805 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0,
806 "http file cache thread: \"%V?%V\"", &r->uri, &r->args);
807
808 r->main->blocked--;
809 r->aio = 0;
810
811 r->write_event_handler(r);
812
813 ngx_http_run_posted_requests(c);
814 }
815
816 #endif
817
818
819 static ngx_int_t
ngx_http_file_cache_exists(ngx_http_file_cache_t * cache,ngx_http_cache_t * c)820 ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
821 {
822 ngx_int_t rc;
823 ngx_http_file_cache_node_t *fcn;
824
825 ngx_shmtx_lock(&cache->shpool->mutex);
826
827 fcn = c->node;
828
829 if (fcn == NULL) {
830 fcn = ngx_http_file_cache_lookup(cache, c->key);
831 }
832
833 if (fcn) {
834 ngx_queue_remove(&fcn->queue);
835
836 if (c->node == NULL) {
837 fcn->uses++;
838 fcn->count++;
839 }
840
841 if (fcn->error) {
842
843 if (fcn->valid_sec < ngx_time()) {
844 goto renew;
845 }
846
847 rc = NGX_OK;
848
849 goto done;
850 }
851
852 if (fcn->exists || fcn->uses >= c->min_uses) {
853
854 c->exists = fcn->exists;
855 if (fcn->body_start) {
856 c->body_start = fcn->body_start;
857 }
858
859 rc = NGX_OK;
860
861 goto done;
862 }
863
864 rc = NGX_AGAIN;
865
866 goto done;
867 }
868
869 fcn = ngx_slab_calloc_locked(cache->shpool,
870 sizeof(ngx_http_file_cache_node_t));
871 if (fcn == NULL) {
872 ngx_http_file_cache_set_watermark(cache);
873
874 ngx_shmtx_unlock(&cache->shpool->mutex);
875
876 (void) ngx_http_file_cache_forced_expire(cache);
877
878 ngx_shmtx_lock(&cache->shpool->mutex);
879
880 fcn = ngx_slab_calloc_locked(cache->shpool,
881 sizeof(ngx_http_file_cache_node_t));
882 if (fcn == NULL) {
883 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
884 "could not allocate node%s", cache->shpool->log_ctx);
885 rc = NGX_ERROR;
886 goto failed;
887 }
888 }
889
890 cache->sh->count++;
891
892 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
893
894 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
895 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
896
897 ngx_rbtree_insert(&cache->sh->rbtree, &fcn->node);
898
899 fcn->uses = 1;
900 fcn->count = 1;
901
902 renew:
903
904 rc = NGX_DECLINED;
905
906 fcn->valid_msec = 0;
907 fcn->error = 0;
908 fcn->exists = 0;
909 fcn->valid_sec = 0;
910 fcn->uniq = 0;
911 fcn->body_start = 0;
912 fcn->fs_size = 0;
913
914 done:
915
916 fcn->expire = ngx_time() + cache->inactive;
917
918 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
919
920 c->uniq = fcn->uniq;
921 c->error = fcn->error;
922 c->node = fcn;
923
924 failed:
925
926 ngx_shmtx_unlock(&cache->shpool->mutex);
927
928 return rc;
929 }
930
931
932 static ngx_int_t
ngx_http_file_cache_name(ngx_http_request_t * r,ngx_path_t * path)933 ngx_http_file_cache_name(ngx_http_request_t *r, ngx_path_t *path)
934 {
935 u_char *p;
936 ngx_http_cache_t *c;
937
938 c = r->cache;
939
940 if (c->file.name.len) {
941 return NGX_OK;
942 }
943
944 c->file.name.len = path->name.len + 1 + path->len
945 + 2 * NGX_HTTP_CACHE_KEY_LEN;
946
947 c->file.name.data = ngx_pnalloc(r->pool, c->file.name.len + 1);
948 if (c->file.name.data == NULL) {
949 return NGX_ERROR;
950 }
951
952 ngx_memcpy(c->file.name.data, path->name.data, path->name.len);
953
954 p = c->file.name.data + path->name.len + 1 + path->len;
955 p = ngx_hex_dump(p, c->key, NGX_HTTP_CACHE_KEY_LEN);
956 *p = '\0';
957
958 ngx_create_hashed_filename(path, c->file.name.data, c->file.name.len);
959
960 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
961 "cache file: \"%s\"", c->file.name.data);
962
963 return NGX_OK;
964 }
965
966
967 static ngx_http_file_cache_node_t *
ngx_http_file_cache_lookup(ngx_http_file_cache_t * cache,u_char * key)968 ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key)
969 {
970 ngx_int_t rc;
971 ngx_rbtree_key_t node_key;
972 ngx_rbtree_node_t *node, *sentinel;
973 ngx_http_file_cache_node_t *fcn;
974
975 ngx_memcpy((u_char *) &node_key, key, sizeof(ngx_rbtree_key_t));
976
977 node = cache->sh->rbtree.root;
978 sentinel = cache->sh->rbtree.sentinel;
979
980 while (node != sentinel) {
981
982 if (node_key < node->key) {
983 node = node->left;
984 continue;
985 }
986
987 if (node_key > node->key) {
988 node = node->right;
989 continue;
990 }
991
992 /* node_key == node->key */
993
994 fcn = (ngx_http_file_cache_node_t *) node;
995
996 rc = ngx_memcmp(&key[sizeof(ngx_rbtree_key_t)], fcn->key,
997 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
998
999 if (rc == 0) {
1000 return fcn;
1001 }
1002
1003 node = (rc < 0) ? node->left : node->right;
1004 }
1005
1006 /* not found */
1007
1008 return NULL;
1009 }
1010
1011
1012 static void
ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t * temp,ngx_rbtree_node_t * node,ngx_rbtree_node_t * sentinel)1013 ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp,
1014 ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
1015 {
1016 ngx_rbtree_node_t **p;
1017 ngx_http_file_cache_node_t *cn, *cnt;
1018
1019 for ( ;; ) {
1020
1021 if (node->key < temp->key) {
1022
1023 p = &temp->left;
1024
1025 } else if (node->key > temp->key) {
1026
1027 p = &temp->right;
1028
1029 } else { /* node->key == temp->key */
1030
1031 cn = (ngx_http_file_cache_node_t *) node;
1032 cnt = (ngx_http_file_cache_node_t *) temp;
1033
1034 p = (ngx_memcmp(cn->key, cnt->key,
1035 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t))
1036 < 0)
1037 ? &temp->left : &temp->right;
1038 }
1039
1040 if (*p == sentinel) {
1041 break;
1042 }
1043
1044 temp = *p;
1045 }
1046
1047 *p = node;
1048 node->parent = temp;
1049 node->left = sentinel;
1050 node->right = sentinel;
1051 ngx_rbt_red(node);
1052 }
1053
1054
1055 static void
ngx_http_file_cache_vary(ngx_http_request_t * r,u_char * vary,size_t len,u_char * hash)1056 ngx_http_file_cache_vary(ngx_http_request_t *r, u_char *vary, size_t len,
1057 u_char *hash)
1058 {
1059 u_char *p, *last;
1060 ngx_str_t name;
1061 ngx_md5_t md5;
1062 u_char buf[NGX_HTTP_CACHE_VARY_LEN];
1063
1064 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1065 "http file cache vary: \"%*s\"", len, vary);
1066
1067 ngx_md5_init(&md5);
1068 ngx_md5_update(&md5, r->cache->main, NGX_HTTP_CACHE_KEY_LEN);
1069
1070 ngx_strlow(buf, vary, len);
1071
1072 p = buf;
1073 last = buf + len;
1074
1075 while (p < last) {
1076
1077 while (p < last && (*p == ' ' || *p == ',')) { p++; }
1078
1079 name.data = p;
1080
1081 while (p < last && *p != ',' && *p != ' ') { p++; }
1082
1083 name.len = p - name.data;
1084
1085 if (name.len == 0) {
1086 break;
1087 }
1088
1089 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1090 "http file cache vary: %V", &name);
1091
1092 ngx_md5_update(&md5, name.data, name.len);
1093 ngx_md5_update(&md5, (u_char *) ":", sizeof(":") - 1);
1094
1095 ngx_http_file_cache_vary_header(r, &md5, &name);
1096
1097 ngx_md5_update(&md5, (u_char *) CRLF, sizeof(CRLF) - 1);
1098 }
1099
1100 ngx_md5_final(hash, &md5);
1101 }
1102
1103
1104 static void
ngx_http_file_cache_vary_header(ngx_http_request_t * r,ngx_md5_t * md5,ngx_str_t * name)1105 ngx_http_file_cache_vary_header(ngx_http_request_t *r, ngx_md5_t *md5,
1106 ngx_str_t *name)
1107 {
1108 size_t len;
1109 u_char *p, *start, *last;
1110 ngx_uint_t i, multiple, normalize;
1111 ngx_list_part_t *part;
1112 ngx_table_elt_t *header;
1113
1114 multiple = 0;
1115 normalize = 0;
1116
1117 if (name->len == sizeof("Accept-Charset") - 1
1118 && ngx_strncasecmp(name->data, (u_char *) "Accept-Charset",
1119 sizeof("Accept-Charset") - 1) == 0)
1120 {
1121 normalize = 1;
1122
1123 } else if (name->len == sizeof("Accept-Encoding") - 1
1124 && ngx_strncasecmp(name->data, (u_char *) "Accept-Encoding",
1125 sizeof("Accept-Encoding") - 1) == 0)
1126 {
1127 normalize = 1;
1128
1129 } else if (name->len == sizeof("Accept-Language") - 1
1130 && ngx_strncasecmp(name->data, (u_char *) "Accept-Language",
1131 sizeof("Accept-Language") - 1) == 0)
1132 {
1133 normalize = 1;
1134 }
1135
1136 part = &r->headers_in.headers.part;
1137 header = part->elts;
1138
1139 for (i = 0; /* void */ ; i++) {
1140
1141 if (i >= part->nelts) {
1142 if (part->next == NULL) {
1143 break;
1144 }
1145
1146 part = part->next;
1147 header = part->elts;
1148 i = 0;
1149 }
1150
1151 if (header[i].hash == 0) {
1152 continue;
1153 }
1154
1155 if (header[i].key.len != name->len) {
1156 continue;
1157 }
1158
1159 if (ngx_strncasecmp(header[i].key.data, name->data, name->len) != 0) {
1160 continue;
1161 }
1162
1163 if (!normalize) {
1164
1165 if (multiple) {
1166 ngx_md5_update(md5, (u_char *) ",", sizeof(",") - 1);
1167 }
1168
1169 ngx_md5_update(md5, header[i].value.data, header[i].value.len);
1170
1171 multiple = 1;
1172
1173 continue;
1174 }
1175
1176 /* normalize spaces */
1177
1178 p = header[i].value.data;
1179 last = p + header[i].value.len;
1180
1181 while (p < last) {
1182
1183 while (p < last && (*p == ' ' || *p == ',')) { p++; }
1184
1185 start = p;
1186
1187 while (p < last && *p != ',' && *p != ' ') { p++; }
1188
1189 len = p - start;
1190
1191 if (len == 0) {
1192 break;
1193 }
1194
1195 if (multiple) {
1196 ngx_md5_update(md5, (u_char *) ",", sizeof(",") - 1);
1197 }
1198
1199 ngx_md5_update(md5, start, len);
1200
1201 multiple = 1;
1202 }
1203 }
1204 }
1205
1206
1207 static ngx_int_t
ngx_http_file_cache_reopen(ngx_http_request_t * r,ngx_http_cache_t * c)1208 ngx_http_file_cache_reopen(ngx_http_request_t *r, ngx_http_cache_t *c)
1209 {
1210 ngx_http_file_cache_t *cache;
1211
1212 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->file.log, 0,
1213 "http file cache reopen");
1214
1215 if (c->secondary) {
1216 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
1217 "cache file \"%s\" has incorrect vary hash",
1218 c->file.name.data);
1219 return NGX_DECLINED;
1220 }
1221
1222 cache = c->file_cache;
1223
1224 ngx_shmtx_lock(&cache->shpool->mutex);
1225
1226 c->node->count--;
1227 c->node = NULL;
1228
1229 ngx_shmtx_unlock(&cache->shpool->mutex);
1230
1231 c->secondary = 1;
1232 c->file.name.len = 0;
1233 c->body_start = c->buf->end - c->buf->start;
1234
1235 ngx_memcpy(c->key, c->variant, NGX_HTTP_CACHE_KEY_LEN);
1236
1237 return ngx_http_file_cache_open(r);
1238 }
1239
1240
1241 ngx_int_t
ngx_http_file_cache_set_header(ngx_http_request_t * r,u_char * buf)1242 ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf)
1243 {
1244 ngx_http_file_cache_header_t *h = (ngx_http_file_cache_header_t *) buf;
1245
1246 u_char *p;
1247 ngx_str_t *key;
1248 ngx_uint_t i;
1249 ngx_http_cache_t *c;
1250
1251 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1252 "http file cache set header");
1253
1254 c = r->cache;
1255
1256 ngx_memzero(h, sizeof(ngx_http_file_cache_header_t));
1257
1258 h->version = NGX_HTTP_CACHE_VERSION;
1259 h->valid_sec = c->valid_sec;
1260 h->updating_sec = c->updating_sec;
1261 h->error_sec = c->error_sec;
1262 h->last_modified = c->last_modified;
1263 h->date = c->date;
1264 h->crc32 = c->crc32;
1265 h->valid_msec = (u_short) c->valid_msec;
1266 h->header_start = (u_short) c->header_start;
1267 h->body_start = (u_short) c->body_start;
1268
1269 if (c->etag.len <= NGX_HTTP_CACHE_ETAG_LEN) {
1270 h->etag_len = (u_char) c->etag.len;
1271 ngx_memcpy(h->etag, c->etag.data, c->etag.len);
1272 }
1273
1274 if (c->vary.len) {
1275 if (c->vary.len > NGX_HTTP_CACHE_VARY_LEN) {
1276 /* should not happen */
1277 c->vary.len = NGX_HTTP_CACHE_VARY_LEN;
1278 }
1279
1280 h->vary_len = (u_char) c->vary.len;
1281 ngx_memcpy(h->vary, c->vary.data, c->vary.len);
1282
1283 ngx_http_file_cache_vary(r, c->vary.data, c->vary.len, c->variant);
1284 ngx_memcpy(h->variant, c->variant, NGX_HTTP_CACHE_KEY_LEN);
1285 }
1286
1287 if (ngx_http_file_cache_update_variant(r, c) != NGX_OK) {
1288 return NGX_ERROR;
1289 }
1290
1291 p = buf + sizeof(ngx_http_file_cache_header_t);
1292
1293 p = ngx_cpymem(p, ngx_http_file_cache_key, sizeof(ngx_http_file_cache_key));
1294
1295 key = c->keys.elts;
1296 for (i = 0; i < c->keys.nelts; i++) {
1297 p = ngx_copy(p, key[i].data, key[i].len);
1298 }
1299
1300 *p = LF;
1301
1302 return NGX_OK;
1303 }
1304
1305
1306 static ngx_int_t
ngx_http_file_cache_update_variant(ngx_http_request_t * r,ngx_http_cache_t * c)1307 ngx_http_file_cache_update_variant(ngx_http_request_t *r, ngx_http_cache_t *c)
1308 {
1309 ngx_http_file_cache_t *cache;
1310
1311 if (!c->secondary) {
1312 return NGX_OK;
1313 }
1314
1315 if (c->vary.len
1316 && ngx_memcmp(c->variant, c->key, NGX_HTTP_CACHE_KEY_LEN) == 0)
1317 {
1318 return NGX_OK;
1319 }
1320
1321 /*
1322 * if the variant hash doesn't match one we used as a secondary
1323 * cache key, switch back to the original key
1324 */
1325
1326 cache = c->file_cache;
1327
1328 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1329 "http file cache main key");
1330
1331 ngx_shmtx_lock(&cache->shpool->mutex);
1332
1333 c->node->count--;
1334 c->node->updating = 0;
1335 c->node = NULL;
1336
1337 ngx_shmtx_unlock(&cache->shpool->mutex);
1338
1339 c->file.name.len = 0;
1340
1341 ngx_memcpy(c->key, c->main, NGX_HTTP_CACHE_KEY_LEN);
1342
1343 if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) {
1344 return NGX_ERROR;
1345 }
1346
1347 if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) {
1348 return NGX_ERROR;
1349 }
1350
1351 return NGX_OK;
1352 }
1353
1354
1355 void
ngx_http_file_cache_update(ngx_http_request_t * r,ngx_temp_file_t * tf)1356 ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf)
1357 {
1358 off_t fs_size;
1359 ngx_int_t rc;
1360 ngx_file_uniq_t uniq;
1361 ngx_file_info_t fi;
1362 ngx_http_cache_t *c;
1363 ngx_ext_rename_file_t ext;
1364 ngx_http_file_cache_t *cache;
1365
1366 c = r->cache;
1367
1368 if (c->updated) {
1369 return;
1370 }
1371
1372 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1373 "http file cache update");
1374
1375 cache = c->file_cache;
1376
1377 c->updated = 1;
1378 c->updating = 0;
1379
1380 uniq = 0;
1381 fs_size = 0;
1382
1383 ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1384 "http file cache rename: \"%s\" to \"%s\"",
1385 tf->file.name.data, c->file.name.data);
1386
1387 ext.access = NGX_FILE_OWNER_ACCESS;
1388 ext.path_access = NGX_FILE_OWNER_ACCESS;
1389 ext.time = -1;
1390 ext.create_path = 1;
1391 ext.delete_file = 1;
1392 ext.log = r->connection->log;
1393
1394 rc = ngx_ext_rename_file(&tf->file.name, &c->file.name, &ext);
1395
1396 if (rc == NGX_OK) {
1397
1398 if (ngx_fd_info(tf->file.fd, &fi) == NGX_FILE_ERROR) {
1399 ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
1400 ngx_fd_info_n " \"%s\" failed", tf->file.name.data);
1401
1402 rc = NGX_ERROR;
1403
1404 } else {
1405 uniq = ngx_file_uniq(&fi);
1406 fs_size = (ngx_file_fs_size(&fi) + cache->bsize - 1) / cache->bsize;
1407 }
1408 }
1409
1410 ngx_shmtx_lock(&cache->shpool->mutex);
1411
1412 c->node->count--;
1413 c->node->error = 0;
1414 c->node->uniq = uniq;
1415 c->node->body_start = c->body_start;
1416
1417 cache->sh->size += fs_size - c->node->fs_size;
1418 c->node->fs_size = fs_size;
1419
1420 if (rc == NGX_OK) {
1421 c->node->exists = 1;
1422 }
1423
1424 c->node->updating = 0;
1425
1426 ngx_shmtx_unlock(&cache->shpool->mutex);
1427 }
1428
1429
1430 void
ngx_http_file_cache_update_header(ngx_http_request_t * r)1431 ngx_http_file_cache_update_header(ngx_http_request_t *r)
1432 {
1433 ssize_t n;
1434 ngx_err_t err;
1435 ngx_file_t file;
1436 ngx_file_info_t fi;
1437 ngx_http_cache_t *c;
1438 ngx_http_file_cache_header_t h;
1439
1440 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1441 "http file cache update header");
1442
1443 c = r->cache;
1444
1445 ngx_memzero(&file, sizeof(ngx_file_t));
1446
1447 file.name = c->file.name;
1448 file.log = r->connection->log;
1449 file.fd = ngx_open_file(file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
1450
1451 if (file.fd == NGX_INVALID_FILE) {
1452 err = ngx_errno;
1453
1454 /* cache file may have been deleted */
1455
1456 if (err == NGX_ENOENT) {
1457 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1458 "http file cache \"%s\" not found",
1459 file.name.data);
1460 return;
1461 }
1462
1463 ngx_log_error(NGX_LOG_CRIT, r->connection->log, err,
1464 ngx_open_file_n " \"%s\" failed", file.name.data);
1465 return;
1466 }
1467
1468 /*
1469 * make sure cache file wasn't replaced;
1470 * if it was, do nothing
1471 */
1472
1473 if (ngx_fd_info(file.fd, &fi) == NGX_FILE_ERROR) {
1474 ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
1475 ngx_fd_info_n " \"%s\" failed", file.name.data);
1476 goto done;
1477 }
1478
1479 if (c->uniq != ngx_file_uniq(&fi)
1480 || c->length != ngx_file_size(&fi))
1481 {
1482 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1483 "http file cache \"%s\" changed",
1484 file.name.data);
1485 goto done;
1486 }
1487
1488 n = ngx_read_file(&file, (u_char *) &h,
1489 sizeof(ngx_http_file_cache_header_t), 0);
1490
1491 if (n == NGX_ERROR) {
1492 goto done;
1493 }
1494
1495 if ((size_t) n != sizeof(ngx_http_file_cache_header_t)) {
1496 ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
1497 ngx_read_file_n " read only %z of %z from \"%s\"",
1498 n, sizeof(ngx_http_file_cache_header_t), file.name.data);
1499 goto done;
1500 }
1501
1502 if (h.version != NGX_HTTP_CACHE_VERSION
1503 || h.last_modified != c->last_modified
1504 || h.crc32 != c->crc32
1505 || (size_t) h.header_start != c->header_start
1506 || (size_t) h.body_start != c->body_start)
1507 {
1508 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1509 "http file cache \"%s\" content changed",
1510 file.name.data);
1511 goto done;
1512 }
1513
1514 /*
1515 * update cache file header with new data,
1516 * notably h.valid_sec and h.date
1517 */
1518
1519 ngx_memzero(&h, sizeof(ngx_http_file_cache_header_t));
1520
1521 h.version = NGX_HTTP_CACHE_VERSION;
1522 h.valid_sec = c->valid_sec;
1523 h.updating_sec = c->updating_sec;
1524 h.error_sec = c->error_sec;
1525 h.last_modified = c->last_modified;
1526 h.date = c->date;
1527 h.crc32 = c->crc32;
1528 h.valid_msec = (u_short) c->valid_msec;
1529 h.header_start = (u_short) c->header_start;
1530 h.body_start = (u_short) c->body_start;
1531
1532 if (c->etag.len <= NGX_HTTP_CACHE_ETAG_LEN) {
1533 h.etag_len = (u_char) c->etag.len;
1534 ngx_memcpy(h.etag, c->etag.data, c->etag.len);
1535 }
1536
1537 if (c->vary.len) {
1538 if (c->vary.len > NGX_HTTP_CACHE_VARY_LEN) {
1539 /* should not happen */
1540 c->vary.len = NGX_HTTP_CACHE_VARY_LEN;
1541 }
1542
1543 h.vary_len = (u_char) c->vary.len;
1544 ngx_memcpy(h.vary, c->vary.data, c->vary.len);
1545
1546 ngx_http_file_cache_vary(r, c->vary.data, c->vary.len, c->variant);
1547 ngx_memcpy(h.variant, c->variant, NGX_HTTP_CACHE_KEY_LEN);
1548 }
1549
1550 (void) ngx_write_file(&file, (u_char *) &h,
1551 sizeof(ngx_http_file_cache_header_t), 0);
1552
1553 done:
1554
1555 if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
1556 ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
1557 ngx_close_file_n " \"%s\" failed", file.name.data);
1558 }
1559 }
1560
1561
1562 ngx_int_t
ngx_http_cache_send(ngx_http_request_t * r)1563 ngx_http_cache_send(ngx_http_request_t *r)
1564 {
1565 ngx_int_t rc;
1566 ngx_buf_t *b;
1567 ngx_chain_t out;
1568 ngx_http_cache_t *c;
1569
1570 c = r->cache;
1571
1572 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
1573 "http file cache send: %s", c->file.name.data);
1574
1575 if (r != r->main && c->length - c->body_start == 0) {
1576 return ngx_http_send_header(r);
1577 }
1578
1579 /* we need to allocate all before the header would be sent */
1580
1581 b = ngx_calloc_buf(r->pool);
1582 if (b == NULL) {
1583 return NGX_HTTP_INTERNAL_SERVER_ERROR;
1584 }
1585
1586 b->file = ngx_pcalloc(r->pool, sizeof(ngx_file_t));
1587 if (b->file == NULL) {
1588 return NGX_HTTP_INTERNAL_SERVER_ERROR;
1589 }
1590
1591 rc = ngx_http_send_header(r);
1592
1593 if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
1594 return rc;
1595 }
1596
1597 b->file_pos = c->body_start;
1598 b->file_last = c->length;
1599
1600 b->in_file = (c->length - c->body_start) ? 1: 0;
1601 b->last_buf = (r == r->main) ? 1: 0;
1602 b->last_in_chain = 1;
1603
1604 b->file->fd = c->file.fd;
1605 b->file->name = c->file.name;
1606 b->file->log = r->connection->log;
1607
1608 out.buf = b;
1609 out.next = NULL;
1610
1611 return ngx_http_output_filter(r, &out);
1612 }
1613
1614
1615 void
ngx_http_file_cache_free(ngx_http_cache_t * c,ngx_temp_file_t * tf)1616 ngx_http_file_cache_free(ngx_http_cache_t *c, ngx_temp_file_t *tf)
1617 {
1618 ngx_http_file_cache_t *cache;
1619 ngx_http_file_cache_node_t *fcn;
1620
1621 if (c->updated || c->node == NULL) {
1622 return;
1623 }
1624
1625 cache = c->file_cache;
1626
1627 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->file.log, 0,
1628 "http file cache free, fd: %d", c->file.fd);
1629
1630 ngx_shmtx_lock(&cache->shpool->mutex);
1631
1632 fcn = c->node;
1633 fcn->count--;
1634
1635 if (c->updating && fcn->lock_time == c->lock_time) {
1636 fcn->updating = 0;
1637 }
1638
1639 if (c->error) {
1640 fcn->error = c->error;
1641
1642 if (c->valid_sec) {
1643 fcn->valid_sec = c->valid_sec;
1644 fcn->valid_msec = c->valid_msec;
1645 }
1646
1647 } else if (!fcn->exists && fcn->count == 0 && c->min_uses == 1) {
1648 ngx_queue_remove(&fcn->queue);
1649 ngx_rbtree_delete(&cache->sh->rbtree, &fcn->node);
1650 ngx_slab_free_locked(cache->shpool, fcn);
1651 cache->sh->count--;
1652 c->node = NULL;
1653 }
1654
1655 ngx_shmtx_unlock(&cache->shpool->mutex);
1656
1657 c->updated = 1;
1658 c->updating = 0;
1659
1660 if (c->temp_file) {
1661 if (tf && tf->file.fd != NGX_INVALID_FILE) {
1662 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->file.log, 0,
1663 "http file cache incomplete: \"%s\"",
1664 tf->file.name.data);
1665
1666 if (ngx_delete_file(tf->file.name.data) == NGX_FILE_ERROR) {
1667 ngx_log_error(NGX_LOG_CRIT, c->file.log, ngx_errno,
1668 ngx_delete_file_n " \"%s\" failed",
1669 tf->file.name.data);
1670 }
1671 }
1672 }
1673
1674 if (c->wait_event.timer_set) {
1675 ngx_del_timer(&c->wait_event);
1676 }
1677 }
1678
1679
1680 static void
ngx_http_file_cache_cleanup(void * data)1681 ngx_http_file_cache_cleanup(void *data)
1682 {
1683 ngx_http_cache_t *c = data;
1684
1685 if (c->updated) {
1686 return;
1687 }
1688
1689 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->file.log, 0,
1690 "http file cache cleanup");
1691
1692 if (c->updating && !c->background) {
1693 ngx_log_error(NGX_LOG_ALERT, c->file.log, 0,
1694 "stalled cache updating, error:%ui", c->error);
1695 }
1696
1697 ngx_http_file_cache_free(c, NULL);
1698 }
1699
1700
1701 static time_t
ngx_http_file_cache_forced_expire(ngx_http_file_cache_t * cache)1702 ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache)
1703 {
1704 u_char *name, *p;
1705 size_t len;
1706 time_t wait;
1707 ngx_uint_t tries;
1708 ngx_path_t *path;
1709 ngx_queue_t *q, *sentinel;
1710 ngx_http_file_cache_node_t *fcn;
1711 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN];
1712
1713 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1714 "http file cache forced expire");
1715
1716 path = cache->path;
1717 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
1718
1719 name = ngx_alloc(len + 1, ngx_cycle->log);
1720 if (name == NULL) {
1721 return 10;
1722 }
1723
1724 ngx_memcpy(name, path->name.data, path->name.len);
1725
1726 wait = 10;
1727 tries = 20;
1728 sentinel = NULL;
1729
1730 ngx_shmtx_lock(&cache->shpool->mutex);
1731
1732 for ( ;; ) {
1733 if (ngx_queue_empty(&cache->sh->queue)) {
1734 break;
1735 }
1736
1737 q = ngx_queue_last(&cache->sh->queue);
1738
1739 if (q == sentinel) {
1740 break;
1741 }
1742
1743 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
1744
1745 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1746 "http file cache forced expire: #%d %d %02xd%02xd%02xd%02xd",
1747 fcn->count, fcn->exists,
1748 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
1749
1750 if (fcn->count == 0) {
1751 ngx_http_file_cache_delete(cache, q, name);
1752 wait = 0;
1753 break;
1754 }
1755
1756 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
1757 sizeof(ngx_rbtree_key_t));
1758 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
1759 (void) ngx_hex_dump(p, fcn->key, len);
1760
1761 /*
1762 * abnormally exited workers may leave locked cache entries,
1763 * and although it may be safe to remove them completely,
1764 * we prefer to just move them to the top of the inactive queue
1765 */
1766
1767 ngx_queue_remove(q);
1768 fcn->expire = ngx_time() + cache->inactive;
1769 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
1770
1771 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
1772 "ignore long locked inactive cache entry %*s, count:%d",
1773 (size_t) 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
1774
1775 if (sentinel == NULL) {
1776 sentinel = q;
1777 }
1778
1779 if (--tries) {
1780 continue;
1781 }
1782
1783 wait = 1;
1784 break;
1785 }
1786
1787 ngx_shmtx_unlock(&cache->shpool->mutex);
1788
1789 ngx_free(name);
1790
1791 return wait;
1792 }
1793
1794
1795 static time_t
ngx_http_file_cache_expire(ngx_http_file_cache_t * cache)1796 ngx_http_file_cache_expire(ngx_http_file_cache_t *cache)
1797 {
1798 u_char *name, *p;
1799 size_t len;
1800 time_t now, wait;
1801 ngx_path_t *path;
1802 ngx_msec_t elapsed;
1803 ngx_queue_t *q;
1804 ngx_http_file_cache_node_t *fcn;
1805 u_char key[2 * NGX_HTTP_CACHE_KEY_LEN];
1806
1807 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1808 "http file cache expire");
1809
1810 path = cache->path;
1811 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
1812
1813 name = ngx_alloc(len + 1, ngx_cycle->log);
1814 if (name == NULL) {
1815 return 10;
1816 }
1817
1818 ngx_memcpy(name, path->name.data, path->name.len);
1819
1820 now = ngx_time();
1821
1822 ngx_shmtx_lock(&cache->shpool->mutex);
1823
1824 for ( ;; ) {
1825
1826 if (ngx_quit || ngx_terminate) {
1827 wait = 1;
1828 break;
1829 }
1830
1831 if (ngx_queue_empty(&cache->sh->queue)) {
1832 wait = 10;
1833 break;
1834 }
1835
1836 q = ngx_queue_last(&cache->sh->queue);
1837
1838 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
1839
1840 wait = fcn->expire - now;
1841
1842 if (wait > 0) {
1843 wait = wait > 10 ? 10 : wait;
1844 break;
1845 }
1846
1847 ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1848 "http file cache expire: #%d %d %02xd%02xd%02xd%02xd",
1849 fcn->count, fcn->exists,
1850 fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]);
1851
1852 if (fcn->count == 0) {
1853 ngx_http_file_cache_delete(cache, q, name);
1854 goto next;
1855 }
1856
1857 if (fcn->deleting) {
1858 wait = 1;
1859 break;
1860 }
1861
1862 p = ngx_hex_dump(key, (u_char *) &fcn->node.key,
1863 sizeof(ngx_rbtree_key_t));
1864 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
1865 (void) ngx_hex_dump(p, fcn->key, len);
1866
1867 /*
1868 * abnormally exited workers may leave locked cache entries,
1869 * and although it may be safe to remove them completely,
1870 * we prefer to just move them to the top of the inactive queue
1871 */
1872
1873 ngx_queue_remove(q);
1874 fcn->expire = ngx_time() + cache->inactive;
1875 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
1876
1877 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
1878 "ignore long locked inactive cache entry %*s, count:%d",
1879 (size_t) 2 * NGX_HTTP_CACHE_KEY_LEN, key, fcn->count);
1880
1881 next:
1882
1883 if (++cache->files >= cache->manager_files) {
1884 wait = 0;
1885 break;
1886 }
1887
1888 ngx_time_update();
1889
1890 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
1891
1892 if (elapsed >= cache->manager_threshold) {
1893 wait = 0;
1894 break;
1895 }
1896 }
1897
1898 ngx_shmtx_unlock(&cache->shpool->mutex);
1899
1900 ngx_free(name);
1901
1902 return wait;
1903 }
1904
1905
1906 static void
ngx_http_file_cache_delete(ngx_http_file_cache_t * cache,ngx_queue_t * q,u_char * name)1907 ngx_http_file_cache_delete(ngx_http_file_cache_t *cache, ngx_queue_t *q,
1908 u_char *name)
1909 {
1910 u_char *p;
1911 size_t len;
1912 ngx_path_t *path;
1913 ngx_http_file_cache_node_t *fcn;
1914
1915 fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue);
1916
1917 if (fcn->exists) {
1918 cache->sh->size -= fcn->fs_size;
1919
1920 path = cache->path;
1921 p = name + path->name.len + 1 + path->len;
1922 p = ngx_hex_dump(p, (u_char *) &fcn->node.key,
1923 sizeof(ngx_rbtree_key_t));
1924 len = NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t);
1925 p = ngx_hex_dump(p, fcn->key, len);
1926 *p = '\0';
1927
1928 fcn->count++;
1929 fcn->deleting = 1;
1930 ngx_shmtx_unlock(&cache->shpool->mutex);
1931
1932 len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN;
1933 ngx_create_hashed_filename(path, name, len);
1934
1935 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1936 "http file cache expire: \"%s\"", name);
1937
1938 if (ngx_delete_file(name) == NGX_FILE_ERROR) {
1939 ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno,
1940 ngx_delete_file_n " \"%s\" failed", name);
1941 }
1942
1943 ngx_shmtx_lock(&cache->shpool->mutex);
1944 fcn->count--;
1945 fcn->deleting = 0;
1946 }
1947
1948 if (fcn->count == 0) {
1949 ngx_queue_remove(q);
1950 ngx_rbtree_delete(&cache->sh->rbtree, &fcn->node);
1951 ngx_slab_free_locked(cache->shpool, fcn);
1952 cache->sh->count--;
1953 }
1954 }
1955
1956
1957 static ngx_msec_t
ngx_http_file_cache_manager(void * data)1958 ngx_http_file_cache_manager(void *data)
1959 {
1960 ngx_http_file_cache_t *cache = data;
1961
1962 off_t size;
1963 time_t wait;
1964 ngx_msec_t elapsed, next;
1965 ngx_uint_t count, watermark;
1966
1967 cache->last = ngx_current_msec;
1968 cache->files = 0;
1969
1970 next = (ngx_msec_t) ngx_http_file_cache_expire(cache) * 1000;
1971
1972 if (next == 0) {
1973 next = cache->manager_sleep;
1974 goto done;
1975 }
1976
1977 for ( ;; ) {
1978 ngx_shmtx_lock(&cache->shpool->mutex);
1979
1980 size = cache->sh->size;
1981 count = cache->sh->count;
1982 watermark = cache->sh->watermark;
1983
1984 ngx_shmtx_unlock(&cache->shpool->mutex);
1985
1986 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
1987 "http file cache size: %O c:%ui w:%i",
1988 size, count, (ngx_int_t) watermark);
1989
1990 if (size < cache->max_size && count < watermark) {
1991 break;
1992 }
1993
1994 wait = ngx_http_file_cache_forced_expire(cache);
1995
1996 if (wait > 0) {
1997 next = (ngx_msec_t) wait * 1000;
1998 break;
1999 }
2000
2001 if (ngx_quit || ngx_terminate) {
2002 break;
2003 }
2004
2005 if (++cache->files >= cache->manager_files) {
2006 next = cache->manager_sleep;
2007 break;
2008 }
2009
2010 ngx_time_update();
2011
2012 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
2013
2014 if (elapsed >= cache->manager_threshold) {
2015 next = cache->manager_sleep;
2016 break;
2017 }
2018 }
2019
2020 done:
2021
2022 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
2023
2024 ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
2025 "http file cache manager: %ui e:%M n:%M",
2026 cache->files, elapsed, next);
2027
2028 return next;
2029 }
2030
2031
2032 static void
ngx_http_file_cache_loader(void * data)2033 ngx_http_file_cache_loader(void *data)
2034 {
2035 ngx_http_file_cache_t *cache = data;
2036
2037 ngx_tree_ctx_t tree;
2038
2039 if (!cache->sh->cold || cache->sh->loading) {
2040 return;
2041 }
2042
2043 if (!ngx_atomic_cmp_set(&cache->sh->loading, 0, ngx_pid)) {
2044 return;
2045 }
2046
2047 ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
2048 "http file cache loader");
2049
2050 tree.init_handler = NULL;
2051 tree.file_handler = ngx_http_file_cache_manage_file;
2052 tree.pre_tree_handler = ngx_http_file_cache_manage_directory;
2053 tree.post_tree_handler = ngx_http_file_cache_noop;
2054 tree.spec_handler = ngx_http_file_cache_delete_file;
2055 tree.data = cache;
2056 tree.alloc = 0;
2057 tree.log = ngx_cycle->log;
2058
2059 cache->last = ngx_current_msec;
2060 cache->files = 0;
2061
2062 if (ngx_walk_tree(&tree, &cache->path->name) == NGX_ABORT) {
2063 cache->sh->loading = 0;
2064 return;
2065 }
2066
2067 cache->sh->cold = 0;
2068 cache->sh->loading = 0;
2069
2070 ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0,
2071 "http file cache: %V %.3fM, bsize: %uz",
2072 &cache->path->name,
2073 ((double) cache->sh->size * cache->bsize) / (1024 * 1024),
2074 cache->bsize);
2075 }
2076
2077
2078 static ngx_int_t
ngx_http_file_cache_noop(ngx_tree_ctx_t * ctx,ngx_str_t * path)2079 ngx_http_file_cache_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path)
2080 {
2081 return NGX_OK;
2082 }
2083
2084
2085 static ngx_int_t
ngx_http_file_cache_manage_file(ngx_tree_ctx_t * ctx,ngx_str_t * path)2086 ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
2087 {
2088 ngx_msec_t elapsed;
2089 ngx_http_file_cache_t *cache;
2090
2091 cache = ctx->data;
2092
2093 if (ngx_http_file_cache_add_file(ctx, path) != NGX_OK) {
2094 (void) ngx_http_file_cache_delete_file(ctx, path);
2095 }
2096
2097 if (++cache->files >= cache->loader_files) {
2098 ngx_http_file_cache_loader_sleep(cache);
2099
2100 } else {
2101 ngx_time_update();
2102
2103 elapsed = ngx_abs((ngx_msec_int_t) (ngx_current_msec - cache->last));
2104
2105 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
2106 "http file cache loader time elapsed: %M", elapsed);
2107
2108 if (elapsed >= cache->loader_threshold) {
2109 ngx_http_file_cache_loader_sleep(cache);
2110 }
2111 }
2112
2113 return (ngx_quit || ngx_terminate) ? NGX_ABORT : NGX_OK;
2114 }
2115
2116
2117 static ngx_int_t
ngx_http_file_cache_manage_directory(ngx_tree_ctx_t * ctx,ngx_str_t * path)2118 ngx_http_file_cache_manage_directory(ngx_tree_ctx_t *ctx, ngx_str_t *path)
2119 {
2120 if (path->len >= 5
2121 && ngx_strncmp(path->data + path->len - 5, "/temp", 5) == 0)
2122 {
2123 return NGX_DECLINED;
2124 }
2125
2126 return NGX_OK;
2127 }
2128
2129
2130 static void
ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t * cache)2131 ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t *cache)
2132 {
2133 ngx_msleep(cache->loader_sleep);
2134
2135 ngx_time_update();
2136
2137 cache->last = ngx_current_msec;
2138 cache->files = 0;
2139 }
2140
2141
2142 static ngx_int_t
ngx_http_file_cache_add_file(ngx_tree_ctx_t * ctx,ngx_str_t * name)2143 ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *name)
2144 {
2145 u_char *p;
2146 ngx_int_t n;
2147 ngx_uint_t i;
2148 ngx_http_cache_t c;
2149 ngx_http_file_cache_t *cache;
2150
2151 if (name->len < 2 * NGX_HTTP_CACHE_KEY_LEN) {
2152 return NGX_ERROR;
2153 }
2154
2155 /*
2156 * Temporary files in cache have a suffix consisting of a dot
2157 * followed by 10 digits.
2158 */
2159
2160 if (name->len >= 2 * NGX_HTTP_CACHE_KEY_LEN + 1 + 10
2161 && name->data[name->len - 10 - 1] == '.')
2162 {
2163 return NGX_OK;
2164 }
2165
2166 if (ctx->size < (off_t) sizeof(ngx_http_file_cache_header_t)) {
2167 ngx_log_error(NGX_LOG_CRIT, ctx->log, 0,
2168 "cache file \"%s\" is too small", name->data);
2169 return NGX_ERROR;
2170 }
2171
2172 ngx_memzero(&c, sizeof(ngx_http_cache_t));
2173 cache = ctx->data;
2174
2175 c.length = ctx->size;
2176 c.fs_size = (ctx->fs_size + cache->bsize - 1) / cache->bsize;
2177
2178 p = &name->data[name->len - 2 * NGX_HTTP_CACHE_KEY_LEN];
2179
2180 for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) {
2181 n = ngx_hextoi(p, 2);
2182
2183 if (n == NGX_ERROR) {
2184 return NGX_ERROR;
2185 }
2186
2187 p += 2;
2188
2189 c.key[i] = (u_char) n;
2190 }
2191
2192 return ngx_http_file_cache_add(cache, &c);
2193 }
2194
2195
2196 static ngx_int_t
ngx_http_file_cache_add(ngx_http_file_cache_t * cache,ngx_http_cache_t * c)2197 ngx_http_file_cache_add(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
2198 {
2199 ngx_http_file_cache_node_t *fcn;
2200
2201 ngx_shmtx_lock(&cache->shpool->mutex);
2202
2203 fcn = ngx_http_file_cache_lookup(cache, c->key);
2204
2205 if (fcn == NULL) {
2206
2207 fcn = ngx_slab_calloc_locked(cache->shpool,
2208 sizeof(ngx_http_file_cache_node_t));
2209 if (fcn == NULL) {
2210 ngx_http_file_cache_set_watermark(cache);
2211
2212 if (cache->fail_time != ngx_time()) {
2213 cache->fail_time = ngx_time();
2214 ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0,
2215 "could not allocate node%s", cache->shpool->log_ctx);
2216 }
2217
2218 ngx_shmtx_unlock(&cache->shpool->mutex);
2219 return NGX_ERROR;
2220 }
2221
2222 cache->sh->count++;
2223
2224 ngx_memcpy((u_char *) &fcn->node.key, c->key, sizeof(ngx_rbtree_key_t));
2225
2226 ngx_memcpy(fcn->key, &c->key[sizeof(ngx_rbtree_key_t)],
2227 NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t));
2228
2229 ngx_rbtree_insert(&cache->sh->rbtree, &fcn->node);
2230
2231 fcn->uses = 1;
2232 fcn->exists = 1;
2233 fcn->fs_size = c->fs_size;
2234
2235 cache->sh->size += c->fs_size;
2236
2237 } else {
2238 ngx_queue_remove(&fcn->queue);
2239 }
2240
2241 fcn->expire = ngx_time() + cache->inactive;
2242
2243 ngx_queue_insert_head(&cache->sh->queue, &fcn->queue);
2244
2245 ngx_shmtx_unlock(&cache->shpool->mutex);
2246
2247 return NGX_OK;
2248 }
2249
2250
2251 static ngx_int_t
ngx_http_file_cache_delete_file(ngx_tree_ctx_t * ctx,ngx_str_t * path)2252 ngx_http_file_cache_delete_file(ngx_tree_ctx_t *ctx, ngx_str_t *path)
2253 {
2254 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0,
2255 "http file cache delete: \"%s\"", path->data);
2256
2257 if (ngx_delete_file(path->data) == NGX_FILE_ERROR) {
2258 ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno,
2259 ngx_delete_file_n " \"%s\" failed", path->data);
2260 }
2261
2262 return NGX_OK;
2263 }
2264
2265
2266 static void
ngx_http_file_cache_set_watermark(ngx_http_file_cache_t * cache)2267 ngx_http_file_cache_set_watermark(ngx_http_file_cache_t *cache)
2268 {
2269 cache->sh->watermark = cache->sh->count - cache->sh->count / 8;
2270
2271 ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
2272 "http file cache watermark: %ui", cache->sh->watermark);
2273 }
2274
2275
2276 time_t
ngx_http_file_cache_valid(ngx_array_t * cache_valid,ngx_uint_t status)2277 ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status)
2278 {
2279 ngx_uint_t i;
2280 ngx_http_cache_valid_t *valid;
2281
2282 if (cache_valid == NULL) {
2283 return 0;
2284 }
2285
2286 valid = cache_valid->elts;
2287 for (i = 0; i < cache_valid->nelts; i++) {
2288
2289 if (valid[i].status == 0) {
2290 return valid[i].valid;
2291 }
2292
2293 if (valid[i].status == status) {
2294 return valid[i].valid;
2295 }
2296 }
2297
2298 return 0;
2299 }
2300
2301
2302 char *
ngx_http_file_cache_set_slot(ngx_conf_t * cf,ngx_command_t * cmd,void * conf)2303 ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
2304 {
2305 char *confp = conf;
2306
2307 off_t max_size;
2308 u_char *last, *p;
2309 time_t inactive;
2310 ssize_t size;
2311 ngx_str_t s, name, *value;
2312 ngx_int_t loader_files, manager_files;
2313 ngx_msec_t loader_sleep, manager_sleep, loader_threshold,
2314 manager_threshold;
2315 ngx_uint_t i, n, use_temp_path;
2316 ngx_array_t *caches;
2317 ngx_http_file_cache_t *cache, **ce;
2318
2319 cache = ngx_pcalloc(cf->pool, sizeof(ngx_http_file_cache_t));
2320 if (cache == NULL) {
2321 return NGX_CONF_ERROR;
2322 }
2323
2324 cache->path = ngx_pcalloc(cf->pool, sizeof(ngx_path_t));
2325 if (cache->path == NULL) {
2326 return NGX_CONF_ERROR;
2327 }
2328
2329 use_temp_path = 1;
2330
2331 inactive = 600;
2332
2333 loader_files = 100;
2334 loader_sleep = 50;
2335 loader_threshold = 200;
2336
2337 manager_files = 100;
2338 manager_sleep = 50;
2339 manager_threshold = 200;
2340
2341 name.len = 0;
2342 size = 0;
2343 max_size = NGX_MAX_OFF_T_VALUE;
2344
2345 value = cf->args->elts;
2346
2347 cache->path->name = value[1];
2348
2349 if (cache->path->name.data[cache->path->name.len - 1] == '/') {
2350 cache->path->name.len--;
2351 }
2352
2353 if (ngx_conf_full_name(cf->cycle, &cache->path->name, 0) != NGX_OK) {
2354 return NGX_CONF_ERROR;
2355 }
2356
2357 for (i = 2; i < cf->args->nelts; i++) {
2358
2359 if (ngx_strncmp(value[i].data, "levels=", 7) == 0) {
2360
2361 p = value[i].data + 7;
2362 last = value[i].data + value[i].len;
2363
2364 for (n = 0; n < NGX_MAX_PATH_LEVEL && p < last; n++) {
2365
2366 if (*p > '0' && *p < '3') {
2367
2368 cache->path->level[n] = *p++ - '0';
2369 cache->path->len += cache->path->level[n] + 1;
2370
2371 if (p == last) {
2372 break;
2373 }
2374
2375 if (*p++ == ':' && n < NGX_MAX_PATH_LEVEL - 1 && p < last) {
2376 continue;
2377 }
2378
2379 goto invalid_levels;
2380 }
2381
2382 goto invalid_levels;
2383 }
2384
2385 if (cache->path->len < 10 + NGX_MAX_PATH_LEVEL) {
2386 continue;
2387 }
2388
2389 invalid_levels:
2390
2391 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2392 "invalid \"levels\" \"%V\"", &value[i]);
2393 return NGX_CONF_ERROR;
2394 }
2395
2396 if (ngx_strncmp(value[i].data, "use_temp_path=", 14) == 0) {
2397
2398 if (ngx_strcmp(&value[i].data[14], "on") == 0) {
2399 use_temp_path = 1;
2400
2401 } else if (ngx_strcmp(&value[i].data[14], "off") == 0) {
2402 use_temp_path = 0;
2403
2404 } else {
2405 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2406 "invalid use_temp_path value \"%V\", "
2407 "it must be \"on\" or \"off\"",
2408 &value[i]);
2409 return NGX_CONF_ERROR;
2410 }
2411
2412 continue;
2413 }
2414
2415 if (ngx_strncmp(value[i].data, "keys_zone=", 10) == 0) {
2416
2417 name.data = value[i].data + 10;
2418
2419 p = (u_char *) ngx_strchr(name.data, ':');
2420
2421 if (p == NULL) {
2422 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2423 "invalid keys zone size \"%V\"", &value[i]);
2424 return NGX_CONF_ERROR;
2425 }
2426
2427 name.len = p - name.data;
2428
2429 s.data = p + 1;
2430 s.len = value[i].data + value[i].len - s.data;
2431
2432 size = ngx_parse_size(&s);
2433
2434 if (size == NGX_ERROR) {
2435 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2436 "invalid keys zone size \"%V\"", &value[i]);
2437 return NGX_CONF_ERROR;
2438 }
2439
2440 if (size < (ssize_t) (2 * ngx_pagesize)) {
2441 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2442 "keys zone \"%V\" is too small", &value[i]);
2443 return NGX_CONF_ERROR;
2444 }
2445
2446 continue;
2447 }
2448
2449 if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) {
2450
2451 s.len = value[i].len - 9;
2452 s.data = value[i].data + 9;
2453
2454 inactive = ngx_parse_time(&s, 1);
2455 if (inactive == (time_t) NGX_ERROR) {
2456 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2457 "invalid inactive value \"%V\"", &value[i]);
2458 return NGX_CONF_ERROR;
2459 }
2460
2461 continue;
2462 }
2463
2464 if (ngx_strncmp(value[i].data, "max_size=", 9) == 0) {
2465
2466 s.len = value[i].len - 9;
2467 s.data = value[i].data + 9;
2468
2469 max_size = ngx_parse_offset(&s);
2470 if (max_size < 0) {
2471 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2472 "invalid max_size value \"%V\"", &value[i]);
2473 return NGX_CONF_ERROR;
2474 }
2475
2476 continue;
2477 }
2478
2479 if (ngx_strncmp(value[i].data, "loader_files=", 13) == 0) {
2480
2481 loader_files = ngx_atoi(value[i].data + 13, value[i].len - 13);
2482 if (loader_files == NGX_ERROR) {
2483 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2484 "invalid loader_files value \"%V\"", &value[i]);
2485 return NGX_CONF_ERROR;
2486 }
2487
2488 continue;
2489 }
2490
2491 if (ngx_strncmp(value[i].data, "loader_sleep=", 13) == 0) {
2492
2493 s.len = value[i].len - 13;
2494 s.data = value[i].data + 13;
2495
2496 loader_sleep = ngx_parse_time(&s, 0);
2497 if (loader_sleep == (ngx_msec_t) NGX_ERROR) {
2498 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2499 "invalid loader_sleep value \"%V\"", &value[i]);
2500 return NGX_CONF_ERROR;
2501 }
2502
2503 continue;
2504 }
2505
2506 if (ngx_strncmp(value[i].data, "loader_threshold=", 17) == 0) {
2507
2508 s.len = value[i].len - 17;
2509 s.data = value[i].data + 17;
2510
2511 loader_threshold = ngx_parse_time(&s, 0);
2512 if (loader_threshold == (ngx_msec_t) NGX_ERROR) {
2513 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2514 "invalid loader_threshold value \"%V\"", &value[i]);
2515 return NGX_CONF_ERROR;
2516 }
2517
2518 continue;
2519 }
2520
2521 if (ngx_strncmp(value[i].data, "manager_files=", 14) == 0) {
2522
2523 manager_files = ngx_atoi(value[i].data + 14, value[i].len - 14);
2524 if (manager_files == NGX_ERROR) {
2525 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2526 "invalid manager_files value \"%V\"", &value[i]);
2527 return NGX_CONF_ERROR;
2528 }
2529
2530 continue;
2531 }
2532
2533 if (ngx_strncmp(value[i].data, "manager_sleep=", 14) == 0) {
2534
2535 s.len = value[i].len - 14;
2536 s.data = value[i].data + 14;
2537
2538 manager_sleep = ngx_parse_time(&s, 0);
2539 if (manager_sleep == (ngx_msec_t) NGX_ERROR) {
2540 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2541 "invalid manager_sleep value \"%V\"", &value[i]);
2542 return NGX_CONF_ERROR;
2543 }
2544
2545 continue;
2546 }
2547
2548 if (ngx_strncmp(value[i].data, "manager_threshold=", 18) == 0) {
2549
2550 s.len = value[i].len - 18;
2551 s.data = value[i].data + 18;
2552
2553 manager_threshold = ngx_parse_time(&s, 0);
2554 if (manager_threshold == (ngx_msec_t) NGX_ERROR) {
2555 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2556 "invalid manager_threshold value \"%V\"", &value[i]);
2557 return NGX_CONF_ERROR;
2558 }
2559
2560 continue;
2561 }
2562
2563 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2564 "invalid parameter \"%V\"", &value[i]);
2565 return NGX_CONF_ERROR;
2566 }
2567
2568 if (name.len == 0 || size == 0) {
2569 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2570 "\"%V\" must have \"keys_zone\" parameter",
2571 &cmd->name);
2572 return NGX_CONF_ERROR;
2573 }
2574
2575 cache->path->manager = ngx_http_file_cache_manager;
2576 cache->path->loader = ngx_http_file_cache_loader;
2577 cache->path->data = cache;
2578 cache->path->conf_file = cf->conf_file->file.name.data;
2579 cache->path->line = cf->conf_file->line;
2580 cache->loader_files = loader_files;
2581 cache->loader_sleep = loader_sleep;
2582 cache->loader_threshold = loader_threshold;
2583 cache->manager_files = manager_files;
2584 cache->manager_sleep = manager_sleep;
2585 cache->manager_threshold = manager_threshold;
2586
2587 if (ngx_add_path(cf, &cache->path) != NGX_OK) {
2588 return NGX_CONF_ERROR;
2589 }
2590
2591 cache->shm_zone = ngx_shared_memory_add(cf, &name, size, cmd->post);
2592 if (cache->shm_zone == NULL) {
2593 return NGX_CONF_ERROR;
2594 }
2595
2596 if (cache->shm_zone->data) {
2597 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2598 "duplicate zone \"%V\"", &name);
2599 return NGX_CONF_ERROR;
2600 }
2601
2602
2603 cache->shm_zone->init = ngx_http_file_cache_init;
2604 cache->shm_zone->data = cache;
2605
2606 cache->use_temp_path = use_temp_path;
2607
2608 cache->inactive = inactive;
2609 cache->max_size = max_size;
2610
2611 caches = (ngx_array_t *) (confp + cmd->offset);
2612
2613 ce = ngx_array_push(caches);
2614 if (ce == NULL) {
2615 return NGX_CONF_ERROR;
2616 }
2617
2618 *ce = cache;
2619
2620 return NGX_CONF_OK;
2621 }
2622
2623
2624 char *
ngx_http_file_cache_valid_set_slot(ngx_conf_t * cf,ngx_command_t * cmd,void * conf)2625 ngx_http_file_cache_valid_set_slot(ngx_conf_t *cf, ngx_command_t *cmd,
2626 void *conf)
2627 {
2628 char *p = conf;
2629
2630 time_t valid;
2631 ngx_str_t *value;
2632 ngx_int_t status;
2633 ngx_uint_t i, n;
2634 ngx_array_t **a;
2635 ngx_http_cache_valid_t *v;
2636 static ngx_uint_t statuses[] = { 200, 301, 302 };
2637
2638 a = (ngx_array_t **) (p + cmd->offset);
2639
2640 if (*a == NGX_CONF_UNSET_PTR) {
2641 *a = ngx_array_create(cf->pool, 1, sizeof(ngx_http_cache_valid_t));
2642 if (*a == NULL) {
2643 return NGX_CONF_ERROR;
2644 }
2645 }
2646
2647 value = cf->args->elts;
2648 n = cf->args->nelts - 1;
2649
2650 valid = ngx_parse_time(&value[n], 1);
2651 if (valid == (time_t) NGX_ERROR) {
2652 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2653 "invalid time value \"%V\"", &value[n]);
2654 return NGX_CONF_ERROR;
2655 }
2656
2657 if (n == 1) {
2658
2659 for (i = 0; i < 3; i++) {
2660 v = ngx_array_push(*a);
2661 if (v == NULL) {
2662 return NGX_CONF_ERROR;
2663 }
2664
2665 v->status = statuses[i];
2666 v->valid = valid;
2667 }
2668
2669 return NGX_CONF_OK;
2670 }
2671
2672 for (i = 1; i < n; i++) {
2673
2674 if (ngx_strcmp(value[i].data, "any") == 0) {
2675
2676 status = 0;
2677
2678 } else {
2679
2680 status = ngx_atoi(value[i].data, value[i].len);
2681 if (status < 100 || status > 599) {
2682 ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
2683 "invalid status \"%V\"", &value[i]);
2684 return NGX_CONF_ERROR;
2685 }
2686 }
2687
2688 v = ngx_array_push(*a);
2689 if (v == NULL) {
2690 return NGX_CONF_ERROR;
2691 }
2692
2693 v->status = status;
2694 v->valid = valid;
2695 }
2696
2697 return NGX_CONF_OK;
2698 }
2699