1
2 /*
3 * Copyright (C) Igor Sysoev
4 * Copyright (C) Nginx, Inc.
5 */
6
7
8 #include <ngx_config.h>
9 #include <ngx_core.h>
10 #include <ngx_event.h>
11
12
13 #if 0
14 #define NGX_SENDFILE_LIMIT 4096
15 #endif
16
17 /*
18 * When DIRECTIO is enabled FreeBSD, Solaris, and MacOSX read directly
19 * to an application memory from a device if parameters are aligned
20 * to device sector boundary (512 bytes). They fallback to usual read
21 * operation if the parameters are not aligned.
22 * Linux allows DIRECTIO only if the parameters are aligned to a filesystem
23 * sector boundary, otherwise it returns EINVAL. The sector size is
24 * usually 512 bytes, however, on XFS it may be 4096 bytes.
25 */
26
27 #define NGX_NONE 1
28
29
30 static ngx_inline ngx_int_t
31 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
32 #if (NGX_HAVE_AIO_SENDFILE)
33 static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx,
34 ngx_file_t *file);
35 #endif
36 static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
37 ngx_chain_t **chain, ngx_chain_t *in);
38 static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
39 off_t bsize);
40 static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
41 off_t bsize);
42 static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
43
44
45 ngx_int_t
ngx_output_chain(ngx_output_chain_ctx_t * ctx,ngx_chain_t * in)46 ngx_output_chain(ngx_output_chain_ctx_t *ctx, ngx_chain_t *in)
47 {
48 off_t bsize;
49 ngx_int_t rc, last;
50 ngx_chain_t *cl, *out, **last_out;
51
52 if (ctx->in == NULL && ctx->busy == NULL
53 #if (NGX_HAVE_FILE_AIO || NGX_THREADS)
54 && !ctx->aio
55 #endif
56 )
57 {
58 /*
59 * the short path for the case when the ctx->in and ctx->busy chains
60 * are empty, the incoming chain is empty too or has the single buf
61 * that does not require the copy
62 */
63
64 if (in == NULL) {
65 return ctx->output_filter(ctx->filter_ctx, in);
66 }
67
68 if (in->next == NULL
69 #if (NGX_SENDFILE_LIMIT)
70 && !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
71 #endif
72 && ngx_output_chain_as_is(ctx, in->buf))
73 {
74 return ctx->output_filter(ctx->filter_ctx, in);
75 }
76 }
77
78 /* add the incoming buf to the chain ctx->in */
79
80 if (in) {
81 if (ngx_output_chain_add_copy(ctx->pool, &ctx->in, in) == NGX_ERROR) {
82 return NGX_ERROR;
83 }
84 }
85
86 out = NULL;
87 last_out = &out;
88 last = NGX_NONE;
89
90 for ( ;; ) {
91
92 #if (NGX_HAVE_FILE_AIO || NGX_THREADS)
93 if (ctx->aio) {
94 return NGX_AGAIN;
95 }
96 #endif
97
98 while (ctx->in) {
99
100 /*
101 * cycle while there are the ctx->in bufs
102 * and there are the free output bufs to copy in
103 */
104
105 bsize = ngx_buf_size(ctx->in->buf);
106
107 if (bsize == 0 && !ngx_buf_special(ctx->in->buf)) {
108
109 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
110 "zero size buf in output "
111 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
112 ctx->in->buf->temporary,
113 ctx->in->buf->recycled,
114 ctx->in->buf->in_file,
115 ctx->in->buf->start,
116 ctx->in->buf->pos,
117 ctx->in->buf->last,
118 ctx->in->buf->file,
119 ctx->in->buf->file_pos,
120 ctx->in->buf->file_last);
121
122 ngx_debug_point();
123
124 ctx->in = ctx->in->next;
125
126 continue;
127 }
128
129 if (bsize < 0) {
130
131 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
132 "negative size buf in output "
133 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
134 ctx->in->buf->temporary,
135 ctx->in->buf->recycled,
136 ctx->in->buf->in_file,
137 ctx->in->buf->start,
138 ctx->in->buf->pos,
139 ctx->in->buf->last,
140 ctx->in->buf->file,
141 ctx->in->buf->file_pos,
142 ctx->in->buf->file_last);
143
144 ngx_debug_point();
145
146 return NGX_ERROR;
147 }
148
149 if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
150
151 /* move the chain link to the output chain */
152
153 cl = ctx->in;
154 ctx->in = cl->next;
155
156 *last_out = cl;
157 last_out = &cl->next;
158 cl->next = NULL;
159
160 continue;
161 }
162
163 if (ctx->buf == NULL) {
164
165 rc = ngx_output_chain_align_file_buf(ctx, bsize);
166
167 if (rc == NGX_ERROR) {
168 return NGX_ERROR;
169 }
170
171 if (rc != NGX_OK) {
172
173 if (ctx->free) {
174
175 /* get the free buf */
176
177 cl = ctx->free;
178 ctx->buf = cl->buf;
179 ctx->free = cl->next;
180
181 ngx_free_chain(ctx->pool, cl);
182
183 } else if (out || ctx->allocated == ctx->bufs.num) {
184
185 break;
186
187 } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
188 return NGX_ERROR;
189 }
190 }
191 }
192
193 rc = ngx_output_chain_copy_buf(ctx);
194
195 if (rc == NGX_ERROR) {
196 return rc;
197 }
198
199 if (rc == NGX_AGAIN) {
200 if (out) {
201 break;
202 }
203
204 return rc;
205 }
206
207 /* delete the completed buf from the ctx->in chain */
208
209 if (ngx_buf_size(ctx->in->buf) == 0) {
210 ctx->in = ctx->in->next;
211 }
212
213 cl = ngx_alloc_chain_link(ctx->pool);
214 if (cl == NULL) {
215 return NGX_ERROR;
216 }
217
218 cl->buf = ctx->buf;
219 cl->next = NULL;
220 *last_out = cl;
221 last_out = &cl->next;
222 ctx->buf = NULL;
223 }
224
225 if (out == NULL && last != NGX_NONE) {
226
227 if (ctx->in) {
228 return NGX_AGAIN;
229 }
230
231 return last;
232 }
233
234 last = ctx->output_filter(ctx->filter_ctx, out);
235
236 if (last == NGX_ERROR || last == NGX_DONE) {
237 return last;
238 }
239
240 ngx_chain_update_chains(ctx->pool, &ctx->free, &ctx->busy, &out,
241 ctx->tag);
242 last_out = &out;
243 }
244 }
245
246
247 static ngx_inline ngx_int_t
ngx_output_chain_as_is(ngx_output_chain_ctx_t * ctx,ngx_buf_t * buf)248 ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
249 {
250 ngx_uint_t sendfile;
251
252 if (ngx_buf_special(buf)) {
253 return 1;
254 }
255
256 #if (NGX_THREADS)
257 if (buf->in_file) {
258 buf->file->thread_handler = ctx->thread_handler;
259 buf->file->thread_ctx = ctx->filter_ctx;
260 }
261 #endif
262
263 if (buf->in_file && buf->file->directio) {
264 return 0;
265 }
266
267 sendfile = ctx->sendfile;
268
269 #if (NGX_SENDFILE_LIMIT)
270
271 if (buf->in_file && buf->file_pos >= NGX_SENDFILE_LIMIT) {
272 sendfile = 0;
273 }
274
275 #endif
276
277 if (!sendfile) {
278
279 if (!ngx_buf_in_memory(buf)) {
280 return 0;
281 }
282
283 buf->in_file = 0;
284 }
285
286 #if (NGX_HAVE_AIO_SENDFILE)
287 if (ctx->aio_preload && buf->in_file) {
288 (void) ngx_output_chain_aio_setup(ctx, buf->file);
289 }
290 #endif
291
292 if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
293 return 0;
294 }
295
296 if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
297 return 0;
298 }
299
300 return 1;
301 }
302
303
304 #if (NGX_HAVE_AIO_SENDFILE)
305
306 static ngx_int_t
ngx_output_chain_aio_setup(ngx_output_chain_ctx_t * ctx,ngx_file_t * file)307 ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file)
308 {
309 ngx_event_aio_t *aio;
310
311 if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) {
312 return NGX_ERROR;
313 }
314
315 aio = file->aio;
316
317 aio->data = ctx->filter_ctx;
318 aio->preload_handler = ctx->aio_preload;
319
320 return NGX_OK;
321 }
322
323 #endif
324
325
326 static ngx_int_t
ngx_output_chain_add_copy(ngx_pool_t * pool,ngx_chain_t ** chain,ngx_chain_t * in)327 ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain,
328 ngx_chain_t *in)
329 {
330 ngx_chain_t *cl, **ll;
331 #if (NGX_SENDFILE_LIMIT)
332 ngx_buf_t *b, *buf;
333 #endif
334
335 ll = chain;
336
337 for (cl = *chain; cl; cl = cl->next) {
338 ll = &cl->next;
339 }
340
341 while (in) {
342
343 cl = ngx_alloc_chain_link(pool);
344 if (cl == NULL) {
345 return NGX_ERROR;
346 }
347
348 #if (NGX_SENDFILE_LIMIT)
349
350 buf = in->buf;
351
352 if (buf->in_file
353 && buf->file_pos < NGX_SENDFILE_LIMIT
354 && buf->file_last > NGX_SENDFILE_LIMIT)
355 {
356 /* split a file buf on two bufs by the sendfile limit */
357
358 b = ngx_calloc_buf(pool);
359 if (b == NULL) {
360 return NGX_ERROR;
361 }
362
363 ngx_memcpy(b, buf, sizeof(ngx_buf_t));
364
365 if (ngx_buf_in_memory(buf)) {
366 buf->pos += (ssize_t) (NGX_SENDFILE_LIMIT - buf->file_pos);
367 b->last = buf->pos;
368 }
369
370 buf->file_pos = NGX_SENDFILE_LIMIT;
371 b->file_last = NGX_SENDFILE_LIMIT;
372
373 cl->buf = b;
374
375 } else {
376 cl->buf = buf;
377 in = in->next;
378 }
379
380 #else
381 cl->buf = in->buf;
382 in = in->next;
383
384 #endif
385
386 cl->next = NULL;
387 *ll = cl;
388 ll = &cl->next;
389 }
390
391 return NGX_OK;
392 }
393
394
395 static ngx_int_t
ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t * ctx,off_t bsize)396 ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
397 {
398 size_t size;
399 ngx_buf_t *in;
400
401 in = ctx->in->buf;
402
403 if (in->file == NULL || !in->file->directio) {
404 return NGX_DECLINED;
405 }
406
407 ctx->directio = 1;
408
409 size = (size_t) (in->file_pos - (in->file_pos & ~(ctx->alignment - 1)));
410
411 if (size == 0) {
412
413 if (bsize >= (off_t) ctx->bufs.size) {
414 return NGX_DECLINED;
415 }
416
417 size = (size_t) bsize;
418
419 } else {
420 size = (size_t) ctx->alignment - size;
421
422 if ((off_t) size > bsize) {
423 size = (size_t) bsize;
424 }
425 }
426
427 ctx->buf = ngx_create_temp_buf(ctx->pool, size);
428 if (ctx->buf == NULL) {
429 return NGX_ERROR;
430 }
431
432 /*
433 * we do not set ctx->buf->tag, because we do not want
434 * to reuse the buf via ctx->free list
435 */
436
437 #if (NGX_HAVE_ALIGNED_DIRECTIO)
438 ctx->unaligned = 1;
439 #endif
440
441 return NGX_OK;
442 }
443
444
445 static ngx_int_t
ngx_output_chain_get_buf(ngx_output_chain_ctx_t * ctx,off_t bsize)446 ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
447 {
448 size_t size;
449 ngx_buf_t *b, *in;
450 ngx_uint_t recycled;
451
452 in = ctx->in->buf;
453 size = ctx->bufs.size;
454 recycled = 1;
455
456 if (in->last_in_chain) {
457
458 if (bsize < (off_t) size) {
459
460 /*
461 * allocate a small temp buf for a small last buf
462 * or its small last part
463 */
464
465 size = (size_t) bsize;
466 recycled = 0;
467
468 } else if (!ctx->directio
469 && ctx->bufs.num == 1
470 && (bsize < (off_t) (size + size / 4)))
471 {
472 /*
473 * allocate a temp buf that equals to a last buf,
474 * if there is no directio, the last buf size is lesser
475 * than 1.25 of bufs.size and the temp buf is single
476 */
477
478 size = (size_t) bsize;
479 recycled = 0;
480 }
481 }
482
483 b = ngx_calloc_buf(ctx->pool);
484 if (b == NULL) {
485 return NGX_ERROR;
486 }
487
488 if (ctx->directio) {
489
490 /*
491 * allocate block aligned to a disk sector size to enable
492 * userland buffer direct usage conjunctly with directio
493 */
494
495 b->start = ngx_pmemalign(ctx->pool, size, (size_t) ctx->alignment);
496 if (b->start == NULL) {
497 return NGX_ERROR;
498 }
499
500 } else {
501 b->start = ngx_palloc(ctx->pool, size);
502 if (b->start == NULL) {
503 return NGX_ERROR;
504 }
505 }
506
507 b->pos = b->start;
508 b->last = b->start;
509 b->end = b->last + size;
510 b->temporary = 1;
511 b->tag = ctx->tag;
512 b->recycled = recycled;
513
514 ctx->buf = b;
515 ctx->allocated++;
516
517 return NGX_OK;
518 }
519
520
521 static ngx_int_t
ngx_output_chain_copy_buf(ngx_output_chain_ctx_t * ctx)522 ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
523 {
524 off_t size;
525 ssize_t n;
526 ngx_buf_t *src, *dst;
527 ngx_uint_t sendfile;
528
529 src = ctx->in->buf;
530 dst = ctx->buf;
531
532 size = ngx_buf_size(src);
533 size = ngx_min(size, dst->end - dst->pos);
534
535 sendfile = ctx->sendfile && !ctx->directio;
536
537 #if (NGX_SENDFILE_LIMIT)
538
539 if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
540 sendfile = 0;
541 }
542
543 #endif
544
545 if (ngx_buf_in_memory(src)) {
546 ngx_memcpy(dst->pos, src->pos, (size_t) size);
547 src->pos += (size_t) size;
548 dst->last += (size_t) size;
549
550 if (src->in_file) {
551
552 if (sendfile) {
553 dst->in_file = 1;
554 dst->file = src->file;
555 dst->file_pos = src->file_pos;
556 dst->file_last = src->file_pos + size;
557
558 } else {
559 dst->in_file = 0;
560 }
561
562 src->file_pos += size;
563
564 } else {
565 dst->in_file = 0;
566 }
567
568 if (src->pos == src->last) {
569 dst->flush = src->flush;
570 dst->last_buf = src->last_buf;
571 dst->last_in_chain = src->last_in_chain;
572 }
573
574 } else {
575
576 #if (NGX_HAVE_ALIGNED_DIRECTIO)
577
578 if (ctx->unaligned) {
579 if (ngx_directio_off(src->file->fd) == NGX_FILE_ERROR) {
580 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
581 ngx_directio_off_n " \"%s\" failed",
582 src->file->name.data);
583 }
584 }
585
586 #endif
587
588 #if (NGX_HAVE_FILE_AIO)
589 if (ctx->aio_handler) {
590 n = ngx_file_aio_read(src->file, dst->pos, (size_t) size,
591 src->file_pos, ctx->pool);
592 if (n == NGX_AGAIN) {
593 ctx->aio_handler(ctx, src->file);
594 return NGX_AGAIN;
595 }
596
597 } else
598 #endif
599 #if (NGX_THREADS)
600 if (ctx->thread_handler) {
601 src->file->thread_task = ctx->thread_task;
602 src->file->thread_handler = ctx->thread_handler;
603 src->file->thread_ctx = ctx->filter_ctx;
604
605 n = ngx_thread_read(src->file, dst->pos, (size_t) size,
606 src->file_pos, ctx->pool);
607 if (n == NGX_AGAIN) {
608 ctx->thread_task = src->file->thread_task;
609 return NGX_AGAIN;
610 }
611
612 } else
613 #endif
614 {
615 n = ngx_read_file(src->file, dst->pos, (size_t) size,
616 src->file_pos);
617 }
618
619 #if (NGX_HAVE_ALIGNED_DIRECTIO)
620
621 if (ctx->unaligned) {
622 ngx_err_t err;
623
624 err = ngx_errno;
625
626 if (ngx_directio_on(src->file->fd) == NGX_FILE_ERROR) {
627 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, ngx_errno,
628 ngx_directio_on_n " \"%s\" failed",
629 src->file->name.data);
630 }
631
632 ngx_set_errno(err);
633
634 ctx->unaligned = 0;
635 }
636
637 #endif
638
639 if (n == NGX_ERROR) {
640 return (ngx_int_t) n;
641 }
642
643 if (n != size) {
644 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
645 ngx_read_file_n " read only %z of %O from \"%s\"",
646 n, size, src->file->name.data);
647 return NGX_ERROR;
648 }
649
650 dst->last += n;
651
652 if (sendfile) {
653 dst->in_file = 1;
654 dst->file = src->file;
655 dst->file_pos = src->file_pos;
656 dst->file_last = src->file_pos + n;
657
658 } else {
659 dst->in_file = 0;
660 }
661
662 src->file_pos += n;
663
664 if (src->file_pos == src->file_last) {
665 dst->flush = src->flush;
666 dst->last_buf = src->last_buf;
667 dst->last_in_chain = src->last_in_chain;
668 }
669 }
670
671 return NGX_OK;
672 }
673
674
675 ngx_int_t
ngx_chain_writer(void * data,ngx_chain_t * in)676 ngx_chain_writer(void *data, ngx_chain_t *in)
677 {
678 ngx_chain_writer_ctx_t *ctx = data;
679
680 off_t size;
681 ngx_chain_t *cl, *ln, *chain;
682 ngx_connection_t *c;
683
684 c = ctx->connection;
685
686 for (size = 0; in; in = in->next) {
687
688 if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) {
689
690 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
691 "zero size buf in chain writer "
692 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
693 in->buf->temporary,
694 in->buf->recycled,
695 in->buf->in_file,
696 in->buf->start,
697 in->buf->pos,
698 in->buf->last,
699 in->buf->file,
700 in->buf->file_pos,
701 in->buf->file_last);
702
703 ngx_debug_point();
704
705 continue;
706 }
707
708 if (ngx_buf_size(in->buf) < 0) {
709
710 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
711 "negative size buf in chain writer "
712 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
713 in->buf->temporary,
714 in->buf->recycled,
715 in->buf->in_file,
716 in->buf->start,
717 in->buf->pos,
718 in->buf->last,
719 in->buf->file,
720 in->buf->file_pos,
721 in->buf->file_last);
722
723 ngx_debug_point();
724
725 return NGX_ERROR;
726 }
727
728 size += ngx_buf_size(in->buf);
729
730 ngx_log_debug2(NGX_LOG_DEBUG_CORE, c->log, 0,
731 "chain writer buf fl:%d s:%uO",
732 in->buf->flush, ngx_buf_size(in->buf));
733
734 cl = ngx_alloc_chain_link(ctx->pool);
735 if (cl == NULL) {
736 return NGX_ERROR;
737 }
738
739 cl->buf = in->buf;
740 cl->next = NULL;
741 *ctx->last = cl;
742 ctx->last = &cl->next;
743 }
744
745 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
746 "chain writer in: %p", ctx->out);
747
748 for (cl = ctx->out; cl; cl = cl->next) {
749
750 if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) {
751
752 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
753 "zero size buf in chain writer "
754 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
755 cl->buf->temporary,
756 cl->buf->recycled,
757 cl->buf->in_file,
758 cl->buf->start,
759 cl->buf->pos,
760 cl->buf->last,
761 cl->buf->file,
762 cl->buf->file_pos,
763 cl->buf->file_last);
764
765 ngx_debug_point();
766
767 continue;
768 }
769
770 if (ngx_buf_size(cl->buf) < 0) {
771
772 ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
773 "negative size buf in chain writer "
774 "t:%d r:%d f:%d %p %p-%p %p %O-%O",
775 cl->buf->temporary,
776 cl->buf->recycled,
777 cl->buf->in_file,
778 cl->buf->start,
779 cl->buf->pos,
780 cl->buf->last,
781 cl->buf->file,
782 cl->buf->file_pos,
783 cl->buf->file_last);
784
785 ngx_debug_point();
786
787 return NGX_ERROR;
788 }
789
790 size += ngx_buf_size(cl->buf);
791 }
792
793 if (size == 0 && !c->buffered) {
794 return NGX_OK;
795 }
796
797 chain = c->send_chain(c, ctx->out, ctx->limit);
798
799 ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0,
800 "chain writer out: %p", chain);
801
802 if (chain == NGX_CHAIN_ERROR) {
803 return NGX_ERROR;
804 }
805
806 for (cl = ctx->out; cl && cl != chain; /* void */) {
807 ln = cl;
808 cl = cl->next;
809 ngx_free_chain(ctx->pool, ln);
810 }
811
812 ctx->out = chain;
813
814 if (ctx->out == NULL) {
815 ctx->last = &ctx->out;
816
817 if (!c->buffered) {
818 return NGX_OK;
819 }
820 }
821
822 return NGX_AGAIN;
823 }
824