1 #ifndef _CHUNK_H_
2 #define _CHUNK_H_
3 #include "first.h"
4
5 #include "buffer.h"
6 #include "array.h"
7 #include "fdlog.h"
8
9 /* both should be way smaller than SSIZE_MAX :) */
10 #define MAX_READ_LIMIT (256*1024)
11 #define MAX_WRITE_LIMIT (256*1024)
12
13 typedef struct chunk_file_view {
14 char *mptr; /* base pointer of mmap'ed area */
15 off_t mlen; /* length of mmap'ed area */
16 off_t foff; /* offset from the start of the file */
17 int refcnt;
18 } chunk_file_view;
19
20 typedef struct chunk {
21 struct chunk *next;
22 enum { MEM_CHUNK, FILE_CHUNK } type;
23
24 buffer *mem; /* either the storage of the mem-chunk or the name of the file */
25
26 /* the size of the chunk is either:
27 * - mem-chunk: buffer_string_length(chunk::mem) - c->offset
28 * - file-chunk: chunk::file.length - c->offset
29 */
30 off_t offset;
31
32 struct {
33 /* filechunk */
34 off_t length; /* end pos + 1 in file (octets to send: file.length - c->offset) */
35
36 int fd;
37 int is_temp; /* file is temporary and will be deleted if on cleanup */
38 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
39 chunk_file_view *view;
40 #endif
41 void *ref;
42 void(*refchg)(void *, int);
43 } file;
44 } chunk;
45
46 typedef struct chunkqueue {
47 chunk *first;
48 chunk *last;
49
50 off_t bytes_in, bytes_out;
51
52 const array *tempdirs;
53 off_t upload_temp_file_size;
54 unsigned int tempdir_idx;
55 } chunkqueue;
56
57 ssize_t chunk_file_pread (int fd, void *buf, size_t count, off_t offset);
58
59 __attribute_returns_nonnull__
60 buffer * chunk_buffer_acquire(void);
61
62 void chunk_buffer_release(buffer *b);
63
64 __attribute_nonnull__()
65 void chunk_buffer_yield(buffer *b);
66
67 size_t chunk_buffer_prepare_append (buffer *b, size_t sz);
68
69 void chunkqueue_chunk_pool_clear(void);
70 void chunkqueue_chunk_pool_free(void);
71
72 __attribute_returns_nonnull__
73 chunkqueue *chunkqueue_init(chunkqueue *cq);
74
75 __attribute_cold__
76 void chunkqueue_set_chunk_size (size_t sz);
77
78 __attribute_cold__
79 void chunkqueue_set_tempdirs_default_reset (void);
80
81 __attribute_cold__
82 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size);
83
84 void chunkqueue_set_tempdirs(chunkqueue * restrict cq, const array * restrict tempdirs, off_t upload_temp_file_size);
85
86 void chunkqueue_append_file(chunkqueue * restrict cq, const buffer * restrict fn, off_t offset, off_t len); /* copies "fn" */
87 void chunkqueue_append_file_fd(chunkqueue * restrict cq, const buffer * restrict fn, int fd, off_t offset, off_t len); /* copies "fn" */
88 void chunkqueue_append_mem(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */
89 void chunkqueue_append_mem_min(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */
90
91 __attribute_nonnull__()
92 void chunkqueue_append_buffer(chunkqueue * restrict cq, buffer * restrict mem); /* may reset "mem" */
93
94 __attribute_nonnull__()
95 void chunkqueue_append_chunkqueue(chunkqueue * restrict cq, chunkqueue * restrict src);
96
97 __attribute_returns_nonnull__
98 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz);
99
100 __attribute_returns_nonnull__
101 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq);
102
103 void chunkqueue_prepend_buffer_commit(chunkqueue *cq);
104
105 __attribute_returns_nonnull__
106 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz);
107
108 __attribute_returns_nonnull__
109 buffer * chunkqueue_append_buffer_open(chunkqueue *cq);
110
111 void chunkqueue_append_buffer_commit(chunkqueue *cq);
112
113 int chunkqueue_append_mem_to_tempfile(chunkqueue * restrict cq, const char * restrict mem, size_t len, log_error_st * const restrict errh);
114
115 #ifdef HAVE_SPLICE
116 ssize_t chunkqueue_append_splice_pipe_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh);
117 ssize_t chunkqueue_append_splice_sock_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh);
118 __attribute_cold__
119 void chunkqueue_internal_pipes(int init);
120 #else
121 #define chunkqueue_internal_pipes(init) do { } while (0)
122 #endif
123
124 /* functions to handle buffers to read into: */
125 /* obtain/reserve memory in chunkqueue at least len (input) size,
126 * return pointer to memory with len (output) available for use
127 * modifying the chunkqueue invalidates the memory area.
128 * should always be followed by chunkqueue_get_memory(),
129 * even if nothing was read.
130 * pass 0 in len for mem at least half of chunk_buf_sz
131 */
132 __attribute_returns_nonnull__
133 char * chunkqueue_get_memory(chunkqueue * restrict cq, size_t * restrict len);
134 /* commit len bytes of mem obtained from chunkqueue_get_memory() */
135 void chunkqueue_use_memory(chunkqueue * restrict cq, chunk *ckpt, size_t len);
136 void chunkqueue_update_file(chunkqueue * restrict cq, chunk *c, off_t len);
137
138 /* mark first "len" bytes as written (incrementing chunk offsets)
139 * and remove finished chunks
140 */
141 void chunkqueue_mark_written(chunkqueue *cq, off_t len);
142
143 void chunkqueue_remove_finished_chunks(chunkqueue *cq);
144
145 __attribute_cold__
146 void chunkqueue_remove_empty_chunks(chunkqueue *cq);
147
148 void chunkqueue_steal(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len);
149 int chunkqueue_steal_with_tempfiles(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len, log_error_st * const restrict errh);
150 void chunkqueue_append_cq_range (chunkqueue *dst, const chunkqueue *src, off_t offset, off_t len);
151
152 int chunkqueue_open_file_chunk(chunkqueue * restrict cq, log_error_st * const restrict errh);
153
154 void chunkqueue_compact_mem_offset(chunkqueue *cq);
155 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen);
156
157 void chunkqueue_small_resp_optim (chunkqueue * restrict cq);
158
159 ssize_t chunkqueue_write_chunk (int fd, chunkqueue * restrict cq, log_error_st * restrict errh);
160 ssize_t chunkqueue_write_chunk_to_pipe (int fd, chunkqueue * restrict cq, log_error_st * restrict errh);
161
162 int chunkqueue_peek_data (chunkqueue *cq, char **data, uint32_t *dlen, log_error_st * restrict errh);
163 int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, log_error_st * restrict errh);
164
165 buffer * chunkqueue_read_squash (chunkqueue * restrict cq, log_error_st * restrict errh);
166
167 __attribute_pure__
168 __attribute_nonnull__()
169 static inline off_t chunkqueue_length(const chunkqueue *cq);
chunkqueue_length(const chunkqueue * cq)170 static inline off_t chunkqueue_length(const chunkqueue *cq) {
171 return cq->bytes_in - cq->bytes_out;
172 }
173
174 __attribute_cold__
175 void chunkqueue_free(chunkqueue *cq);
176
177 void chunkqueue_reset(chunkqueue *cq);
178
179 __attribute_pure__
180 __attribute_nonnull__()
181 static inline int chunkqueue_is_empty(const chunkqueue *cq);
chunkqueue_is_empty(const chunkqueue * cq)182 static inline int chunkqueue_is_empty(const chunkqueue *cq) {
183 return NULL == cq->first;
184 }
185
186 const chunk_file_view * chunkqueue_chunk_file_viewadj (chunk *c, off_t n, log_error_st * restrict errh);
187
188 __attribute_pure__
189 __attribute_nonnull__()
190 static inline char *
191 chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset);
192 static inline char *
chunk_file_view_dptr(const chunk_file_view * const cfv,off_t offset)193 chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset)
194 {
195 return cfv->mptr - cfv->foff + offset;
196 }
197
198 __attribute_pure__
199 __attribute_nonnull__()
200 static inline off_t
201 chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset);
202 static inline off_t
chunk_file_view_dlen(const chunk_file_view * const cfv,off_t offset)203 chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset)
204 {
205 return cfv->mlen + cfv->foff - offset;
206 }
207
208 static inline const chunk_file_view *
209 chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh);
210 static inline const chunk_file_view *
chunkqueue_chunk_file_view(chunk * const c,const off_t n,log_error_st * const restrict errh)211 chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh)
212 {
213 /*assert(c->type == FILE_CHUNK);*/
214 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
215 /* mmap buffer if offset is outside old mmap area or not mapped at all */
216 const chunk_file_view * const restrict cfv = c->file.view;
217 if (NULL == cfv
218 ? c->file.length - c->offset >= 131072 /* TBD: min chunk size to mmap */
219 : (c->offset - cfv->foff < 0
220 || chunk_file_view_dlen(cfv, c->offset) < (n ? n : 1)))
221 return chunkqueue_chunk_file_viewadj(c, n, errh);
222 return cfv;
223 #else
224 UNUSED(c);
225 UNUSED(n);
226 UNUSED(errh);
227 return NULL;
228 #endif
229 }
230
231 #endif
232