1 #ifndef _CHUNK_H_ 2 #define _CHUNK_H_ 3 #include "first.h" 4 5 #include "buffer.h" 6 #include "array.h" 7 #include "fdlog.h" 8 9 /* both should be way smaller than SSIZE_MAX :) */ 10 #define MAX_READ_LIMIT (256*1024) 11 #define MAX_WRITE_LIMIT (256*1024) 12 13 typedef struct chunk_file_view { 14 char *mptr; /* base pointer of mmap'ed area */ 15 off_t mlen; /* length of mmap'ed area */ 16 off_t foff; /* offset from the start of the file */ 17 int refcnt; 18 } chunk_file_view; 19 20 typedef struct chunk { 21 struct chunk *next; 22 enum { MEM_CHUNK, FILE_CHUNK } type; 23 24 buffer *mem; /* either the storage of the mem-chunk or the name of the file */ 25 26 /* the size of the chunk is either: 27 * - mem-chunk: buffer_string_length(chunk::mem) - c->offset 28 * - file-chunk: chunk::file.length - c->offset 29 */ 30 off_t offset; 31 32 struct { 33 /* filechunk */ 34 off_t length; /* end pos + 1 in file (octets to send: file.length - c->offset) */ 35 36 int fd; 37 int is_temp; /* file is temporary and will be deleted if on cleanup */ 38 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ 39 chunk_file_view *view; 40 #endif 41 void *ref; 42 void(*refchg)(void *, int); 43 } file; 44 } chunk; 45 46 typedef struct chunkqueue { 47 chunk *first; 48 chunk *last; 49 50 off_t bytes_in, bytes_out; 51 52 const array *tempdirs; 53 off_t upload_temp_file_size; 54 unsigned int tempdir_idx; 55 } chunkqueue; 56 57 ssize_t chunk_file_pread (int fd, void *buf, size_t count, off_t offset); 58 59 __attribute_returns_nonnull__ 60 buffer * chunk_buffer_acquire(void); 61 62 void chunk_buffer_release(buffer *b); 63 64 __attribute_nonnull__() 65 void chunk_buffer_yield(buffer *b); 66 67 size_t chunk_buffer_prepare_append (buffer *b, size_t sz); 68 69 void chunkqueue_chunk_pool_clear(void); 70 void chunkqueue_chunk_pool_free(void); 71 72 __attribute_returns_nonnull__ 73 chunkqueue *chunkqueue_init(chunkqueue *cq); 74 75 __attribute_cold__ 76 void chunkqueue_set_chunk_size (size_t sz); 77 78 __attribute_cold__ 79 void chunkqueue_set_tempdirs_default_reset (void); 80 81 __attribute_cold__ 82 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size); 83 84 void chunkqueue_set_tempdirs(chunkqueue * restrict cq, const array * restrict tempdirs, off_t upload_temp_file_size); 85 86 void chunkqueue_append_file(chunkqueue * restrict cq, const buffer * restrict fn, off_t offset, off_t len); /* copies "fn" */ 87 void chunkqueue_append_file_fd(chunkqueue * restrict cq, const buffer * restrict fn, int fd, off_t offset, off_t len); /* copies "fn" */ 88 void chunkqueue_append_mem(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 89 void chunkqueue_append_mem_min(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 90 91 __attribute_nonnull__() 92 void chunkqueue_append_buffer(chunkqueue * restrict cq, buffer * restrict mem); /* may reset "mem" */ 93 94 __attribute_nonnull__() 95 void chunkqueue_append_chunkqueue(chunkqueue * restrict cq, chunkqueue * restrict src); 96 97 __attribute_returns_nonnull__ 98 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz); 99 100 __attribute_returns_nonnull__ 101 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq); 102 103 void chunkqueue_prepend_buffer_commit(chunkqueue *cq); 104 105 __attribute_returns_nonnull__ 106 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz); 107 108 __attribute_returns_nonnull__ 109 buffer * chunkqueue_append_buffer_open(chunkqueue *cq); 110 111 void chunkqueue_append_buffer_commit(chunkqueue *cq); 112 113 int chunkqueue_append_mem_to_tempfile(chunkqueue * restrict cq, const char * restrict mem, size_t len, log_error_st * const restrict errh); 114 115 #ifdef HAVE_SPLICE 116 ssize_t chunkqueue_append_splice_pipe_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh); 117 ssize_t chunkqueue_append_splice_sock_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh); 118 __attribute_cold__ 119 void chunkqueue_internal_pipes(int init); 120 #else 121 #define chunkqueue_internal_pipes(init) do { } while (0) 122 #endif 123 124 /* functions to handle buffers to read into: */ 125 /* obtain/reserve memory in chunkqueue at least len (input) size, 126 * return pointer to memory with len (output) available for use 127 * modifying the chunkqueue invalidates the memory area. 128 * should always be followed by chunkqueue_get_memory(), 129 * even if nothing was read. 130 * pass 0 in len for mem at least half of chunk_buf_sz 131 */ 132 __attribute_returns_nonnull__ 133 char * chunkqueue_get_memory(chunkqueue * restrict cq, size_t * restrict len); 134 /* commit len bytes of mem obtained from chunkqueue_get_memory() */ 135 void chunkqueue_use_memory(chunkqueue * restrict cq, chunk *ckpt, size_t len); 136 void chunkqueue_update_file(chunkqueue * restrict cq, chunk *c, off_t len); 137 138 /* mark first "len" bytes as written (incrementing chunk offsets) 139 * and remove finished chunks 140 */ 141 void chunkqueue_mark_written(chunkqueue *cq, off_t len); 142 143 void chunkqueue_remove_finished_chunks(chunkqueue *cq); 144 145 __attribute_cold__ 146 void chunkqueue_remove_empty_chunks(chunkqueue *cq); 147 148 void chunkqueue_steal(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len); 149 int chunkqueue_steal_with_tempfiles(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len, log_error_st * const restrict errh); 150 void chunkqueue_append_cq_range (chunkqueue *dst, const chunkqueue *src, off_t offset, off_t len); 151 152 int chunkqueue_open_file_chunk(chunkqueue * restrict cq, log_error_st * const restrict errh); 153 154 void chunkqueue_compact_mem_offset(chunkqueue *cq); 155 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen); 156 157 void chunkqueue_small_resp_optim (chunkqueue * restrict cq); 158 159 ssize_t chunkqueue_write_chunk (int fd, chunkqueue * restrict cq, log_error_st * restrict errh); 160 ssize_t chunkqueue_write_chunk_to_pipe (int fd, chunkqueue * restrict cq, log_error_st * restrict errh); 161 162 int chunkqueue_peek_data (chunkqueue *cq, char **data, uint32_t *dlen, log_error_st * restrict errh); 163 int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, log_error_st * restrict errh); 164 165 buffer * chunkqueue_read_squash (chunkqueue * restrict cq, log_error_st * restrict errh); 166 167 __attribute_pure__ 168 __attribute_nonnull__() 169 static inline off_t chunkqueue_length(const chunkqueue *cq); 170 static inline off_t chunkqueue_length(const chunkqueue *cq) { 171 return cq->bytes_in - cq->bytes_out; 172 } 173 174 __attribute_cold__ 175 void chunkqueue_free(chunkqueue *cq); 176 177 void chunkqueue_reset(chunkqueue *cq); 178 179 __attribute_pure__ 180 __attribute_nonnull__() 181 static inline int chunkqueue_is_empty(const chunkqueue *cq); 182 static inline int chunkqueue_is_empty(const chunkqueue *cq) { 183 return NULL == cq->first; 184 } 185 186 const chunk_file_view * chunkqueue_chunk_file_viewadj (chunk *c, off_t n, log_error_st * restrict errh); 187 188 __attribute_pure__ 189 __attribute_nonnull__() 190 static inline char * 191 chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset); 192 static inline char * 193 chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset) 194 { 195 return cfv->mptr - cfv->foff + offset; 196 } 197 198 __attribute_pure__ 199 __attribute_nonnull__() 200 static inline off_t 201 chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset); 202 static inline off_t 203 chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset) 204 { 205 return cfv->mlen + cfv->foff - offset; 206 } 207 208 static inline const chunk_file_view * 209 chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh); 210 static inline const chunk_file_view * 211 chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh) 212 { 213 /*assert(c->type == FILE_CHUNK);*/ 214 #if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/ 215 /* mmap buffer if offset is outside old mmap area or not mapped at all */ 216 const chunk_file_view * const restrict cfv = c->file.view; 217 if (NULL == cfv 218 ? c->file.length - c->offset >= 131072 /* TBD: min chunk size to mmap */ 219 : (c->offset - cfv->foff < 0 220 || chunk_file_view_dlen(cfv, c->offset) < (n ? n : 1))) 221 return chunkqueue_chunk_file_viewadj(c, n, errh); 222 return cfv; 223 #else 224 UNUSED(c); 225 UNUSED(n); 226 UNUSED(errh); 227 return NULL; 228 #endif 229 } 230 231 #endif 232