1 #ifndef _CHUNK_H_ 2 #define _CHUNK_H_ 3 #include "first.h" 4 5 #ifdef _AIX /*(AIX might #define mmap mmap64)*/ 6 #include "sys-mmap.h" 7 #endif 8 9 #include "buffer.h" 10 #include "array.h" 11 #include "fdlog.h" 12 13 /* both should be way smaller than SSIZE_MAX :) */ 14 #define MAX_READ_LIMIT (256*1024) 15 #define MAX_WRITE_LIMIT (256*1024) 16 17 typedef struct chunk { 18 struct chunk *next; 19 enum { MEM_CHUNK, FILE_CHUNK } type; 20 21 buffer *mem; /* either the storage of the mem-chunk or the name of the file */ 22 23 /* the size of the chunk is either: 24 * - mem-chunk: buffer_string_length(chunk::mem) - c->offset 25 * - file-chunk: chunk::file.length - c->offset 26 */ 27 off_t offset; 28 29 struct { 30 /* filechunk */ 31 off_t length; /* end pos + 1 in file (octets to send: file.length - c->offset) */ 32 33 int fd; 34 int is_temp; /* file is temporary and will be deleted if on cleanup */ 35 struct { 36 char *start; /* the start pointer of the mmap'ed area */ 37 size_t length; /* size of the mmap'ed area */ 38 off_t offset; /* start is <n> octet away from the start of the file */ 39 } mmap; 40 void *ref; 41 void(*refchg)(void *, int); 42 } file; 43 } chunk; 44 45 typedef struct chunkqueue { 46 chunk *first; 47 chunk *last; 48 49 off_t bytes_in, bytes_out; 50 51 const array *tempdirs; 52 off_t upload_temp_file_size; 53 unsigned int tempdir_idx; 54 } chunkqueue; 55 56 __attribute_returns_nonnull__ 57 buffer * chunk_buffer_acquire(void); 58 59 void chunk_buffer_release(buffer *b); 60 61 __attribute_nonnull__() 62 void chunk_buffer_yield(buffer *b); 63 64 size_t chunk_buffer_prepare_append (buffer *b, size_t sz); 65 66 void chunkqueue_chunk_pool_clear(void); 67 void chunkqueue_chunk_pool_free(void); 68 69 __attribute_returns_nonnull__ 70 chunkqueue *chunkqueue_init(chunkqueue *cq); 71 72 __attribute_cold__ 73 void chunkqueue_set_chunk_size (size_t sz); 74 75 __attribute_cold__ 76 void chunkqueue_set_tempdirs_default_reset (void); 77 78 __attribute_cold__ 79 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size); 80 81 void chunkqueue_set_tempdirs(chunkqueue * restrict cq, const array * restrict tempdirs, off_t upload_temp_file_size); 82 83 void chunkqueue_append_file(chunkqueue * restrict cq, const buffer * restrict fn, off_t offset, off_t len); /* copies "fn" */ 84 void chunkqueue_append_file_fd(chunkqueue * restrict cq, const buffer * restrict fn, int fd, off_t offset, off_t len); /* copies "fn" */ 85 void chunkqueue_append_mem(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 86 void chunkqueue_append_mem_min(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 87 88 __attribute_nonnull__() 89 void chunkqueue_append_buffer(chunkqueue * restrict cq, buffer * restrict mem); /* may reset "mem" */ 90 91 __attribute_nonnull__() 92 void chunkqueue_append_chunkqueue(chunkqueue * restrict cq, chunkqueue * restrict src); 93 94 __attribute_returns_nonnull__ 95 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz); 96 97 __attribute_returns_nonnull__ 98 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq); 99 100 void chunkqueue_prepend_buffer_commit(chunkqueue *cq); 101 102 __attribute_returns_nonnull__ 103 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz); 104 105 __attribute_returns_nonnull__ 106 buffer * chunkqueue_append_buffer_open(chunkqueue *cq); 107 108 void chunkqueue_append_buffer_commit(chunkqueue *cq); 109 110 int chunkqueue_append_mem_to_tempfile(chunkqueue * restrict cq, const char * restrict mem, size_t len, log_error_st * const restrict errh); 111 112 #ifdef HAVE_SPLICE 113 ssize_t chunkqueue_append_splice_pipe_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh); 114 ssize_t chunkqueue_append_splice_sock_tempfile(chunkqueue * restrict cq, int fd, unsigned int len, log_error_st * restrict errh); 115 __attribute_cold__ 116 void chunkqueue_internal_pipes(int init); 117 #else 118 #define chunkqueue_internal_pipes(init) do { } while (0) 119 #endif 120 121 /* functions to handle buffers to read into: */ 122 /* obtain/reserve memory in chunkqueue at least len (input) size, 123 * return pointer to memory with len (output) available for use 124 * modifying the chunkqueue invalidates the memory area. 125 * should always be followed by chunkqueue_get_memory(), 126 * even if nothing was read. 127 * pass 0 in len for mem at least half of chunk_buf_sz 128 */ 129 __attribute_returns_nonnull__ 130 char * chunkqueue_get_memory(chunkqueue * restrict cq, size_t * restrict len); 131 /* commit len bytes of mem obtained from chunkqueue_get_memory() */ 132 void chunkqueue_use_memory(chunkqueue * restrict cq, chunk *ckpt, size_t len); 133 void chunkqueue_update_file(chunkqueue * restrict cq, chunk *c, off_t len); 134 135 /* mark first "len" bytes as written (incrementing chunk offsets) 136 * and remove finished chunks 137 */ 138 void chunkqueue_mark_written(chunkqueue *cq, off_t len); 139 140 void chunkqueue_remove_finished_chunks(chunkqueue *cq); 141 142 __attribute_cold__ 143 void chunkqueue_remove_empty_chunks(chunkqueue *cq); 144 145 void chunkqueue_steal(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len); 146 int chunkqueue_steal_with_tempfiles(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len, log_error_st * const restrict errh); 147 void chunkqueue_append_cq_range (chunkqueue *dst, const chunkqueue *src, off_t offset, off_t len); 148 149 int chunkqueue_open_file_chunk(chunkqueue * restrict cq, log_error_st * const restrict errh); 150 151 void chunkqueue_compact_mem_offset(chunkqueue *cq); 152 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen); 153 154 void chunkqueue_small_resp_optim (chunkqueue * restrict cq); 155 156 ssize_t chunkqueue_write_chunk (int fd, chunkqueue * restrict cq, log_error_st * restrict errh); 157 ssize_t chunkqueue_write_chunk_to_pipe (int fd, chunkqueue * restrict cq, log_error_st * restrict errh); 158 159 int chunkqueue_peek_data (chunkqueue *cq, char **data, uint32_t *dlen, log_error_st * restrict errh); 160 int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, log_error_st * restrict errh); 161 162 buffer * chunkqueue_read_squash (chunkqueue * restrict cq, log_error_st * restrict errh); 163 164 __attribute_pure__ 165 static inline off_t chunkqueue_length(const chunkqueue *cq); 166 static inline off_t chunkqueue_length(const chunkqueue *cq) { 167 return cq->bytes_in - cq->bytes_out; 168 } 169 170 __attribute_cold__ 171 void chunkqueue_free(chunkqueue *cq); 172 173 void chunkqueue_reset(chunkqueue *cq); 174 175 __attribute_pure__ 176 static inline int chunkqueue_is_empty(const chunkqueue *cq); 177 static inline int chunkqueue_is_empty(const chunkqueue *cq) { 178 return NULL == cq->first; 179 } 180 181 #endif 182