1 #ifndef _CHUNK_H_ 2 #define _CHUNK_H_ 3 #include "first.h" 4 5 #ifdef _AIX /*(AIX might #define mmap mmap64)*/ 6 #include "sys-mmap.h" 7 #endif 8 9 #include "buffer.h" 10 #include "array.h" 11 12 /* both should be way smaller than SSIZE_MAX :) */ 13 #define MAX_READ_LIMIT (256*1024) 14 #define MAX_WRITE_LIMIT (256*1024) 15 16 struct log_error_st; /*(declaration)*/ 17 18 typedef struct chunk { 19 struct chunk *next; 20 enum { MEM_CHUNK, FILE_CHUNK } type; 21 22 buffer *mem; /* either the storage of the mem-chunk or the name of the file */ 23 24 /* the size of the chunk is either: 25 * - mem-chunk: buffer_string_length(chunk::mem) - c->offset 26 * - file-chunk: chunk::file.length - c->offset 27 */ 28 off_t offset; 29 30 struct { 31 /* filechunk */ 32 off_t length; /* end pos + 1 in file (octets to send: file.length - c->offset) */ 33 34 int fd; 35 int is_temp; /* file is temporary and will be deleted if on cleanup */ 36 struct { 37 char *start; /* the start pointer of the mmap'ed area */ 38 size_t length; /* size of the mmap'ed area */ 39 off_t offset; /* start is <n> octet away from the start of the file */ 40 } mmap; 41 void *ref; 42 void(*refchg)(void *, int); 43 } file; 44 } chunk; 45 46 typedef struct chunkqueue { 47 chunk *first; 48 chunk *last; 49 50 off_t bytes_in, bytes_out; 51 52 const array *tempdirs; 53 off_t upload_temp_file_size; 54 unsigned int tempdir_idx; 55 } chunkqueue; 56 57 __attribute_returns_nonnull__ 58 buffer * chunk_buffer_acquire(void); 59 60 void chunk_buffer_release(buffer *b); 61 62 size_t chunk_buffer_prepare_append (buffer *b, size_t sz); 63 64 void chunkqueue_chunk_pool_clear(void); 65 void chunkqueue_chunk_pool_free(void); 66 67 __attribute_returns_nonnull__ 68 chunkqueue *chunkqueue_init(chunkqueue *cq); 69 70 __attribute_cold__ 71 void chunkqueue_set_chunk_size (size_t sz); 72 73 __attribute_cold__ 74 void chunkqueue_set_tempdirs_default_reset (void); 75 76 __attribute_cold__ 77 void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size); 78 79 __attribute_cold__ 80 void chunkqueue_set_tempdirs(chunkqueue * restrict cq, const array * restrict tempdirs, off_t upload_temp_file_size); 81 82 void chunkqueue_append_file(chunkqueue * restrict cq, const buffer * restrict fn, off_t offset, off_t len); /* copies "fn" */ 83 void chunkqueue_append_file_fd(chunkqueue * restrict cq, const buffer * restrict fn, int fd, off_t offset, off_t len); /* copies "fn" */ 84 void chunkqueue_append_mem(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 85 void chunkqueue_append_mem_min(chunkqueue * restrict cq, const char * restrict mem, size_t len); /* copies memory */ 86 void chunkqueue_append_buffer(chunkqueue * restrict cq, buffer * restrict mem); /* may reset "mem" */ 87 void chunkqueue_append_chunkqueue(chunkqueue * restrict cq, chunkqueue * restrict src); 88 89 __attribute_returns_nonnull__ 90 buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz); 91 92 __attribute_returns_nonnull__ 93 buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq); 94 95 void chunkqueue_prepend_buffer_commit(chunkqueue *cq); 96 97 __attribute_returns_nonnull__ 98 buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz); 99 100 __attribute_returns_nonnull__ 101 buffer * chunkqueue_append_buffer_open(chunkqueue *cq); 102 103 void chunkqueue_append_buffer_commit(chunkqueue *cq); 104 105 int chunkqueue_append_mem_to_tempfile(chunkqueue * restrict cq, const char * restrict mem, size_t len, struct log_error_st * const restrict errh); 106 107 /* functions to handle buffers to read into: */ 108 /* obtain/reserve memory in chunkqueue at least len (input) size, 109 * return pointer to memory with len (output) available for use 110 * modifying the chunkqueue invalidates the memory area. 111 * should always be followed by chunkqueue_get_memory(), 112 * even if nothing was read. 113 * pass 0 in len for mem at least half of chunk_buf_sz 114 */ 115 __attribute_returns_nonnull__ 116 char * chunkqueue_get_memory(chunkqueue * restrict cq, size_t * restrict len); 117 /* commit len bytes of mem obtained from chunkqueue_get_memory() */ 118 void chunkqueue_use_memory(chunkqueue * restrict cq, chunk *ckpt, size_t len); 119 void chunkqueue_update_file(chunkqueue * restrict cq, chunk *c, off_t len); 120 121 /* mark first "len" bytes as written (incrementing chunk offsets) 122 * and remove finished chunks 123 */ 124 void chunkqueue_mark_written(chunkqueue *cq, off_t len); 125 126 void chunkqueue_remove_finished_chunks(chunkqueue *cq); 127 128 void chunkqueue_steal(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len); 129 int chunkqueue_steal_with_tempfiles(chunkqueue * restrict dest, chunkqueue * restrict src, off_t len, struct log_error_st * const restrict errh); 130 void chunkqueue_append_cq_range (chunkqueue *dst, const chunkqueue *src, off_t offset, off_t len); 131 132 int chunkqueue_open_file_chunk(chunkqueue * restrict cq, struct log_error_st * const restrict errh); 133 134 void chunkqueue_compact_mem_offset(chunkqueue *cq); 135 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen); 136 137 void chunkqueue_small_resp_optim (chunkqueue * restrict cq); 138 139 ssize_t chunkqueue_write_chunk (int fd, chunkqueue * restrict cq, struct log_error_st * restrict errh); 140 ssize_t chunkqueue_write_chunk_to_pipe (int fd, chunkqueue * restrict cq, struct log_error_st * restrict errh); 141 142 int chunkqueue_peek_data (chunkqueue *cq, char **data, uint32_t *dlen, struct log_error_st * restrict errh); 143 int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, struct log_error_st * restrict errh); 144 145 buffer * chunkqueue_read_squash (chunkqueue * restrict cq, struct log_error_st * restrict errh); 146 147 __attribute_pure__ 148 static inline off_t chunkqueue_length(const chunkqueue *cq); 149 static inline off_t chunkqueue_length(const chunkqueue *cq) { 150 return cq->bytes_in - cq->bytes_out; 151 } 152 153 __attribute_cold__ 154 void chunkqueue_free(chunkqueue *cq); 155 156 void chunkqueue_reset(chunkqueue *cq); 157 158 __attribute_pure__ 159 static inline int chunkqueue_is_empty(const chunkqueue *cq); 160 static inline int chunkqueue_is_empty(const chunkqueue *cq) { 161 return NULL == cq->first; 162 } 163 164 #endif 165