1 #include "jemalloc/internal/jemalloc_internal.h" 2 3 /* 4 * quarantine pointers close to NULL are used to encode state information that 5 * is used for cleaning up during thread shutdown. 6 */ 7 #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 8 #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 9 #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 10 11 /******************************************************************************/ 12 /* Data. */ 13 14 typedef struct quarantine_obj_s quarantine_obj_t; 15 typedef struct quarantine_s quarantine_t; 16 17 struct quarantine_obj_s { 18 void *ptr; 19 size_t usize; 20 }; 21 22 struct quarantine_s { 23 size_t curbytes; 24 size_t curobjs; 25 size_t first; 26 #define LG_MAXOBJS_INIT 10 27 size_t lg_maxobjs; 28 quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ 29 }; 30 31 static void quarantine_cleanup(void *arg); 32 33 malloc_tsd_data(static, quarantine, quarantine_t *, NULL) 34 malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL, 35 quarantine_cleanup) 36 37 /******************************************************************************/ 38 /* Function prototypes for non-inline static functions. */ 39 40 static quarantine_t *quarantine_init(size_t lg_maxobjs); 41 static quarantine_t *quarantine_grow(quarantine_t *quarantine); 42 static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); 43 44 /******************************************************************************/ 45 46 static quarantine_t * 47 quarantine_init(size_t lg_maxobjs) 48 { 49 quarantine_t *quarantine; 50 51 quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + 52 ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); 53 if (quarantine == NULL) 54 return (NULL); 55 quarantine->curbytes = 0; 56 quarantine->curobjs = 0; 57 quarantine->first = 0; 58 quarantine->lg_maxobjs = lg_maxobjs; 59 60 quarantine_tsd_set(&quarantine); 61 62 return (quarantine); 63 } 64 65 static quarantine_t * 66 quarantine_grow(quarantine_t *quarantine) 67 { 68 quarantine_t *ret; 69 70 ret = quarantine_init(quarantine->lg_maxobjs + 1); 71 if (ret == NULL) 72 return (quarantine); 73 74 ret->curbytes = quarantine->curbytes; 75 ret->curobjs = quarantine->curobjs; 76 if (quarantine->first + quarantine->curobjs <= (ZU(1) << 77 quarantine->lg_maxobjs)) { 78 /* objs ring buffer data are contiguous. */ 79 memcpy(ret->objs, &quarantine->objs[quarantine->first], 80 quarantine->curobjs * sizeof(quarantine_obj_t)); 81 } else { 82 /* objs ring buffer data wrap around. */ 83 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 84 quarantine->first; 85 size_t ncopy_b = quarantine->curobjs - ncopy_a; 86 87 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 88 * sizeof(quarantine_obj_t)); 89 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 90 sizeof(quarantine_obj_t)); 91 } 92 93 return (ret); 94 } 95 96 static void 97 quarantine_drain(quarantine_t *quarantine, size_t upper_bound) 98 { 99 100 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { 101 quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 102 assert(obj->usize == isalloc(obj->ptr, config_prof)); 103 idalloc(obj->ptr); 104 quarantine->curbytes -= obj->usize; 105 quarantine->curobjs--; 106 quarantine->first = (quarantine->first + 1) & ((ZU(1) << 107 quarantine->lg_maxobjs) - 1); 108 } 109 } 110 111 void 112 quarantine(void *ptr) 113 { 114 quarantine_t *quarantine; 115 size_t usize = isalloc(ptr, config_prof); 116 117 cassert(config_fill); 118 assert(opt_quarantine); 119 120 quarantine = *quarantine_tsd_get(); 121 if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { 122 if (quarantine == NULL) { 123 if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) == 124 NULL) { 125 idalloc(ptr); 126 return; 127 } 128 } else { 129 if (quarantine == QUARANTINE_STATE_PURGATORY) { 130 /* 131 * Make a note that quarantine() was called 132 * after quarantine_cleanup() was called. 133 */ 134 quarantine = QUARANTINE_STATE_REINCARNATED; 135 quarantine_tsd_set(&quarantine); 136 } 137 idalloc(ptr); 138 return; 139 } 140 } 141 /* 142 * Drain one or more objects if the quarantine size limit would be 143 * exceeded by appending ptr. 144 */ 145 if (quarantine->curbytes + usize > opt_quarantine) { 146 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 147 - usize : 0; 148 quarantine_drain(quarantine, upper_bound); 149 } 150 /* Grow the quarantine ring buffer if it's full. */ 151 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 152 quarantine = quarantine_grow(quarantine); 153 /* quarantine_grow() must free a slot if it fails to grow. */ 154 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 155 /* Append ptr if its size doesn't exceed the quarantine size. */ 156 if (quarantine->curbytes + usize <= opt_quarantine) { 157 size_t offset = (quarantine->first + quarantine->curobjs) & 158 ((ZU(1) << quarantine->lg_maxobjs) - 1); 159 quarantine_obj_t *obj = &quarantine->objs[offset]; 160 obj->ptr = ptr; 161 obj->usize = usize; 162 quarantine->curbytes += usize; 163 quarantine->curobjs++; 164 if (opt_junk) 165 memset(ptr, 0x5a, usize); 166 } else { 167 assert(quarantine->curbytes == 0); 168 idalloc(ptr); 169 } 170 } 171 172 static void 173 quarantine_cleanup(void *arg) 174 { 175 quarantine_t *quarantine = *(quarantine_t **)arg; 176 177 if (quarantine == QUARANTINE_STATE_REINCARNATED) { 178 /* 179 * Another destructor deallocated memory after this destructor 180 * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY 181 * in order to receive another callback. 182 */ 183 quarantine = QUARANTINE_STATE_PURGATORY; 184 quarantine_tsd_set(&quarantine); 185 } else if (quarantine == QUARANTINE_STATE_PURGATORY) { 186 /* 187 * The previous time this destructor was called, we set the key 188 * to QUARANTINE_STATE_PURGATORY so that other destructors 189 * wouldn't cause re-creation of the quarantine. This time, do 190 * nothing, so that the destructor will not be called again. 191 */ 192 } else if (quarantine != NULL) { 193 quarantine_drain(quarantine, 0); 194 idalloc(quarantine); 195 quarantine = QUARANTINE_STATE_PURGATORY; 196 quarantine_tsd_set(&quarantine); 197 } 198 } 199 200 bool 201 quarantine_boot(void) 202 { 203 204 cassert(config_fill); 205 206 if (quarantine_tsd_boot()) 207 return (true); 208 209 return (false); 210 } 211