1 #define JEMALLOC_QUARANTINE_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /* 5 * quarantine pointers close to NULL are used to encode state information that 6 * is used for cleaning up during thread shutdown. 7 */ 8 #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 9 #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 10 #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 11 12 /******************************************************************************/ 13 /* Data. */ 14 15 malloc_tsd_data(, quarantine, quarantine_t *, NULL) 16 17 /******************************************************************************/ 18 /* Function prototypes for non-inline static functions. */ 19 20 static quarantine_t *quarantine_grow(quarantine_t *quarantine); 21 static void quarantine_drain_one(quarantine_t *quarantine); 22 static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); 23 24 /******************************************************************************/ 25 26 quarantine_t * 27 quarantine_init(size_t lg_maxobjs) 28 { 29 quarantine_t *quarantine; 30 31 quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + 32 ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); 33 if (quarantine == NULL) 34 return (NULL); 35 quarantine->curbytes = 0; 36 quarantine->curobjs = 0; 37 quarantine->first = 0; 38 quarantine->lg_maxobjs = lg_maxobjs; 39 40 quarantine_tsd_set(&quarantine); 41 42 return (quarantine); 43 } 44 45 static quarantine_t * 46 quarantine_grow(quarantine_t *quarantine) 47 { 48 quarantine_t *ret; 49 50 ret = quarantine_init(quarantine->lg_maxobjs + 1); 51 if (ret == NULL) { 52 quarantine_drain_one(quarantine); 53 return (quarantine); 54 } 55 56 ret->curbytes = quarantine->curbytes; 57 ret->curobjs = quarantine->curobjs; 58 if (quarantine->first + quarantine->curobjs <= (ZU(1) << 59 quarantine->lg_maxobjs)) { 60 /* objs ring buffer data are contiguous. */ 61 memcpy(ret->objs, &quarantine->objs[quarantine->first], 62 quarantine->curobjs * sizeof(quarantine_obj_t)); 63 } else { 64 /* objs ring buffer data wrap around. */ 65 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 66 quarantine->first; 67 size_t ncopy_b = quarantine->curobjs - ncopy_a; 68 69 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 70 * sizeof(quarantine_obj_t)); 71 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 72 sizeof(quarantine_obj_t)); 73 } 74 idalloc(quarantine); 75 76 return (ret); 77 } 78 79 static void 80 quarantine_drain_one(quarantine_t *quarantine) 81 { 82 quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 83 assert(obj->usize == isalloc(obj->ptr, config_prof)); 84 idalloc(obj->ptr); 85 quarantine->curbytes -= obj->usize; 86 quarantine->curobjs--; 87 quarantine->first = (quarantine->first + 1) & ((ZU(1) << 88 quarantine->lg_maxobjs) - 1); 89 } 90 91 static void 92 quarantine_drain(quarantine_t *quarantine, size_t upper_bound) 93 { 94 95 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) 96 quarantine_drain_one(quarantine); 97 } 98 99 void 100 quarantine(void *ptr) 101 { 102 quarantine_t *quarantine; 103 size_t usize = isalloc(ptr, config_prof); 104 105 cassert(config_fill); 106 assert(opt_quarantine); 107 108 quarantine = *quarantine_tsd_get(); 109 if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { 110 if (quarantine == QUARANTINE_STATE_PURGATORY) { 111 /* 112 * Make a note that quarantine() was called after 113 * quarantine_cleanup() was called. 114 */ 115 quarantine = QUARANTINE_STATE_REINCARNATED; 116 quarantine_tsd_set(&quarantine); 117 } 118 idalloc(ptr); 119 return; 120 } 121 /* 122 * Drain one or more objects if the quarantine size limit would be 123 * exceeded by appending ptr. 124 */ 125 if (quarantine->curbytes + usize > opt_quarantine) { 126 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 127 - usize : 0; 128 quarantine_drain(quarantine, upper_bound); 129 } 130 /* Grow the quarantine ring buffer if it's full. */ 131 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 132 quarantine = quarantine_grow(quarantine); 133 /* quarantine_grow() must free a slot if it fails to grow. */ 134 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 135 /* Append ptr if its size doesn't exceed the quarantine size. */ 136 if (quarantine->curbytes + usize <= opt_quarantine) { 137 size_t offset = (quarantine->first + quarantine->curobjs) & 138 ((ZU(1) << quarantine->lg_maxobjs) - 1); 139 quarantine_obj_t *obj = &quarantine->objs[offset]; 140 obj->ptr = ptr; 141 obj->usize = usize; 142 quarantine->curbytes += usize; 143 quarantine->curobjs++; 144 if (config_fill && opt_junk) { 145 /* 146 * Only do redzone validation if Valgrind isn't in 147 * operation. 148 */ 149 if ((config_valgrind == false || opt_valgrind == false) 150 && usize <= SMALL_MAXCLASS) 151 arena_quarantine_junk_small(ptr, usize); 152 else 153 memset(ptr, 0x5a, usize); 154 } 155 } else { 156 assert(quarantine->curbytes == 0); 157 idalloc(ptr); 158 } 159 } 160 161 void 162 quarantine_cleanup(void *arg) 163 { 164 quarantine_t *quarantine = *(quarantine_t **)arg; 165 166 if (quarantine == QUARANTINE_STATE_REINCARNATED) { 167 /* 168 * Another destructor deallocated memory after this destructor 169 * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY 170 * in order to receive another callback. 171 */ 172 quarantine = QUARANTINE_STATE_PURGATORY; 173 quarantine_tsd_set(&quarantine); 174 } else if (quarantine == QUARANTINE_STATE_PURGATORY) { 175 /* 176 * The previous time this destructor was called, we set the key 177 * to QUARANTINE_STATE_PURGATORY so that other destructors 178 * wouldn't cause re-creation of the quarantine. This time, do 179 * nothing, so that the destructor will not be called again. 180 */ 181 } else if (quarantine != NULL) { 182 quarantine_drain(quarantine, 0); 183 idalloc(quarantine); 184 quarantine = QUARANTINE_STATE_PURGATORY; 185 quarantine_tsd_set(&quarantine); 186 } 187 } 188 189 bool 190 quarantine_boot(void) 191 { 192 193 cassert(config_fill); 194 195 if (quarantine_tsd_boot()) 196 return (true); 197 198 return (false); 199 } 200