1 /******************************************************************************/
2 #ifdef JEMALLOC_H_TYPES
3
4 typedef struct tcache_bin_info_s tcache_bin_info_t;
5 typedef struct tcache_bin_s tcache_bin_t;
6 typedef struct tcache_s tcache_t;
7 typedef struct tcaches_s tcaches_t;
8
9 /*
10 * tcache pointers close to NULL are used to encode state information that is
11 * used for two purposes: preventing thread caching on a per thread basis and
12 * cleaning up during thread shutdown.
13 */
14 #define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1)
15 #define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2)
16 #define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
17 #define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
18
19 /*
20 * Absolute minimum number of cache slots for each small bin.
21 */
22 #define TCACHE_NSLOTS_SMALL_MIN 20
23
24 /*
25 * Absolute maximum number of cache slots for each small bin in the thread
26 * cache. This is an additional constraint beyond that imposed as: twice the
27 * number of regions per run for this size class.
28 *
29 * This constant must be an even number.
30 */
31 #define TCACHE_NSLOTS_SMALL_MAX 200
32
33 /* Number of cache slots for large size classes. */
34 #define TCACHE_NSLOTS_LARGE 20
35
36 /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
37 #define LG_TCACHE_MAXCLASS_DEFAULT 15
38
39 /*
40 * TCACHE_GC_SWEEP is the approximate number of allocation events between
41 * full GC sweeps. Integer rounding may cause the actual number to be
42 * slightly higher, since GC is performed incrementally.
43 */
44 #define TCACHE_GC_SWEEP 8192
45
46 /* Number of tcache allocation/deallocation events between incremental GCs. */
47 #define TCACHE_GC_INCR \
48 ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
49
50 #endif /* JEMALLOC_H_TYPES */
51 /******************************************************************************/
52 #ifdef JEMALLOC_H_STRUCTS
53
54 typedef enum {
55 tcache_enabled_false = 0, /* Enable cast to/from bool. */
56 tcache_enabled_true = 1,
57 tcache_enabled_default = 2
58 } tcache_enabled_t;
59
60 /*
61 * Read-only information associated with each element of tcache_t's tbins array
62 * is stored separately, mainly to reduce memory usage.
63 */
64 struct tcache_bin_info_s {
65 unsigned ncached_max; /* Upper limit on ncached. */
66 };
67
68 struct tcache_bin_s {
69 tcache_bin_stats_t tstats;
70 int low_water; /* Min # cached since last GC. */
71 unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */
72 unsigned ncached; /* # of cached objects. */
73 void **avail; /* Stack of available objects. */
74 };
75
76 struct tcache_s {
77 ql_elm(tcache_t) link; /* Used for aggregating stats. */
78 uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */
79 unsigned ev_cnt; /* Event count since incremental GC. */
80 szind_t next_gc_bin; /* Next bin to GC. */
81 tcache_bin_t tbins[1]; /* Dynamically sized. */
82 /*
83 * The pointer stacks associated with tbins follow as a contiguous
84 * array. During tcache initialization, the avail pointer in each
85 * element of tbins is initialized to point to the proper offset within
86 * this array.
87 */
88 };
89
90 /* Linkage for list of available (previously used) explicit tcache IDs. */
91 struct tcaches_s {
92 union {
93 tcache_t *tcache;
94 tcaches_t *next;
95 };
96 };
97
98 #endif /* JEMALLOC_H_STRUCTS */
99 /******************************************************************************/
100 #ifdef JEMALLOC_H_EXTERNS
101
102 extern bool opt_tcache;
103 extern ssize_t opt_lg_tcache_max;
104
105 extern tcache_bin_info_t *tcache_bin_info;
106
107 /*
108 * Number of tcache bins. There are NBINS small-object bins, plus 0 or more
109 * large-object bins.
110 */
111 extern size_t nhbins;
112
113 /* Maximum cached size class. */
114 extern size_t tcache_maxclass;
115
116 /*
117 * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
118 * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
119 * completely disjoint from this data structure. tcaches starts off as a sparse
120 * array, so it has no physical memory footprint until individual pages are
121 * touched. This allows the entire array to be allocated the first time an
122 * explicit tcache is created without a disproportionate impact on memory usage.
123 */
124 extern tcaches_t *tcaches;
125
126 size_t tcache_salloc(const void *ptr);
127 void tcache_event_hard(tsd_t *tsd, tcache_t *tcache);
128 void *tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
129 tcache_bin_t *tbin, szind_t binind);
130 void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
131 szind_t binind, unsigned rem);
132 void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
133 unsigned rem, tcache_t *tcache);
134 void tcache_arena_associate(tcache_t *tcache, arena_t *arena);
135 void tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena,
136 arena_t *newarena);
137 void tcache_arena_dissociate(tcache_t *tcache, arena_t *arena);
138 tcache_t *tcache_get_hard(tsd_t *tsd);
139 tcache_t *tcache_create(tsd_t *tsd, arena_t *arena);
140 void tcache_cleanup(tsd_t *tsd);
141 void tcache_enabled_cleanup(tsd_t *tsd);
142 void tcache_stats_merge(tcache_t *tcache, arena_t *arena);
143 bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
144 void tcaches_flush(tsd_t *tsd, unsigned ind);
145 void tcaches_destroy(tsd_t *tsd, unsigned ind);
146 bool tcache_boot(void);
147
148 #endif /* JEMALLOC_H_EXTERNS */
149 /******************************************************************************/
150 #ifdef JEMALLOC_H_INLINES
151
152 #ifndef JEMALLOC_ENABLE_INLINE
153 void tcache_event(tsd_t *tsd, tcache_t *tcache);
154 void tcache_flush(void);
155 bool tcache_enabled_get(void);
156 tcache_t *tcache_get(tsd_t *tsd, bool create);
157 void tcache_enabled_set(bool enabled);
158 void *tcache_alloc_easy(tcache_bin_t *tbin);
159 void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
160 size_t size, bool zero);
161 void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
162 size_t size, bool zero);
163 void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
164 szind_t binind);
165 void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
166 size_t size);
167 tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
168 #endif
169
170 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
171 JEMALLOC_INLINE void
tcache_flush(void)172 tcache_flush(void)
173 {
174 tsd_t *tsd;
175
176 cassert(config_tcache);
177
178 tsd = tsd_fetch();
179 tcache_cleanup(tsd);
180 }
181
182 JEMALLOC_INLINE bool
tcache_enabled_get(void)183 tcache_enabled_get(void)
184 {
185 tsd_t *tsd;
186 tcache_enabled_t tcache_enabled;
187
188 cassert(config_tcache);
189
190 tsd = tsd_fetch();
191 tcache_enabled = tsd_tcache_enabled_get(tsd);
192 if (tcache_enabled == tcache_enabled_default) {
193 tcache_enabled = (tcache_enabled_t)opt_tcache;
194 tsd_tcache_enabled_set(tsd, tcache_enabled);
195 }
196
197 return ((bool)tcache_enabled);
198 }
199
200 JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)201 tcache_enabled_set(bool enabled)
202 {
203 tsd_t *tsd;
204 tcache_enabled_t tcache_enabled;
205
206 cassert(config_tcache);
207
208 tsd = tsd_fetch();
209
210 tcache_enabled = (tcache_enabled_t)enabled;
211 tsd_tcache_enabled_set(tsd, tcache_enabled);
212
213 if (!enabled)
214 tcache_cleanup(tsd);
215 }
216
217 JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(tsd_t * tsd,bool create)218 tcache_get(tsd_t *tsd, bool create)
219 {
220 tcache_t *tcache;
221
222 if (!config_tcache)
223 return (NULL);
224
225 tcache = tsd_tcache_get(tsd);
226 if (!create)
227 return (tcache);
228 if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
229 tcache = tcache_get_hard(tsd);
230 tsd_tcache_set(tsd, tcache);
231 }
232
233 return (tcache);
234 }
235
236 JEMALLOC_ALWAYS_INLINE void
tcache_event(tsd_t * tsd,tcache_t * tcache)237 tcache_event(tsd_t *tsd, tcache_t *tcache)
238 {
239
240 if (TCACHE_GC_INCR == 0)
241 return;
242
243 tcache->ev_cnt++;
244 assert(tcache->ev_cnt <= TCACHE_GC_INCR);
245 if (unlikely(tcache->ev_cnt == TCACHE_GC_INCR))
246 tcache_event_hard(tsd, tcache);
247 }
248
249 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_easy(tcache_bin_t * tbin)250 tcache_alloc_easy(tcache_bin_t *tbin)
251 {
252 void *ret;
253
254 if (unlikely(tbin->ncached == 0)) {
255 tbin->low_water = -1;
256 return (NULL);
257 }
258 tbin->ncached--;
259 if (unlikely((int)tbin->ncached < tbin->low_water))
260 tbin->low_water = tbin->ncached;
261 ret = tbin->avail[tbin->ncached];
262 return (ret);
263 }
264
265 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t * tsd,arena_t * arena,tcache_t * tcache,size_t size,bool zero)266 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
267 bool zero)
268 {
269 void *ret;
270 szind_t binind;
271 size_t usize;
272 tcache_bin_t *tbin;
273
274 binind = size2index(size);
275 assert(binind < NBINS);
276 tbin = &tcache->tbins[binind];
277 usize = index2size(binind);
278 ret = tcache_alloc_easy(tbin);
279 if (unlikely(ret == NULL)) {
280 ret = tcache_alloc_small_hard(tsd, arena, tcache, tbin, binind);
281 if (ret == NULL)
282 return (NULL);
283 }
284 assert(tcache_salloc(ret) == usize);
285
286 if (likely(!zero)) {
287 if (config_fill) {
288 if (unlikely(opt_junk_alloc)) {
289 arena_alloc_junk_small(ret,
290 &arena_bin_info[binind], false);
291 } else if (unlikely(opt_zero))
292 memset(ret, 0, usize);
293 }
294 } else {
295 if (config_fill && unlikely(opt_junk_alloc)) {
296 arena_alloc_junk_small(ret, &arena_bin_info[binind],
297 true);
298 }
299 memset(ret, 0, usize);
300 }
301
302 if (config_stats)
303 tbin->tstats.nrequests++;
304 if (config_prof)
305 tcache->prof_accumbytes += usize;
306 tcache_event(tsd, tcache);
307 return (ret);
308 }
309
310 JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_large(tsd_t * tsd,arena_t * arena,tcache_t * tcache,size_t size,bool zero)311 tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
312 bool zero)
313 {
314 void *ret;
315 szind_t binind;
316 size_t usize;
317 tcache_bin_t *tbin;
318
319 binind = size2index(size);
320 usize = index2size(binind);
321 assert(usize <= tcache_maxclass);
322 assert(binind < nhbins);
323 tbin = &tcache->tbins[binind];
324 ret = tcache_alloc_easy(tbin);
325 if (unlikely(ret == NULL)) {
326 /*
327 * Only allocate one large object at a time, because it's quite
328 * expensive to create one and not use it.
329 */
330 ret = arena_malloc_large(arena, usize, zero);
331 if (ret == NULL)
332 return (NULL);
333 } else {
334 if (config_prof && usize == LARGE_MINCLASS) {
335 arena_chunk_t *chunk =
336 (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
337 size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
338 LG_PAGE);
339 arena_mapbits_large_binind_set(chunk, pageind,
340 BININD_INVALID);
341 }
342 if (likely(!zero)) {
343 if (config_fill) {
344 if (unlikely(opt_junk_alloc))
345 memset(ret, 0xa5, usize);
346 else if (unlikely(opt_zero))
347 memset(ret, 0, usize);
348 }
349 } else
350 memset(ret, 0, usize);
351
352 if (config_stats)
353 tbin->tstats.nrequests++;
354 if (config_prof)
355 tcache->prof_accumbytes += usize;
356 }
357
358 tcache_event(tsd, tcache);
359 return (ret);
360 }
361
362 JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_small(tsd_t * tsd,tcache_t * tcache,void * ptr,szind_t binind)363 tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind)
364 {
365 tcache_bin_t *tbin;
366 tcache_bin_info_t *tbin_info;
367
368 assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
369
370 if (config_fill && unlikely(opt_junk_free))
371 arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
372
373 tbin = &tcache->tbins[binind];
374 tbin_info = &tcache_bin_info[binind];
375 if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
376 tcache_bin_flush_small(tsd, tcache, tbin, binind,
377 (tbin_info->ncached_max >> 1));
378 }
379 assert(tbin->ncached < tbin_info->ncached_max);
380 tbin->avail[tbin->ncached] = ptr;
381 tbin->ncached++;
382
383 tcache_event(tsd, tcache);
384 }
385
386 JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_large(tsd_t * tsd,tcache_t * tcache,void * ptr,size_t size)387 tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size)
388 {
389 szind_t binind;
390 tcache_bin_t *tbin;
391 tcache_bin_info_t *tbin_info;
392
393 assert((size & PAGE_MASK) == 0);
394 assert(tcache_salloc(ptr) > SMALL_MAXCLASS);
395 assert(tcache_salloc(ptr) <= tcache_maxclass);
396
397 binind = size2index(size);
398
399 if (config_fill && unlikely(opt_junk_free))
400 arena_dalloc_junk_large(ptr, size);
401
402 tbin = &tcache->tbins[binind];
403 tbin_info = &tcache_bin_info[binind];
404 if (unlikely(tbin->ncached == tbin_info->ncached_max)) {
405 tcache_bin_flush_large(tsd, tbin, binind,
406 (tbin_info->ncached_max >> 1), tcache);
407 }
408 assert(tbin->ncached < tbin_info->ncached_max);
409 tbin->avail[tbin->ncached] = ptr;
410 tbin->ncached++;
411
412 tcache_event(tsd, tcache);
413 }
414
415 JEMALLOC_ALWAYS_INLINE tcache_t *
tcaches_get(tsd_t * tsd,unsigned ind)416 tcaches_get(tsd_t *tsd, unsigned ind)
417 {
418 tcaches_t *elm = &tcaches[ind];
419 if (unlikely(elm->tcache == NULL))
420 elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
421 return (elm->tcache);
422 }
423 #endif
424
425 #endif /* JEMALLOC_H_INLINES */
426 /******************************************************************************/
427