444 arena_bind_locked(tsd_t *tsd, unsigned ind)  in arena_bind_locked()  argument
451 	if (tsd_nominal(tsd))  in arena_bind_locked()
452 		tsd_arena_set(tsd, arena);  in arena_bind_locked()
456 arena_bind(tsd_t *tsd, unsigned ind)  in arena_bind()  argument
460 	arena_bind_locked(tsd, ind);  in arena_bind()
465 arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind)  in arena_migrate()  argument
475 	tsd_arena_set(tsd, newarena);  in arena_migrate()
490 arena_unbind(tsd_t *tsd, unsigned ind)  in arena_unbind()  argument
498 	tsd_arena_set(tsd, NULL);  in arena_unbind()
502 arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing)  in arena_get_hard()  argument
505 	arena_t **arenas_cache = tsd_arenas_cache_get(tsd);  in arena_get_hard()
506 	unsigned narenas_cache = tsd_narenas_cache_get(tsd);  in arena_get_hard()
514 		tsd_arenas_cache_set(tsd, arenas_cache);  in arena_get_hard()
515 		tsd_narenas_cache_set(tsd, narenas_cache);  in arena_get_hard()
520 		bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd);  in arena_get_hard()
524 		if (tsd_nominal(tsd) && !*arenas_cache_bypassp) {  in arena_get_hard()
545 		assert(tsd_nominal(tsd) && !*arenas_cache_bypassp);  in arena_get_hard()
546 		tsd_arenas_cache_set(tsd, arenas_cache);  in arena_get_hard()
547 		tsd_narenas_cache_set(tsd, narenas_cache);  in arena_get_hard()
574 arena_choose_hard(tsd_t *tsd)  in arena_choose_hard()  argument
624 		arena_bind_locked(tsd, choose);  in arena_choose_hard()
628 		arena_bind(tsd, 0);  in arena_choose_hard()
635 thread_allocated_cleanup(tsd_t *tsd)  in thread_allocated_cleanup()  argument
642 thread_deallocated_cleanup(tsd_t *tsd)  in thread_deallocated_cleanup()  argument
649 arena_cleanup(tsd_t *tsd)  in arena_cleanup()  argument
653 	arena = tsd_arena_get(tsd);  in arena_cleanup()
655 		arena_unbind(tsd, arena->ind);  in arena_cleanup()
659 arenas_cache_cleanup(tsd_t *tsd)  in arenas_cache_cleanup()  argument
663 	arenas_cache = tsd_arenas_cache_get(tsd);  in arenas_cache_cleanup()
665 		tsd_arenas_cache_set(tsd, NULL);  in arenas_cache_cleanup()
671 narenas_cache_cleanup(tsd_t *tsd)  in narenas_cache_cleanup()  argument
678 arenas_cache_bypass_cleanup(tsd_t *tsd)  in arenas_cache_bypass_cleanup()  argument
1358 imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)  in imalloc_prof_sample()  argument
1365 		p = imalloc(tsd, LARGE_MINCLASS);  in imalloc_prof_sample()
1370 		p = imalloc(tsd, usize);  in imalloc_prof_sample()
1376 imalloc_prof(tsd_t *tsd, size_t usize)  in imalloc_prof()  argument
1381 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);  in imalloc_prof()
1383 		p = imalloc_prof_sample(tsd, usize, tctx);  in imalloc_prof()
1385 		p = imalloc(tsd, usize);  in imalloc_prof()
1387 		prof_alloc_rollback(tsd, tctx, true);  in imalloc_prof()
1396 imalloc_body(size_t size, tsd_t **tsd, size_t *usize)  in imalloc_body()  argument
1401 	*tsd = tsd_fetch();  in imalloc_body()
1407 		return (imalloc_prof(*tsd, *usize));  in imalloc_body()
1412 	return (imalloc(*tsd, size));  in imalloc_body()
1421 	tsd_t *tsd;  in JEMALLOC_ATTR()  local
1427 	ret = imalloc_body(size, &tsd, &usize);  in JEMALLOC_ATTR()
1438 		*tsd_thread_allocatedp_get(tsd) += usize;  in JEMALLOC_ATTR()
1446 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,  in imemalign_prof_sample()  argument
1455 		p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);  in imemalign_prof_sample()
1460 		p = ipalloc(tsd, usize, alignment, false);  in imemalign_prof_sample()
1466 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)  in imemalign_prof()  argument
1471 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);  in imemalign_prof()
1473 		p = imemalign_prof_sample(tsd, alignment, usize, tctx);  in imemalign_prof()
1475 		p = ipalloc(tsd, usize, alignment, false);  in imemalign_prof()
1477 		prof_alloc_rollback(tsd, tctx, true);  in imemalign_prof()
1490 	tsd_t *tsd;  in imemalign()  local
1500 	tsd = tsd_fetch();  in imemalign()
1524 		result = imemalign_prof(tsd, alignment, usize);  in imemalign()
1526 		result = ipalloc(tsd, usize, alignment, false);  in imemalign()
1536 		*tsd_thread_allocatedp_get(tsd) += usize;  in imemalign()
1579 icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)  in icalloc_prof_sample()  argument
1586 		p = icalloc(tsd, LARGE_MINCLASS);  in icalloc_prof_sample()
1591 		p = icalloc(tsd, usize);  in icalloc_prof_sample()
1597 icalloc_prof(tsd_t *tsd, size_t usize)  in icalloc_prof()  argument
1602 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);  in icalloc_prof()
1604 		p = icalloc_prof_sample(tsd, usize, tctx);  in icalloc_prof()
1606 		p = icalloc(tsd, usize);  in icalloc_prof()
1608 		prof_alloc_rollback(tsd, tctx, true);  in icalloc_prof()
1622 	tsd_t *tsd;  in JEMALLOC_ATTR()  local
1631 	tsd = tsd_fetch();  in JEMALLOC_ATTR()
1659 		ret = icalloc_prof(tsd, usize);  in JEMALLOC_ATTR()
1663 		ret = icalloc(tsd, num_size);  in JEMALLOC_ATTR()
1677 		*tsd_thread_allocatedp_get(tsd) += usize;  in JEMALLOC_ATTR()
1685 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,  in irealloc_prof_sample()  argument
1693 		p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);  in irealloc_prof_sample()
1698 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);  in irealloc_prof_sample()
1704 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)  in irealloc_prof()  argument
1712 	tctx = prof_alloc_prep(tsd, usize, prof_active, true);  in irealloc_prof()
1714 		p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);  in irealloc_prof()
1716 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);  in irealloc_prof()
1718 		prof_alloc_rollback(tsd, tctx, true);  in irealloc_prof()
1721 	prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,  in irealloc_prof()
1728 ifree(tsd_t *tsd, void *ptr, tcache_t *tcache)  in ifree()  argument
1738 		prof_free(tsd, ptr, usize);  in ifree()
1742 		*tsd_thread_deallocatedp_get(tsd) += usize;  in ifree()
1745 	iqalloc(tsd, ptr, tcache);  in ifree()
1750 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache)  in isfree()  argument
1758 		prof_free(tsd, ptr, usize);  in isfree()
1760 		*tsd_thread_deallocatedp_get(tsd) += usize;  in isfree()
1763 	isqalloc(tsd, ptr, usize, tcache);  in isfree()
1773 	tsd_t *tsd JEMALLOC_CC_SILENCE_INIT(NULL);  in je_realloc()
1782 			tsd = tsd_fetch();  in je_realloc()
1783 			ifree(tsd, ptr, tcache_get(tsd, false));  in je_realloc()
1792 		tsd = tsd_fetch();  in je_realloc()
1800 			ret = unlikely(usize == 0) ? NULL : irealloc_prof(tsd,  in je_realloc()
1806 			ret = iralloc(tsd, ptr, old_usize, size, 0, false);  in je_realloc()
1810 		ret = imalloc_body(size, &tsd, &usize);  in je_realloc()
1823 		*tsd_thread_allocatedp_get(tsd) += usize;  in je_realloc()
1824 		*tsd_thread_deallocatedp_get(tsd) += old_usize;  in je_realloc()
1838 		tsd_t *tsd = tsd_fetch();  in je_free()  local
1839 		ifree(tsd, ptr, tcache_get(tsd, false));  in je_free()
1915 imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize,  in imallocx_flags_decode_hard()  argument
1932 			*tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));  in imallocx_flags_decode_hard()
1934 		*tcache = tcache_get(tsd, true);  in imallocx_flags_decode_hard()
1937 		*arena = arena_get(tsd, arena_ind, true, true);  in imallocx_flags_decode_hard()
1946 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,  in imallocx_flags_decode()  argument
1955 		*tcache = tcache_get(tsd, true);  in imallocx_flags_decode()
1959 		return (imallocx_flags_decode_hard(tsd, size, flags, usize,  in imallocx_flags_decode()
1965 imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero,  in imallocx_flags()  argument
1970 		return (ipalloct(tsd, usize, alignment, zero, tcache, arena));  in imallocx_flags()
1972 		return (icalloct(tsd, usize, tcache, arena));  in imallocx_flags()
1973 	return (imalloct(tsd, usize, tcache, arena));  in imallocx_flags()
1977 imallocx_prof_sample(tsd_t *tsd, size_t usize, size_t alignment, bool zero,  in imallocx_prof_sample()  argument
1985 		p = imallocx_flags(tsd, LARGE_MINCLASS, alignment, zero, tcache,  in imallocx_prof_sample()
1991 		p = imallocx_flags(tsd, usize, alignment, zero, tcache, arena);  in imallocx_prof_sample()
1997 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)  in imallocx_prof()  argument
2006 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,  in imallocx_prof()
2009 	tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);  in imallocx_prof()
2011 		p = imallocx_flags(tsd, *usize, alignment, zero, tcache, arena);  in imallocx_prof()
2013 		p = imallocx_prof_sample(tsd, *usize, alignment, zero, tcache,  in imallocx_prof()
2018 		prof_alloc_rollback(tsd, tctx, true);  in imallocx_prof()
2028 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)  in imallocx_no_prof()  argument
2039 		return (imalloc(tsd, size));  in imallocx_no_prof()
2042 	if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize,  in imallocx_no_prof()
2045 	p = imallocx_flags(tsd, *usize, alignment, zero, tcache, arena);  in imallocx_no_prof()
2055 	tsd_t *tsd;  in JEMALLOC_ATTR()  local
2063 	tsd = tsd_fetch();  in JEMALLOC_ATTR()
2066 		p = imallocx_prof(tsd, size, flags, &usize);  in JEMALLOC_ATTR()
2068 		p = imallocx_no_prof(tsd, size, flags, &usize);  in JEMALLOC_ATTR()
2074 		*tsd_thread_allocatedp_get(tsd) += usize;  in JEMALLOC_ATTR()
2089 irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,  in irallocx_prof_sample()  argument
2098 		p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment,  in irallocx_prof_sample()
2104 		p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,  in irallocx_prof_sample()
2112 irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,  in irallocx_prof()  argument
2122 	tctx = prof_alloc_prep(tsd, *usize, prof_active, true);  in irallocx_prof()
2124 		p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,  in irallocx_prof()
2127 		p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,  in irallocx_prof()
2131 		prof_alloc_rollback(tsd, tctx, true);  in irallocx_prof()
2146 	prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr,  in irallocx_prof()
2158 	tsd_t *tsd;  in je_rallocx()  local
2171 	tsd = tsd_fetch();  in je_rallocx()
2175 		arena = arena_get(tsd, arena_ind, true, true);  in je_rallocx()
2185 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));  in je_rallocx()
2187 		tcache = tcache_get(tsd, true);  in je_rallocx()
2196 		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,  in je_rallocx()
2201 		p = iralloct(tsd, ptr, old_usize, size, alignment, zero,  in je_rallocx()
2211 		*tsd_thread_allocatedp_get(tsd) += usize;  in je_rallocx()
2212 		*tsd_thread_deallocatedp_get(tsd) += old_usize;  in je_rallocx()
2254 ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,  in ixallocx_prof()  argument
2272 	tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);  in ixallocx_prof()
2281 		prof_alloc_rollback(tsd, tctx, false);  in ixallocx_prof()
2284 	prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,  in ixallocx_prof()
2293 	tsd_t *tsd;  in je_xallocx()  local
2304 	tsd = tsd_fetch();  in je_xallocx()
2322 		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,  in je_xallocx()
2332 		*tsd_thread_allocatedp_get(tsd) += usize;  in je_xallocx()
2333 		*tsd_thread_deallocatedp_get(tsd) += old_usize;  in je_xallocx()
2362 	tsd_t *tsd;  in je_dallocx()  local
2368 	tsd = tsd_fetch();  in je_dallocx()
2373 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));  in je_dallocx()
2375 		tcache = tcache_get(tsd, false);  in je_dallocx()
2397 	tsd_t *tsd;  in je_sdallocx()  local
2406 	tsd = tsd_fetch();  in je_sdallocx()
2411 			tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));  in je_sdallocx()
2413 		tcache = tcache_get(tsd, false);  in je_sdallocx()
2416 	isfree(tsd, ptr, usize, tcache);  in je_sdallocx()