52 static void	chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
59 chunk_hooks_get_locked(arena_t *arena)  in chunk_hooks_get_locked()  argument
62 	return (arena->chunk_hooks);  in chunk_hooks_get_locked()
66 chunk_hooks_get(arena_t *arena)  in chunk_hooks_get()  argument
70 	malloc_mutex_lock(&arena->chunks_mtx);  in chunk_hooks_get()
71 	chunk_hooks = chunk_hooks_get_locked(arena);  in chunk_hooks_get()
72 	malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_hooks_get()
78 chunk_hooks_set(arena_t *arena, const chunk_hooks_t *chunk_hooks)  in chunk_hooks_set()  argument
82 	malloc_mutex_lock(&arena->chunks_mtx);  in chunk_hooks_set()
83 	old_chunk_hooks = arena->chunk_hooks;  in chunk_hooks_set()
96 	u.n = &arena->chunk_hooks.n;					\  in chunk_hooks_set()
107 	malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_hooks_set()
113 chunk_hooks_assure_initialized_impl(arena_t *arena, chunk_hooks_t *chunk_hooks,  in chunk_hooks_assure_initialized_impl()  argument
121 		*chunk_hooks = locked ? chunk_hooks_get_locked(arena) :  in chunk_hooks_assure_initialized_impl()
122 		    chunk_hooks_get(arena);  in chunk_hooks_assure_initialized_impl()
127 chunk_hooks_assure_initialized_locked(arena_t *arena,  in chunk_hooks_assure_initialized_locked()  argument
131 	chunk_hooks_assure_initialized_impl(arena, chunk_hooks, true);  in chunk_hooks_assure_initialized_locked()
135 chunk_hooks_assure_initialized(arena_t *arena, chunk_hooks_t *chunk_hooks)  in chunk_hooks_assure_initialized()  argument
138 	chunk_hooks_assure_initialized_impl(arena, chunk_hooks, false);  in chunk_hooks_assure_initialized()
188 chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad,  in chunk_first_best_fit()  argument
195 	extent_node_init(&key, arena, NULL, size, false, false);  in chunk_first_best_fit()
200 chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,  in chunk_recycle()  argument
222 	malloc_mutex_lock(&arena->chunks_mtx);  in chunk_recycle()
223 	chunk_hooks_assure_initialized_locked(arena, chunk_hooks);  in chunk_recycle()
226 		extent_node_init(&key, arena, new_addr, alloc_size, false,  in chunk_recycle()
230 		node = chunk_first_best_fit(arena, chunks_szad, chunks_ad,  in chunk_recycle()
235 		malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
253 	    extent_node_size_get(node), leadsize, size, false, arena->ind)) {  in chunk_recycle()
254 		malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
260 	arena_chunk_cache_maybe_remove(arena, node, cache);  in chunk_recycle()
266 		arena_chunk_cache_maybe_insert(arena, node, cache);  in chunk_recycle()
272 		    trailsize, false, arena->ind)) {  in chunk_recycle()
274 				arena_node_dalloc(arena, node);  in chunk_recycle()
275 			malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
276 			chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad,  in chunk_recycle()
282 			node = arena_node_alloc(arena);  in chunk_recycle()
284 				malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
285 				chunk_record(arena, chunk_hooks, chunks_szad,  in chunk_recycle()
291 		extent_node_init(node, arena, (void *)((uintptr_t)(ret) + size),  in chunk_recycle()
295 		arena_chunk_cache_maybe_insert(arena, node, cache);  in chunk_recycle()
298 	if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) {  in chunk_recycle()
299 		malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
300 		chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad, cache,  in chunk_recycle()
304 	malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_recycle()
308 		arena_node_dalloc(arena, node);  in chunk_recycle()
331 chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment,  in chunk_alloc_core()  argument
343 	if ((ret = chunk_recycle(arena, &chunk_hooks,  in chunk_alloc_core()
344 	    &arena->chunks_szad_retained, &arena->chunks_ad_retained, false,  in chunk_alloc_core()
350 	    chunk_alloc_dss(arena, new_addr, size, alignment, zero, commit)) !=  in chunk_alloc_core()
362 	    chunk_alloc_dss(arena, new_addr, size, alignment, zero, commit)) !=  in chunk_alloc_core()
393 chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,  in chunk_alloc_cache()  argument
405 	ret = chunk_recycle(arena, chunk_hooks, &arena->chunks_szad_cached,  in chunk_alloc_cache()
406 	    &arena->chunks_ad_cached, true, new_addr, size, alignment, zero,  in chunk_alloc_cache()
419 	arena_t *arena;  in chunk_arena_get()  local
422 	arena = (arena_ind == 0) ? a0get() : arena_get(tsd_fetch(), arena_ind,  in chunk_arena_get()
428 	assert(arena != NULL);  in chunk_arena_get()
429 	return (arena);  in chunk_arena_get()
437 	arena_t *arena;  in chunk_alloc_default()  local
439 	arena = chunk_arena_get(arena_ind);  in chunk_alloc_default()
440 	ret = chunk_alloc_core(arena, new_addr, size, alignment, zero,  in chunk_alloc_default()
441 	    commit, arena->dss_prec);  in chunk_alloc_default()
451 chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,  in chunk_alloc_wrapper()  argument
456 	chunk_hooks_assure_initialized(arena, chunk_hooks);  in chunk_alloc_wrapper()
458 	    arena->ind);  in chunk_alloc_wrapper()
467 chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,  in chunk_record()  argument
479 	malloc_mutex_lock(&arena->chunks_mtx);  in chunk_record()
480 	chunk_hooks_assure_initialized_locked(arena, chunk_hooks);  in chunk_record()
481 	extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0,  in chunk_record()
489 	    arena->ind)) {  in chunk_record()
496 		arena_chunk_cache_maybe_remove(arena, node, cache);  in chunk_record()
502 		arena_chunk_cache_maybe_insert(arena, node, cache);  in chunk_record()
505 		node = arena_node_alloc(arena);  in chunk_record()
514 				chunk_purge_wrapper(arena, chunk_hooks, chunk,  in chunk_record()
519 		extent_node_init(node, arena, chunk, size, !unzeroed,  in chunk_record()
523 		arena_chunk_cache_maybe_insert(arena, node, cache);  in chunk_record()
532 	    extent_node_size_get(prev), chunk, size, false, arena->ind)) {  in chunk_record()
540 		arena_chunk_cache_maybe_remove(arena, prev, cache);  in chunk_record()
542 		arena_chunk_cache_maybe_remove(arena, node, cache);  in chunk_record()
549 		arena_chunk_cache_maybe_insert(arena, node, cache);  in chunk_record()
551 		arena_node_dalloc(arena, prev);  in chunk_record()
555 	malloc_mutex_unlock(&arena->chunks_mtx);  in chunk_record()
559 chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,  in chunk_dalloc_cache()  argument
568 	chunk_record(arena, chunk_hooks, &arena->chunks_szad_cached,  in chunk_dalloc_cache()
569 	    &arena->chunks_ad_cached, true, chunk, size, false, committed);  in chunk_dalloc_cache()
570 	arena_maybe_purge(arena);  in chunk_dalloc_cache()
574 chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,  in chunk_dalloc_arena()  argument
583 	chunk_hooks_assure_initialized(arena, chunk_hooks);  in chunk_dalloc_arena()
585 	if (!chunk_hooks->dalloc(chunk, size, committed, arena->ind))  in chunk_dalloc_arena()
590 		    arena->ind);  in chunk_dalloc_arena()
593 	    arena->ind);  in chunk_dalloc_arena()
594 	chunk_record(arena, chunk_hooks, &arena->chunks_szad_retained,  in chunk_dalloc_arena()
595 	    &arena->chunks_ad_retained, false, chunk, size, zeroed, committed);  in chunk_dalloc_arena()
609 chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,  in chunk_dalloc_wrapper()  argument
613 	chunk_hooks_assure_initialized(arena, chunk_hooks);  in chunk_dalloc_wrapper()
614 	chunk_hooks->dalloc(chunk, size, committed, arena->ind);  in chunk_dalloc_wrapper()
638 chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length)  in chunk_purge_arena()  argument
661 chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,  in chunk_purge_wrapper()  argument
665 	chunk_hooks_assure_initialized(arena, chunk_hooks);  in chunk_purge_wrapper()
666 	return (chunk_hooks->purge(chunk, size, offset, length, arena->ind));  in chunk_purge_wrapper()