27 static void	arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty,
30     arena_run_t *run, arena_bin_t *bin);
32     arena_run_t *run, arena_bin_t *bin);
292 arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)  in arena_run_reg_alloc()  argument
299 	assert(run->nfree > 0);  in arena_run_reg_alloc()
300 	assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info));  in arena_run_reg_alloc()
302 	regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info);  in arena_run_reg_alloc()
303 	miscelm = arena_run_to_miscelm(run);  in arena_run_reg_alloc()
307 	run->nfree--;  in arena_run_reg_alloc()
312 arena_run_reg_dalloc(arena_run_t *run, void *ptr)  in arena_run_reg_dalloc()  argument
314 	arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_run_reg_dalloc()
319 	unsigned regind = arena_run_regind(run, bin_info, ptr);  in arena_run_reg_dalloc()
321 	assert(run->nfree < bin_info->nregs);  in arena_run_reg_dalloc()
324 	    ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +  in arena_run_reg_dalloc()
328 	    (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) +  in arena_run_reg_dalloc()
331 	assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind));  in arena_run_reg_dalloc()
333 	bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind);  in arena_run_reg_dalloc()
334 	run->nfree++;  in arena_run_reg_dalloc()
423 arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,  in arena_run_split_large_helper()  argument
431 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_run_split_large_helper()
432 	miscelm = arena_run_to_miscelm(run);  in arena_run_split_large_helper()
496 arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)  in arena_run_split_large()  argument
499 	return (arena_run_split_large_helper(arena, run, size, true, zero));  in arena_run_split_large()
503 arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)  in arena_run_init_large()  argument
506 	return (arena_run_split_large_helper(arena, run, size, false, zero));  in arena_run_init_large()
510 arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,  in arena_run_split_small()  argument
519 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_run_split_small()
520 	miscelm = arena_run_to_miscelm(run);  in arena_run_split_small()
1080 	return (&miscelm->run);  in arena_run_first_best_fit()
1086 	arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));  in arena_run_alloc_large_helper()  local
1087 	if (run != NULL) {  in arena_run_alloc_large_helper()
1088 		if (arena_run_split_large(arena, run, size, zero))  in arena_run_alloc_large_helper()
1089 			run = NULL;  in arena_run_alloc_large_helper()
1091 	return (run);  in arena_run_alloc_large_helper()
1098 	arena_run_t *run;  in arena_run_alloc_large()  local
1104 	run = arena_run_alloc_large_helper(arena, size, zero);  in arena_run_alloc_large()
1105 	if (run != NULL)  in arena_run_alloc_large()
1106 		return (run);  in arena_run_alloc_large()
1113 		run = &arena_miscelm_get(chunk, map_bias)->run;  in arena_run_alloc_large()
1114 		if (arena_run_split_large(arena, run, size, zero))  in arena_run_alloc_large()
1115 			run = NULL;  in arena_run_alloc_large()
1116 		return (run);  in arena_run_alloc_large()
1130 	arena_run_t *run = arena_run_first_best_fit(arena, size);  in arena_run_alloc_small_helper()  local
1131 	if (run != NULL) {  in arena_run_alloc_small_helper()
1132 		if (arena_run_split_small(arena, run, size, binind))  in arena_run_alloc_small_helper()
1133 			run = NULL;  in arena_run_alloc_small_helper()
1135 	return (run);  in arena_run_alloc_small_helper()
1142 	arena_run_t *run;  in arena_run_alloc_small()  local
1149 	run = arena_run_alloc_small_helper(arena, size, binind);  in arena_run_alloc_small()
1150 	if (run != NULL)  in arena_run_alloc_small()
1151 		return (run);  in arena_run_alloc_small()
1158 		run = &arena_miscelm_get(chunk, map_bias)->run;  in arena_run_alloc_small()
1159 		if (arena_run_split_small(arena, run, size, binind))  in arena_run_alloc_small()
1160 			run = NULL;  in arena_run_alloc_small()
1161 		return (run);  in arena_run_alloc_small()
1332 			arena_run_t *run = &miscelm->run;  in arena_stash_dirty()  local
1350 			arena_run_split_large(arena, run, run_size, false);  in arena_stash_dirty()
1495 			arena_run_t *run = &miscelm->run;  in arena_unstash_purged()  local
1497 			arena_run_dalloc(arena, run, false, true, decommitted);  in arena_unstash_purged()
1643 arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,  in arena_run_size_get()  argument
1656 		arena_bin_info_t *bin_info = &arena_bin_info[run->binind];  in arena_run_size_get()
1664 arena_run_decommit(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run)  in arena_run_decommit()  argument
1666 	arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);  in arena_run_decommit()
1669 	size_t length = arena_run_size_get(arena, chunk, run, run_ind);  in arena_run_decommit()
1676 arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,  in arena_run_dalloc()  argument
1683 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_run_dalloc()
1684 	miscelm = arena_run_to_miscelm(run);  in arena_run_dalloc()
1688 	size = arena_run_size_get(arena, chunk, run, run_ind);  in arena_run_dalloc()
1754     arena_run_t *run)  in arena_run_dalloc_decommit()  argument
1756 	bool committed = arena_run_decommit(arena, chunk, run);  in arena_run_dalloc_decommit()
1758 	arena_run_dalloc(arena, run, committed, false, !committed);  in arena_run_dalloc_decommit()
1762 arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,  in arena_run_trim_head()  argument
1765 	arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);  in arena_run_trim_head()
1798 	arena_run_dalloc(arena, run, false, false, (flag_decommitted != 0));  in arena_run_trim_head()
1802 arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,  in arena_run_trim_tail()  argument
1805 	arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);  in arena_run_trim_tail()
1841 	tail_run = &tail_miscelm->run;  in arena_run_trim_tail()
1851 		return (&miscelm->run);  in arena_bin_runs_first()
1857 arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run)  in arena_bin_runs_insert()  argument
1859 	arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);  in arena_bin_runs_insert()
1867 arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run)  in arena_bin_runs_remove()  argument
1869 	arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);  in arena_bin_runs_remove()
1879 	arena_run_t *run = arena_bin_runs_first(bin);  in arena_bin_nonfull_run_tryget()  local
1880 	if (run != NULL) {  in arena_bin_nonfull_run_tryget()
1881 		arena_bin_runs_remove(bin, run);  in arena_bin_nonfull_run_tryget()
1885 	return (run);  in arena_bin_nonfull_run_tryget()
1891 	arena_run_t *run;  in arena_bin_nonfull_run_get()  local
1896 	run = arena_bin_nonfull_run_tryget(bin);  in arena_bin_nonfull_run_get()
1897 	if (run != NULL)  in arena_bin_nonfull_run_get()
1898 		return (run);  in arena_bin_nonfull_run_get()
1908 	run = arena_run_alloc_small(arena, bin_info->run_size, binind);  in arena_bin_nonfull_run_get()
1909 	if (run != NULL) {  in arena_bin_nonfull_run_get()
1911 		run->binind = binind;  in arena_bin_nonfull_run_get()
1912 		run->nfree = bin_info->nregs;  in arena_bin_nonfull_run_get()
1913 		bitmap_init(run->bitmap, &bin_info->bitmap_info);  in arena_bin_nonfull_run_get()
1918 	if (run != NULL) {  in arena_bin_nonfull_run_get()
1923 		return (run);  in arena_bin_nonfull_run_get()
1931 	run = arena_bin_nonfull_run_tryget(bin);  in arena_bin_nonfull_run_get()
1932 	if (run != NULL)  in arena_bin_nonfull_run_get()
1933 		return (run);  in arena_bin_nonfull_run_get()
1944 	arena_run_t *run;  in arena_bin_malloc_hard()  local
1949 	run = arena_bin_nonfull_run_get(arena, bin);  in arena_bin_malloc_hard()
1958 		if (run != NULL) {  in arena_bin_malloc_hard()
1969 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_bin_malloc_hard()
1970 			if (run->nfree == bin_info->nregs)  in arena_bin_malloc_hard()
1971 				arena_dalloc_bin_run(arena, chunk, run, bin);  in arena_bin_malloc_hard()
1973 				arena_bin_lower_run(arena, chunk, run, bin);  in arena_bin_malloc_hard()
1978 	if (run == NULL)  in arena_bin_malloc_hard()
1981 	bin->runcur = run;  in arena_bin_malloc_hard()
2003 		arena_run_t *run;  in arena_tcache_fill_small()  local
2005 		if ((run = bin->runcur) != NULL && run->nfree > 0)  in arena_tcache_fill_small()
2006 			ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]);  in arena_tcache_fill_small()
2152 	arena_run_t *run;  in arena_malloc_small()  local
2161 	if ((run = bin->runcur) != NULL && run->nfree > 0)  in arena_malloc_small()
2162 		ret = arena_run_reg_alloc(run, &arena_bin_info[binind]);  in arena_malloc_small()
2207 	arena_run_t *run;  in arena_malloc_large()  local
2228 	run = arena_run_alloc_large(arena, usize + large_pad, zero);  in arena_malloc_large()
2229 	if (run == NULL) {  in arena_malloc_large()
2233 	miscelm = arena_run_to_miscelm(run);  in arena_malloc_large()
2271 	arena_run_t *run;  in arena_palloc_large()  local
2286 	run = arena_run_alloc_large(arena, alloc_size, false);  in arena_palloc_large()
2287 	if (run == NULL) {  in arena_palloc_large()
2291 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);  in arena_palloc_large()
2292 	miscelm = arena_run_to_miscelm(run);  in arena_palloc_large()
2301 		arena_run_t *head_run = run;  in arena_palloc_large()
2306 		run = &miscelm->run;  in arena_palloc_large()
2312 		arena_run_trim_tail(arena, chunk, run, usize + large_pad +  in arena_palloc_large()
2315 	if (arena_run_init_large(arena, run, usize + large_pad, zero)) {  in arena_palloc_large()
2317 		    arena_miscelm_to_pageind(arena_run_to_miscelm(run));  in arena_palloc_large()
2323 		arena_run_dalloc(arena, run, dirty, false, decommitted);  in arena_palloc_large()
2409 arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,  in arena_dissociate_bin_run()  argument
2414 	if (run == bin->runcur)  in arena_dissociate_bin_run()
2427 			arena_bin_runs_remove(bin, run);  in arena_dissociate_bin_run()
2433 arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,  in arena_dalloc_bin_run()  argument
2437 	assert(run != bin->runcur);  in arena_dalloc_bin_run()
2438 	assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) ==  in arena_dalloc_bin_run()
2444 	arena_run_dalloc_decommit(arena, chunk, run);  in arena_dalloc_bin_run()
2453 arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,  in arena_bin_lower_run()  argument
2462 	if ((uintptr_t)run < (uintptr_t)bin->runcur) {  in arena_bin_lower_run()
2466 		bin->runcur = run;  in arena_bin_lower_run()
2470 		arena_bin_runs_insert(bin, run);  in arena_bin_lower_run()
2478 	arena_run_t *run;  in arena_dalloc_bin_locked_impl()  local
2485 	run = &arena_miscelm_get(chunk, rpages_ind)->run;  in arena_dalloc_bin_locked_impl()
2486 	binind = run->binind;  in arena_dalloc_bin_locked_impl()
2493 	arena_run_reg_dalloc(run, ptr);  in arena_dalloc_bin_locked_impl()
2494 	if (run->nfree == bin_info->nregs) {  in arena_dalloc_bin_locked_impl()
2495 		arena_dissociate_bin_run(chunk, run, bin);  in arena_dalloc_bin_locked_impl()
2496 		arena_dalloc_bin_run(arena, chunk, run, bin);  in arena_dalloc_bin_locked_impl()
2497 	} else if (run->nfree == 1 && run != bin->runcur)  in arena_dalloc_bin_locked_impl()
2498 		arena_bin_lower_run(arena, chunk, run, bin);  in arena_dalloc_bin_locked_impl()
2518 	arena_run_t *run;  in arena_dalloc_bin()  local
2523 	run = &arena_miscelm_get(chunk, rpages_ind)->run;  in arena_dalloc_bin()
2524 	bin = &arena->bins[run->binind];  in arena_dalloc_bin()
2569 	arena_run_t *run = &miscelm->run;  in arena_dalloc_large_locked_impl()  local
2587 	arena_run_dalloc_decommit(arena, chunk, run);  in arena_dalloc_large_locked_impl()
2613 	arena_run_t *run = &miscelm->run;  in arena_ralloc_large_shrink()  local
2622 	arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size +  in arena_ralloc_large_shrink()
2666 		arena_run_t *run;  in arena_ralloc_large_grow()  local
2678 		run = &arena_miscelm_get(chunk, pageind+npages)->run;  in arena_ralloc_large_grow()
2679 		if (arena_run_split_large(arena, run, splitsize, zero))  in arena_ralloc_large_grow()