Lines Matching refs:chunk
29 static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk,
31 static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
67 arena_chunk_t *chunk; in arena_miscelm_size_get() local
72 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); in arena_miscelm_size_get()
74 mapbits = arena_mapbits_get(chunk, pageind); in arena_miscelm_size_get()
206 arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in rb_gen()
210 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in rb_gen()
212 arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk, in rb_gen()
217 arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_avail_remove() argument
221 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_avail_remove()
223 arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk, in arena_avail_remove()
228 arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_run_dirty_insert() argument
231 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); in arena_run_dirty_insert()
233 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_run_dirty_insert()
235 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); in arena_run_dirty_insert()
236 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == in arena_run_dirty_insert()
245 arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, in arena_run_dirty_remove() argument
248 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); in arena_run_dirty_remove()
250 assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> in arena_run_dirty_remove()
252 assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); in arena_run_dirty_remove()
253 assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == in arena_run_dirty_remove()
314 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_reg_dalloc() local
315 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_run_reg_dalloc()
316 size_t mapbits = arena_mapbits_get(chunk, pageind); in arena_run_reg_dalloc()
338 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) in arena_run_zero() argument
341 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_zero()
343 memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, in arena_run_zero()
348 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_mark_zeroed() argument
351 JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind in arena_run_page_mark_zeroed()
356 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) in arena_run_page_validate_zeroed() argument
359 UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); in arena_run_page_validate_zeroed()
361 arena_run_page_mark_zeroed(chunk, run_ind); in arena_run_page_validate_zeroed()
380 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, in arena_run_split_remove() argument
387 total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> in arena_run_split_remove()
389 assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == in arena_run_split_remove()
394 arena_avail_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
396 arena_run_dirty_remove(arena, chunk, run_ind, total_pages); in arena_run_split_remove()
406 arena_mapbits_unallocated_set(chunk, run_ind+need_pages, in arena_run_split_remove()
408 (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & in arena_run_split_remove()
410 arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, in arena_run_split_remove()
412 (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & in arena_run_split_remove()
415 arena_run_dirty_insert(arena, chunk, run_ind+need_pages, in arena_run_split_remove()
418 arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); in arena_run_split_remove()
426 arena_chunk_t *chunk; in arena_run_split_large_helper() local
431 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_split_large_helper()
434 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_large_helper()
435 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_large_helper()
439 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, in arena_run_split_large_helper()
444 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_large_helper()
452 *)((uintptr_t)chunk + (run_ind << LG_PAGE)), in arena_run_split_large_helper()
456 arena_run_zero(chunk, run_ind, need_pages); in arena_run_split_large_helper()
464 if (arena_mapbits_unzeroed_get(chunk, run_ind+i) in arena_run_split_large_helper()
466 arena_run_zero(chunk, run_ind+i, 1); in arena_run_split_large_helper()
468 arena_run_page_validate_zeroed(chunk, in arena_run_split_large_helper()
471 arena_run_page_mark_zeroed(chunk, in arena_run_split_large_helper()
477 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_split_large_helper()
487 arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | in arena_run_split_large_helper()
488 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_split_large_helper()
490 arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | in arena_run_split_large_helper()
491 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); in arena_run_split_large_helper()
513 arena_chunk_t *chunk; in arena_run_split_small() local
519 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_split_small()
522 flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); in arena_run_split_small()
523 flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); in arena_run_split_small()
527 if (flag_decommitted != 0 && arena->chunk_hooks.commit(chunk, chunksize, in arena_run_split_small()
531 arena_run_split_remove(arena, chunk, run_ind, flag_dirty, in arena_run_split_small()
535 size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, in arena_run_split_small()
537 arena_mapbits_small_set(chunk, run_ind+i, i, binind, in arena_run_split_small()
540 arena_run_page_validate_zeroed(chunk, run_ind+i); in arena_run_split_small()
542 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + in arena_run_split_small()
550 arena_chunk_t *chunk; in arena_chunk_init_spare() local
554 chunk = arena->spare; in arena_chunk_init_spare()
557 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); in arena_chunk_init_spare()
558 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); in arena_chunk_init_spare()
559 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == in arena_chunk_init_spare()
561 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == in arena_chunk_init_spare()
563 assert(arena_mapbits_dirty_get(chunk, map_bias) == in arena_chunk_init_spare()
564 arena_mapbits_dirty_get(chunk, chunk_npages-1)); in arena_chunk_init_spare()
566 return (chunk); in arena_chunk_init_spare()
570 arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero) in arena_chunk_register() argument
579 extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true); in arena_chunk_register()
580 extent_node_achunk_set(&chunk->node, true); in arena_chunk_register()
581 return (chunk_register(chunk, &chunk->node)); in arena_chunk_register()
588 arena_chunk_t *chunk; in arena_chunk_alloc_internal_hard() local
592 chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL, in arena_chunk_alloc_internal_hard()
594 if (chunk != NULL && !*commit) { in arena_chunk_alloc_internal_hard()
596 if (chunk_hooks->commit(chunk, chunksize, 0, map_bias << in arena_chunk_alloc_internal_hard()
599 (void *)chunk, chunksize, *commit); in arena_chunk_alloc_internal_hard()
600 chunk = NULL; in arena_chunk_alloc_internal_hard()
603 if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) { in arena_chunk_alloc_internal_hard()
606 chunk_hooks->decommit(chunk, chunksize, 0, map_bias << in arena_chunk_alloc_internal_hard()
609 chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk, in arena_chunk_alloc_internal_hard()
611 chunk = NULL; in arena_chunk_alloc_internal_hard()
615 return (chunk); in arena_chunk_alloc_internal_hard()
621 arena_chunk_t *chunk; in arena_chunk_alloc_internal() local
624 chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize, in arena_chunk_alloc_internal()
626 if (chunk != NULL) { in arena_chunk_alloc_internal()
627 if (arena_chunk_register(arena, chunk, *zero)) { in arena_chunk_alloc_internal()
628 chunk_dalloc_cache(arena, &chunk_hooks, chunk, in arena_chunk_alloc_internal()
634 if (chunk == NULL) { in arena_chunk_alloc_internal()
635 chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks, in arena_chunk_alloc_internal()
639 if (config_stats && chunk != NULL) { in arena_chunk_alloc_internal()
644 return (chunk); in arena_chunk_alloc_internal()
650 arena_chunk_t *chunk; in arena_chunk_init_hard() local
658 chunk = arena_chunk_alloc_internal(arena, &zero, &commit); in arena_chunk_init_hard()
659 if (chunk == NULL) in arena_chunk_init_hard()
669 arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, in arena_chunk_init_hard()
677 (void *)arena_bitselm_get(chunk, map_bias+1), in arena_chunk_init_hard()
678 (size_t)((uintptr_t) arena_bitselm_get(chunk, in arena_chunk_init_hard()
679 chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk, in arena_chunk_init_hard()
682 arena_mapbits_internal_set(chunk, i, flag_unzeroed); in arena_chunk_init_hard()
685 *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t) in arena_chunk_init_hard()
686 arena_bitselm_get(chunk, chunk_npages-1) - in arena_chunk_init_hard()
687 (uintptr_t)arena_bitselm_get(chunk, map_bias+1))); in arena_chunk_init_hard()
690 assert(arena_mapbits_unzeroed_get(chunk, i) == in arena_chunk_init_hard()
695 arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, in arena_chunk_init_hard()
698 return (chunk); in arena_chunk_init_hard()
704 arena_chunk_t *chunk; in arena_chunk_alloc() local
707 chunk = arena_chunk_init_spare(arena); in arena_chunk_alloc()
709 chunk = arena_chunk_init_hard(arena); in arena_chunk_alloc()
710 if (chunk == NULL) in arena_chunk_alloc()
715 arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); in arena_chunk_alloc()
717 return (chunk); in arena_chunk_alloc()
721 arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) in arena_chunk_dalloc() argument
724 assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); in arena_chunk_dalloc()
725 assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); in arena_chunk_dalloc()
726 assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == in arena_chunk_dalloc()
728 assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == in arena_chunk_dalloc()
730 assert(arena_mapbits_dirty_get(chunk, map_bias) == in arena_chunk_dalloc()
731 arena_mapbits_dirty_get(chunk, chunk_npages-1)); in arena_chunk_dalloc()
732 assert(arena_mapbits_decommitted_get(chunk, map_bias) == in arena_chunk_dalloc()
733 arena_mapbits_decommitted_get(chunk, chunk_npages-1)); in arena_chunk_dalloc()
739 arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); in arena_chunk_dalloc()
746 arena->spare = chunk; in arena_chunk_dalloc()
777 arena->spare = chunk; in arena_chunk_dalloc()
929 arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) in arena_chunk_dalloc_huge() argument
943 chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize, true); in arena_chunk_dalloc_huge()
948 arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize, in arena_chunk_ralloc_huge_similar() argument
973 arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, in arena_chunk_ralloc_huge_shrink() argument
991 void *nchunk = (void *)((uintptr_t)chunk + in arena_chunk_ralloc_huge_shrink()
1001 void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk, in arena_chunk_ralloc_huge_expand_hard() argument
1019 } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, in arena_chunk_ralloc_huge_expand_hard()
1029 arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, in arena_chunk_ralloc_huge_expand() argument
1034 void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); in arena_chunk_ralloc_huge_expand()
1052 chunk, oldsize, usize, zero, nchunk, udiff, in arena_chunk_ralloc_huge_expand()
1054 } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, in arena_chunk_ralloc_huge_expand()
1097 arena_chunk_t *chunk; in arena_run_alloc_large() local
1111 chunk = arena_chunk_alloc(arena); in arena_run_alloc_large()
1112 if (chunk != NULL) { in arena_run_alloc_large()
1113 run = &arena_miscelm_get(chunk, map_bias)->run; in arena_run_alloc_large()
1141 arena_chunk_t *chunk; in arena_run_alloc_small() local
1156 chunk = arena_chunk_alloc(arena); in arena_run_alloc_small()
1157 if (chunk != NULL) { in arena_run_alloc_small()
1158 run = &arena_miscelm_get(chunk, map_bias)->run; in arena_run_alloc_small()
1251 arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( in arena_dirty_count() local
1256 assert(arena_mapbits_allocated_get(chunk, pageind) == in arena_dirty_count()
1258 assert(arena_mapbits_large_get(chunk, pageind) == 0); in arena_dirty_count()
1259 assert(arena_mapbits_dirty_get(chunk, pageind) != 0); in arena_dirty_count()
1260 npages = arena_mapbits_unallocated_size_get(chunk, in arena_dirty_count()
1308 UNUSED void *chunk; in arena_stash_dirty() local
1316 chunk = chunk_alloc_cache(arena, chunk_hooks, in arena_stash_dirty()
1320 assert(chunk == extent_node_addr_get(chunkselm)); in arena_stash_dirty()
1327 arena_chunk_t *chunk = in arena_stash_dirty() local
1334 arena_mapbits_unallocated_size_get(chunk, pageind); in arena_stash_dirty()
1339 assert(arena_mapbits_dirty_get(chunk, pageind) == in arena_stash_dirty()
1340 arena_mapbits_dirty_get(chunk, pageind+npages-1)); in arena_stash_dirty()
1346 if (chunk == arena->spare) in arena_stash_dirty()
1403 arena_chunk_t *chunk = in arena_purge_stashed() local
1408 run_size = arena_mapbits_large_size_get(chunk, pageind); in arena_purge_stashed()
1412 assert(!arena_mapbits_decommitted_get(chunk, pageind)); in arena_purge_stashed()
1413 assert(!arena_mapbits_decommitted_get(chunk, in arena_purge_stashed()
1415 decommitted = !chunk_hooks->decommit(chunk, chunksize, in arena_purge_stashed()
1422 chunk_hooks, chunk, chunksize, pageind << in arena_purge_stashed()
1426 arena_mapbits_large_set(chunk, pageind+npages-1, 0, in arena_purge_stashed()
1428 arena_mapbits_large_set(chunk, pageind, run_size, in arena_purge_stashed()
1443 arena_mapbits_internal_set(chunk, pageind+i, in arena_purge_stashed()
1488 arena_chunk_t *chunk = in arena_unstash_purged() local
1493 bool decommitted = (arena_mapbits_decommitted_get(chunk, in arena_unstash_purged()
1551 arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, in arena_run_coalesce() argument
1561 arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && in arena_run_coalesce()
1562 arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && in arena_run_coalesce()
1563 arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == in arena_run_coalesce()
1565 size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
1573 assert(arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
1575 assert(arena_mapbits_dirty_get(chunk, in arena_run_coalesce()
1577 assert(arena_mapbits_decommitted_get(chunk, in arena_run_coalesce()
1579 arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); in arena_run_coalesce()
1586 arena_run_dirty_remove(arena, chunk, run_ind+run_pages, in arena_run_coalesce()
1593 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
1594 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
1599 if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, in arena_run_coalesce()
1600 run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == in arena_run_coalesce()
1601 flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == in arena_run_coalesce()
1603 size_t prun_size = arena_mapbits_unallocated_size_get(chunk, in arena_run_coalesce()
1613 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_coalesce()
1615 assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); in arena_run_coalesce()
1616 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_coalesce()
1618 arena_avail_remove(arena, chunk, run_ind, prun_pages); in arena_run_coalesce()
1625 arena_run_dirty_remove(arena, chunk, run_ind, in arena_run_coalesce()
1632 arena_mapbits_unallocated_size_set(chunk, run_ind, size); in arena_run_coalesce()
1633 arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, in arena_run_coalesce()
1643 arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_run_size_get() argument
1651 if (arena_mapbits_large_get(chunk, run_ind) != 0) { in arena_run_size_get()
1652 size = arena_mapbits_large_size_get(chunk, run_ind); in arena_run_size_get()
1653 assert(size == PAGE || arena_mapbits_large_size_get(chunk, in arena_run_size_get()
1664 arena_run_decommit(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run) in arena_run_decommit() argument
1669 size_t length = arena_run_size_get(arena, chunk, run, run_ind); in arena_run_decommit()
1671 return (arena->chunk_hooks.decommit(chunk, chunksize, offset, length, in arena_run_decommit()
1679 arena_chunk_t *chunk; in arena_run_dalloc() local
1683 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_run_dalloc()
1688 size = arena_run_size_get(arena, chunk, run, run_ind); in arena_run_dalloc()
1698 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
1699 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1700 if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) in arena_run_dalloc()
1709 arena_mapbits_unallocated_set(chunk, run_ind, size, flags); in arena_run_dalloc()
1710 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
1713 arena_mapbits_unallocated_set(chunk, run_ind, size, in arena_run_dalloc()
1714 arena_mapbits_unzeroed_get(chunk, run_ind)); in arena_run_dalloc()
1715 arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, in arena_run_dalloc()
1716 arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1719 arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, in arena_run_dalloc()
1723 assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == in arena_run_dalloc()
1724 arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1725 assert(arena_mapbits_dirty_get(chunk, run_ind) == in arena_run_dalloc()
1726 arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1727 assert(arena_mapbits_decommitted_get(chunk, run_ind) == in arena_run_dalloc()
1728 arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); in arena_run_dalloc()
1729 arena_avail_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
1732 arena_run_dirty_insert(arena, chunk, run_ind, run_pages); in arena_run_dalloc()
1738 arena_chunk_dalloc(arena, chunk); in arena_run_dalloc()
1753 arena_run_dalloc_decommit(arena_t *arena, arena_chunk_t *chunk, in arena_run_dalloc_decommit() argument
1756 bool committed = arena_run_decommit(arena, chunk, run); in arena_run_dalloc_decommit()
1762 arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_run_trim_head() argument
1768 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); in arena_run_trim_head()
1769 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); in arena_run_trim_head()
1780 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); in arena_run_trim_head()
1781 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | in arena_run_trim_head()
1782 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_head()
1784 arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty | in arena_run_trim_head()
1785 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); in arena_run_trim_head()
1789 assert(arena_mapbits_large_size_get(chunk, in arena_run_trim_head()
1791 assert(arena_mapbits_dirty_get(chunk, in arena_run_trim_head()
1794 arena_mapbits_large_set(chunk, pageind+head_npages, newsize, in arena_run_trim_head()
1795 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_head()
1802 arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_run_trim_tail() argument
1808 size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); in arena_run_trim_tail()
1809 size_t flag_decommitted = arena_mapbits_decommitted_get(chunk, pageind); in arena_run_trim_tail()
1822 assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); in arena_run_trim_tail()
1823 arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty | in arena_run_trim_tail()
1824 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_tail()
1826 arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty | in arena_run_trim_tail()
1827 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, pageind))); in arena_run_trim_tail()
1831 assert(arena_mapbits_large_size_get(chunk, in arena_run_trim_tail()
1833 assert(arena_mapbits_dirty_get(chunk, in arena_run_trim_tail()
1836 arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, in arena_run_trim_tail()
1837 flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_run_trim_tail()
1840 tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages); in arena_run_trim_tail()
1959 arena_chunk_t *chunk; in arena_bin_malloc_hard() local
1969 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_bin_malloc_hard()
1971 arena_dalloc_bin_run(arena, chunk, run, bin); in arena_bin_malloc_hard()
1973 arena_bin_lower_run(arena, chunk, run, bin); in arena_bin_malloc_hard()
2272 arena_chunk_t *chunk; in arena_palloc_large() local
2291 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); in arena_palloc_large()
2303 miscelm = arena_miscelm_get(chunk, in arena_palloc_large()
2308 arena_run_trim_head(arena, chunk, head_run, alloc_size, in arena_palloc_large()
2312 arena_run_trim_tail(arena, chunk, run, usize + large_pad + in arena_palloc_large()
2318 bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); in arena_palloc_large()
2319 bool decommitted = (arena_mapbits_decommitted_get(chunk, in arena_palloc_large()
2387 arena_chunk_t *chunk; in arena_prof_promoted() local
2398 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); in arena_prof_promoted()
2399 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_prof_promoted()
2402 arena_mapbits_large_binind_set(chunk, pageind, binind); in arena_prof_promoted()
2409 arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, in arena_dissociate_bin_run() argument
2418 &chunk->node), bin); in arena_dissociate_bin_run()
2433 arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_dalloc_bin_run() argument
2444 arena_run_dalloc_decommit(arena, chunk, run); in arena_dalloc_bin_run()
2453 arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, in arena_bin_lower_run() argument
2474 arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_dalloc_bin_locked_impl() argument
2483 pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_dalloc_bin_locked_impl()
2484 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); in arena_dalloc_bin_locked_impl()
2485 run = &arena_miscelm_get(chunk, rpages_ind)->run; in arena_dalloc_bin_locked_impl()
2495 arena_dissociate_bin_run(chunk, run, bin); in arena_dalloc_bin_locked_impl()
2496 arena_dalloc_bin_run(arena, chunk, run, bin); in arena_dalloc_bin_locked_impl()
2498 arena_bin_lower_run(arena, chunk, run, bin); in arena_dalloc_bin_locked_impl()
2507 arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_dalloc_bin_junked_locked() argument
2511 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true); in arena_dalloc_bin_junked_locked()
2515 arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_dalloc_bin() argument
2522 rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); in arena_dalloc_bin()
2523 run = &arena_miscelm_get(chunk, rpages_ind)->run; in arena_dalloc_bin()
2526 arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false); in arena_dalloc_bin()
2531 arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_dalloc_small() argument
2538 assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, in arena_dalloc_small()
2541 bitselm = arena_bitselm_get(chunk, pageind); in arena_dalloc_small()
2542 arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm); in arena_dalloc_small()
2564 arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_large_locked_impl() argument
2567 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_dalloc_large_locked_impl()
2568 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); in arena_dalloc_large_locked_impl()
2572 size_t usize = arena_mapbits_large_size_get(chunk, pageind) - in arena_dalloc_large_locked_impl()
2587 arena_run_dalloc_decommit(arena, chunk, run); in arena_dalloc_large_locked_impl()
2591 arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, in arena_dalloc_large_junked_locked() argument
2595 arena_dalloc_large_locked_impl(arena, chunk, ptr, true); in arena_dalloc_large_junked_locked()
2599 arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) in arena_dalloc_large() argument
2603 arena_dalloc_large_locked_impl(arena, chunk, ptr, false); in arena_dalloc_large()
2608 arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_ralloc_large_shrink() argument
2611 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_ralloc_large_shrink()
2612 arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); in arena_ralloc_large_shrink()
2622 arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size + in arena_ralloc_large_shrink()
2644 arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, in arena_ralloc_large_grow() argument
2647 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; in arena_ralloc_large_grow()
2651 assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - in arena_ralloc_large_grow()
2656 if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, in arena_ralloc_large_grow()
2659 followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); in arena_ralloc_large_grow()
2678 run = &arena_miscelm_get(chunk, pageind+npages)->run; in arena_ralloc_large_grow()
2703 flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | in arena_ralloc_large_grow()
2704 arena_mapbits_dirty_get(chunk, pageind+npages-1); in arena_ralloc_large_grow()
2706 arena_mapbits_large_set(chunk, pageind, size + large_pad, in arena_ralloc_large_grow()
2708 arena_mapbits_unzeroed_get(chunk, pageind))); in arena_ralloc_large_grow()
2709 arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty | in arena_ralloc_large_grow()
2710 (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, in arena_ralloc_large_grow()
2765 arena_chunk_t *chunk; in arena_ralloc_large() local
2773 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); in arena_ralloc_large()
2774 arena = extent_node_arena_get(&chunk->node); in arena_ralloc_large()
2777 bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, in arena_ralloc_large()
2794 arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max); in arena_ralloc_large()