338 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)  in arena_run_zero()  argument
342 	    (run_ind << LG_PAGE)), (npages << LG_PAGE));  in arena_run_zero()
343 	memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,  in arena_run_zero()
348 arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)  in arena_run_page_mark_zeroed()  argument
351 	JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind  in arena_run_page_mark_zeroed()
356 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)  in arena_run_page_validate_zeroed()  argument
359 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));  in arena_run_page_validate_zeroed()
361 	arena_run_page_mark_zeroed(chunk, run_ind);  in arena_run_page_validate_zeroed()
380 arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,  in arena_run_split_remove()  argument
387 	total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>  in arena_run_split_remove()
389 	assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==  in arena_run_split_remove()
394 	arena_avail_remove(arena, chunk, run_ind, total_pages);  in arena_run_split_remove()
396 		arena_run_dirty_remove(arena, chunk, run_ind, total_pages);  in arena_run_split_remove()
406 		arena_mapbits_unallocated_set(chunk, run_ind+need_pages,  in arena_run_split_remove()
408 		    (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) &  in arena_run_split_remove()
410 		arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1,  in arena_run_split_remove()
412 		    (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) &  in arena_run_split_remove()
415 			arena_run_dirty_insert(arena, chunk, run_ind+need_pages,  in arena_run_split_remove()
418 		arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages);  in arena_run_split_remove()
428 	size_t flag_dirty, flag_decommitted, run_ind, need_pages;  in arena_run_split_large_helper()  local
433 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_split_large_helper()
434 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);  in arena_run_split_large_helper()
435 	flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);  in arena_run_split_large_helper()
440 	    run_ind << LG_PAGE, size, arena->ind))  in arena_run_split_large_helper()
444 		arena_run_split_remove(arena, chunk, run_ind, flag_dirty,  in arena_run_split_large_helper()
452 			    *)((uintptr_t)chunk + (run_ind << LG_PAGE)),  in arena_run_split_large_helper()
456 			arena_run_zero(chunk, run_ind, need_pages);  in arena_run_split_large_helper()
464 				if (arena_mapbits_unzeroed_get(chunk, run_ind+i)  in arena_run_split_large_helper()
466 					arena_run_zero(chunk, run_ind+i, 1);  in arena_run_split_large_helper()
469 					    run_ind+i);  in arena_run_split_large_helper()
472 					    run_ind+i);  in arena_run_split_large_helper()
478 		    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));  in arena_run_split_large_helper()
487 	arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty |  in arena_run_split_large_helper()
489 	    run_ind+need_pages-1)));  in arena_run_split_large_helper()
490 	arena_mapbits_large_set(chunk, run_ind, size, flag_dirty |  in arena_run_split_large_helper()
491 	    (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind)));  in arena_run_split_large_helper()
515 	size_t flag_dirty, flag_decommitted, run_ind, need_pages, i;  in arena_run_split_small()  local
521 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_split_small()
522 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);  in arena_run_split_small()
523 	flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind);  in arena_run_split_small()
528 	    run_ind << LG_PAGE, size, arena->ind))  in arena_run_split_small()
531 	arena_run_split_remove(arena, chunk, run_ind, flag_dirty,  in arena_run_split_small()
536 		    run_ind+i);  in arena_run_split_small()
537 		arena_mapbits_small_set(chunk, run_ind+i, i, binind,  in arena_run_split_small()
540 			arena_run_page_validate_zeroed(chunk, run_ind+i);  in arena_run_split_small()
543 	    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));  in arena_run_split_small()
1556 	size_t run_ind = *p_run_ind;  in arena_run_coalesce()  local
1560 	if (run_ind + run_pages < chunk_npages &&  in arena_run_coalesce()
1561 	    arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&  in arena_run_coalesce()
1562 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty &&  in arena_run_coalesce()
1563 	    arena_mapbits_decommitted_get(chunk, run_ind+run_pages) ==  in arena_run_coalesce()
1566 		    run_ind+run_pages);  in arena_run_coalesce()
1574 		    run_ind+run_pages+nrun_pages-1) == nrun_size);  in arena_run_coalesce()
1576 		    run_ind+run_pages+nrun_pages-1) == flag_dirty);  in arena_run_coalesce()
1578 		    run_ind+run_pages+nrun_pages-1) == flag_decommitted);  in arena_run_coalesce()
1579 		arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages);  in arena_run_coalesce()
1586 			arena_run_dirty_remove(arena, chunk, run_ind+run_pages,  in arena_run_coalesce()
1593 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);  in arena_run_coalesce()
1594 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,  in arena_run_coalesce()
1599 	if (run_ind > map_bias && arena_mapbits_allocated_get(chunk,  in arena_run_coalesce()
1600 	    run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) ==  in arena_run_coalesce()
1601 	    flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) ==  in arena_run_coalesce()
1604 		    run_ind-1);  in arena_run_coalesce()
1607 		run_ind -= prun_pages;  in arena_run_coalesce()
1613 		assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==  in arena_run_coalesce()
1615 		assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);  in arena_run_coalesce()
1616 		assert(arena_mapbits_decommitted_get(chunk, run_ind) ==  in arena_run_coalesce()
1618 		arena_avail_remove(arena, chunk, run_ind, prun_pages);  in arena_run_coalesce()
1625 			arena_run_dirty_remove(arena, chunk, run_ind,  in arena_run_coalesce()
1632 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);  in arena_run_coalesce()
1633 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,  in arena_run_coalesce()
1638 	*p_run_ind = run_ind;  in arena_run_coalesce()
1644     size_t run_ind)  in arena_run_size_get()  argument
1648 	assert(run_ind >= map_bias);  in arena_run_size_get()
1649 	assert(run_ind < chunk_npages);  in arena_run_size_get()
1651 	if (arena_mapbits_large_get(chunk, run_ind) != 0) {  in arena_run_size_get()
1652 		size = arena_mapbits_large_size_get(chunk, run_ind);  in arena_run_size_get()
1654 		    run_ind+(size>>LG_PAGE)-1) == 0);  in arena_run_size_get()
1667 	size_t run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_decommit()  local
1668 	size_t offset = run_ind << LG_PAGE;  in arena_run_decommit()
1669 	size_t length = arena_run_size_get(arena, chunk, run, run_ind);  in arena_run_decommit()
1681 	size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;  in arena_run_dalloc()  local
1685 	run_ind = arena_miscelm_to_pageind(miscelm);  in arena_run_dalloc()
1686 	assert(run_ind >= map_bias);  in arena_run_dalloc()
1687 	assert(run_ind < chunk_npages);  in arena_run_dalloc()
1688 	size = arena_run_size_get(arena, chunk, run, run_ind);  in arena_run_dalloc()
1698 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==  in arena_run_dalloc()
1699 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1700 	if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind)  in arena_run_dalloc()
1709 		arena_mapbits_unallocated_set(chunk, run_ind, size, flags);  in arena_run_dalloc()
1710 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,  in arena_run_dalloc()
1713 		arena_mapbits_unallocated_set(chunk, run_ind, size,  in arena_run_dalloc()
1714 		    arena_mapbits_unzeroed_get(chunk, run_ind));  in arena_run_dalloc()
1715 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,  in arena_run_dalloc()
1716 		    arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1719 	arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages,  in arena_run_dalloc()
1723 	assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==  in arena_run_dalloc()
1724 	    arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1725 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==  in arena_run_dalloc()
1726 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1727 	assert(arena_mapbits_decommitted_get(chunk, run_ind) ==  in arena_run_dalloc()
1728 	    arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1));  in arena_run_dalloc()
1729 	arena_avail_insert(arena, chunk, run_ind, run_pages);  in arena_run_dalloc()
1732 		arena_run_dirty_insert(arena, chunk, run_ind, run_pages);  in arena_run_dalloc()
1736 		assert(run_ind == map_bias);  in arena_run_dalloc()
2316 		size_t run_ind =  in arena_palloc_large()  local
2318 		bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);  in arena_palloc_large()
2320 		    run_ind) != 0);  in arena_palloc_large()