Lines Matching refs:z
743 zone_has_index(zone_t z, zone_id_t zid) in zone_has_index() argument
745 return zone_array + zid == z; in zone_has_index()
838 #define zone_counter_sub(z, stat, value) ({ \ argument
839 if (os_sub_overflow((z)->stat, value, &(z)->stat)) { \
840 zone_accounting_panic(z, #stat " wrap-around"); \
842 (z)->stat; \
846 zone_meta_alloc_size_add(zone_t z, struct zone_page_metadata *m, in zone_meta_alloc_size_add() argument
850 zone_page_meta_accounting_panic(z, m, "alloc_size wrap-around"); in zone_meta_alloc_size_add()
856 zone_meta_alloc_size_sub(zone_t z, struct zone_page_metadata *m, in zone_meta_alloc_size_sub() argument
860 zone_page_meta_accounting_panic(z, m, "alloc_size wrap-around"); in zone_meta_alloc_size_sub()
925 zone_queue_set_head(zone_t z, zone_pva_t queue, zone_pva_t oldv, in zone_queue_set_head() argument
931 zone_page_metadata_list_corruption(z, meta); in zone_queue_set_head()
988 zone_meta_validate(zone_t z, struct zone_page_metadata *meta, vm_address_t addr) in zone_meta_validate() argument
990 if (!zone_has_index(z, meta->zm_index)) { in zone_meta_validate()
991 zone_page_metadata_index_confusion_panic(z, addr, meta); in zone_meta_validate()
997 zone_meta_validate(zone_t z, struct zone_page_metadata *meta) in zone_meta_validate() argument
999 zone_meta_validate(z, meta, zone_meta_to_addr(meta)); in zone_meta_validate()
1003 zone_meta_queue_push(zone_t z, zone_pva_t *headp, in zone_meta_queue_push() argument
1014 zone_page_metadata_list_corruption(z, meta); in zone_meta_queue_push()
1023 zone_meta_queue_pop(zone_t z, zone_pva_t *headp) in zone_meta_queue_pop() argument
1029 zone_meta_validate(z, meta); in zone_meta_queue_pop()
1034 zone_page_metadata_list_corruption(z, meta); in zone_meta_queue_pop()
1046 zone_meta_remqueue(zone_t z, struct zone_page_metadata *meta) in zone_meta_remqueue() argument
1054 zone_page_metadata_list_corruption(z, meta); in zone_meta_remqueue()
1059 zone_queue_set_head(z, meta->zm_page_prev, meta_pva, meta); in zone_meta_remqueue()
1063 zone_page_metadata_list_corruption(z, meta); in zone_meta_remqueue()
1072 zone_meta_requeue(zone_t z, zone_pva_t *headp, in zone_meta_requeue() argument
1075 zone_meta_remqueue(z, meta); in zone_meta_requeue()
1076 zone_meta_queue_push(z, headp, meta); in zone_meta_requeue()
1081 zone_meta_lock_in_partial(zone_t z, struct zone_page_metadata *m, uint32_t len) in zone_meta_lock_in_partial() argument
1083 uint16_t new_size = zone_meta_alloc_size_add(z, m, ZM_ALLOC_SIZE_LOCK); in zone_meta_lock_in_partial()
1087 zone_meta_requeue(z, &z->z_pageq_partial, m); in zone_meta_lock_in_partial()
1088 zone_counter_sub(z, z_wired_empty, len); in zone_meta_lock_in_partial()
1094 zone_meta_unlock_from_partial(zone_t z, struct zone_page_metadata *m, uint32_t len) in zone_meta_unlock_from_partial() argument
1096 uint16_t new_size = zone_meta_alloc_size_sub(z, m, ZM_ALLOC_SIZE_LOCK); in zone_meta_unlock_from_partial()
1100 zone_meta_requeue(z, &z->z_pageq_empty, m); in zone_meta_unlock_from_partial()
1101 z->z_wired_empty += len; in zone_meta_unlock_from_partial()
1342 zone_element_size(void *elem, zone_t *z, bool clear_oob, vm_offset_t *oob_offs) in zone_element_size() argument
1401 if (z) { in zone_element_size()
1402 *z = zone; in zone_element_size()
1663 zone_t z = zone_find_largest(&zsize); in zba_memory_exhausted() local
1667 zone_heap_name(z), zone_name(z), in zba_memory_exhausted()
1669 zone_count_allocated(z)); in zba_memory_exhausted()
2514 zone_supports_vm(zone_t z) in zone_supports_vm() argument
2521 return z >= &zone_array[ZONE_ID_VM_MAP_ENTRY] && in zone_supports_vm()
2522 z <= &zone_array[ZONE_ID_VM_MAP_HOLES]; in zone_supports_vm()
2526 zone_name(zone_t z) in zone_name() argument
2528 return z->z_name; in zone_name()
2532 zone_heap_name(zone_t z) in zone_heap_name() argument
2534 zone_security_flags_t zsflags = zone_security_config(z); in zone_heap_name()
2542 zone_alloc_pages_for_nelems(zone_t z, vm_size_t max_elems) in zone_alloc_pages_for_nelems() argument
2546 elem_count = ptoa(z->z_percpu ? 1 : z->z_chunk_pages) / in zone_alloc_pages_for_nelems()
2547 zone_elem_outer_size(z); in zone_alloc_pages_for_nelems()
2550 return (uint32_t)MIN(UINT32_MAX, chunks * z->z_chunk_pages); in zone_alloc_pages_for_nelems()
2603 zone_depot_pop_head_full(struct zone_depot *zd, zone_t z) in zone_depot_pop_head_full() argument
2610 if (z && z->z_recirc_full_min > zd->zd_full) { in zone_depot_pop_head_full()
2611 z->z_recirc_full_min = zd->zd_full; in zone_depot_pop_head_full()
2623 zone_depot_pop_head_empty(struct zone_depot *zd, zone_t z) in zone_depot_pop_head_empty() argument
2630 if (z && z->z_recirc_empty_min > zd->zd_empty) { in zone_depot_pop_head_empty()
2631 z->z_recirc_empty_min = zd->zd_empty; in zone_depot_pop_head_empty()
2644 zone_t z) in zone_depot_move_full() argument
2652 if (z && z->z_recirc_full_min > src->zd_full) { in zone_depot_move_full()
2653 z->z_recirc_full_min = src->zd_full; in zone_depot_move_full()
2665 if (z && zone_security_array[zone_index(z)].z_lifo) { in zone_depot_move_full()
2686 zone_t z) in zone_depot_move_empty() argument
2694 if (z && z->z_recirc_empty_min > src->zd_empty) { in zone_depot_move_empty()
2695 z->z_recirc_empty_min = src->zd_empty; in zone_depot_move_empty()
2939 track_kalloc_zones(zone_t z, const char *logname) in track_kalloc_zones() argument
2943 zone_security_flags_t zsflags = zone_security_config(z); in track_kalloc_zones()
2951 return zone_elem_inner_size(z) == sizeclass; in track_kalloc_zones()
2960 return zone_elem_inner_size(z) == sizeclass; in track_kalloc_zones()
2969 return zone_elem_inner_size(z) == sizeclass; in track_kalloc_zones()
3105 zalloc_uaf_panic(zone_t z, uintptr_t elem, size_t size) in zalloc_uaf_panic() argument
3107 uint32_t esize = (uint32_t)zone_elem_inner_size(z); in zalloc_uaf_panic()
3129 zone_heap_name(z), zone_name(z), in zalloc_uaf_panic()
3252 zone_leaks_record_count(zone_t z) in zone_leaks_record_count() argument
3254 uint32_t recs = (zlsize << 10) / zone_elem_inner_size(z); in zone_leaks_record_count()
3260 zone_leaks_sample_rate(zone_t z) in zone_leaks_sample_rate() argument
3262 return zlfreq / zone_elem_inner_size(z); in zone_leaks_sample_rate()
3291 zone_setup_logging(zone_t z) in zone_setup_logging() argument
3301 snprintf(zone_name, MAX_ZONE_NAME, "%s%s", zone_heap_name(z), z->z_name); in zone_setup_logging()
3309 track_kalloc_zones(z, zlog_val)) { in zone_setup_logging()
3325 track_kalloc_zones(z, zlog_val)) { in zone_setup_logging()
3345 z->z_btlog = btlog_create(BTLOG_LOG, zrecs, 0); in zone_setup_logging()
3347 z->z_btlog = btlog_create(BTLOG_HASH, in zone_setup_logging()
3348 zone_leaks_record_count(z), 0); in zone_setup_logging()
3350 if (z->z_btlog) { in zone_setup_logging()
3351 z->z_log_on = true; in zone_setup_logging()
3353 zone_heap_name(z), z->z_name); in zone_setup_logging()
3356 zone_heap_name(z), z->z_name); in zone_setup_logging()
3387 zone_setup_kasan_logging(zone_t z) in zone_setup_kasan_logging() argument
3389 if (!z->z_tbi_tag) { in zone_setup_kasan_logging()
3391 zone_heap_name(z), z->z_name); in zone_setup_kasan_logging()
3395 z->z_log_on = true; in zone_setup_kasan_logging()
3396 z->z_btlog = btlog_create(BTLOG_LOG, kasan_zrecs, 0); in zone_setup_kasan_logging()
3397 if (!z->z_btlog) { in zone_setup_kasan_logging()
3399 zone_heap_name(z), z->z_name); in zone_setup_kasan_logging()
3431 zleak_should_enable_for_zone(zone_t z) in zleak_should_enable_for_zone() argument
3433 if (z->z_log_on) { in zleak_should_enable_for_zone()
3436 if (z->z_btlog) { in zleak_should_enable_for_zone()
3439 if (z->z_exhausts) { in zleak_should_enable_for_zone()
3442 if (zone_exhaustible(z)) { in zleak_should_enable_for_zone()
3443 return z->z_wired_cur * 8 >= z->z_wired_max * 7; in zleak_should_enable_for_zone()
3445 return z->z_wired_cur >= zleak_pages_per_zone_wired_threshold; in zleak_should_enable_for_zone()
3449 zleak_should_disable_for_zone(zone_t z) in zleak_should_disable_for_zone() argument
3451 if (z->z_log_on) { in zleak_should_disable_for_zone()
3454 if (!z->z_btlog) { in zleak_should_disable_for_zone()
3457 if (zone_exhaustible(z)) { in zleak_should_disable_for_zone()
3458 return z->z_wired_cur * 8 < z->z_wired_max * 7; in zleak_should_disable_for_zone()
3460 return z->z_wired_cur < zleak_pages_per_zone_wired_threshold / 2; in zleak_should_disable_for_zone()
3468 zone_foreach(z) { in zleaks_enable_async()
3469 if (zleak_should_disable_for_zone(z)) { in zleaks_enable_async()
3470 log = z->z_btlog; in zleaks_enable_async()
3471 z->z_btlog = NULL; in zleaks_enable_async()
3472 assert(z->z_btlog_disabled == NULL); in zleaks_enable_async()
3474 z->z_btlog_disabled = log; in zleaks_enable_async()
3478 if (zleak_should_enable_for_zone(z)) { in zleaks_enable_async()
3479 log = z->z_btlog_disabled; in zleaks_enable_async()
3482 zone_leaks_record_count(z), in zleaks_enable_async()
3483 zone_leaks_sample_rate(z)); in zleaks_enable_async()
3485 z->z_btlog_disabled = NULL; in zleaks_enable_async()
3489 os_atomic_store(&z->z_btlog, log, release); in zleaks_enable_async()
3536 zone_foreach(z) { in panic_display_zleaks()
3537 btlog_t log = z->z_btlog; in panic_display_zleaks()
3554 zone_heap_name(z), zone_name(z)); in panic_display_zleaks()
3556 (long)count * zone_scale_for_percpu(z, zone_elem_inner_size(z))); in panic_display_zleaks()
3558 (long)zone_size_wired(z)); in panic_display_zleaks()
3794 static void zone_reclaim_elements(zone_t z, uint16_t n, vm_offset_t *elems);
3795 static void zone_depot_trim(zone_t z, uint32_t target, struct zone_depot *zd);
3813 zone_kma_flags(zone_t z, zone_security_flags_t zsflags, zalloc_flags_t flags) in zone_kma_flags() argument
3826 if (z->z_permanent || (!z->z_destructible && in zone_kma_flags()
3839 zone_add_wired_pages(zone_t z, uint32_t pages) in zone_add_wired_pages() argument
3844 if (__improbable(zleak_should_enable_for_zone(z) && in zone_add_wired_pages()
3849 (void)z; in zone_add_wired_pages()
3854 zone_remove_wired_pages(zone_t z, uint32_t pages) in zone_remove_wired_pages() argument
3859 if (__improbable(zleak_should_disable_for_zone(z) && in zone_remove_wired_pages()
3864 (void)z; in zone_remove_wired_pages()
3913 #define zone_tag_free_element(z, a, s) (a) argument
3914 #define zcram_memtag_init(z, b, s, e) do {} while (0) argument
4209 zone_t z, in zone_scramble_va_and_unlock() argument
4241 zone_lock(z); in zone_scramble_va_and_unlock()
4244 zone_meta_queue_push(z, &z->z_pageq_va, arr[i]); in zone_scramble_va_and_unlock()
4246 z->z_va_cur += z->z_percpu ? runs : pages; in zone_scramble_va_and_unlock()
4333 zone_allocate_va_locked(zone_t z, zalloc_flags_t flags) in zone_allocate_va_locked() argument
4335 zone_security_flags_t zsflags = zone_security_config(z); in zone_allocate_va_locked()
4337 kma_flags_t kmaflags = zone_kma_flags(z, zsflags, flags) | KMA_VAONLY; in zone_allocate_va_locked()
4338 uint32_t chunk_pages = z->z_chunk_pages; in zone_allocate_va_locked()
4345 zone_unlock(z); in zone_allocate_va_locked()
4376 if (!z->z_percpu && zone_submap_is_sequestered(zsflags)) { in zone_allocate_va_locked()
4404 if (z->z_percpu) { in zone_allocate_va_locked()
4410 } else if (!zsflags.z_pgz_use_guards && !z->z_pgz_use_guards) { in zone_allocate_va_locked()
4445 if (!z->z_percpu && zone_submap_is_sequestered(zsflags)) { in zone_allocate_va_locked()
4503 kr, zone_heap_name(z), zone_name(z), in zone_allocate_va_locked()
4517 meta[0].zm_index = zone_index(z); in zone_allocate_va_locked()
4527 meta[n].zm_index = zone_index(z); in zone_allocate_va_locked()
4531 meta[n].zm_index = zone_index(z); in zone_allocate_va_locked()
4542 return zone_scramble_va_and_unlock(z, meta, runs, pages, in zone_allocate_va_locked()
4547 zone_lock(z); in zone_allocate_va_locked()
4550 zone_meta_queue_push(z, &z->z_pageq_va, meta + n); in zone_allocate_va_locked()
4553 z->z_va_cur += z->z_percpu ? runs : pages; in zone_allocate_va_locked()
4584 __ZONE_MAP_EXHAUSTED_AND_WAITING_FOR_GC__(zone_t z, uint32_t pgs) in __ZONE_MAP_EXHAUSTED_AND_WAITING_FOR_GC__() argument
4591 if (zone_supports_vm(z) || (current_thread()->options & TH_OPT_VMPRIV)) { in __ZONE_MAP_EXHAUSTED_AND_WAITING_FOR_GC__()
4667 zalloc_expand_drain_exhausted_caches_locked(zone_t z) in zalloc_expand_drain_exhausted_caches_locked() argument
4672 if (z->z_depot_size) { in zalloc_expand_drain_exhausted_caches_locked()
4673 z->z_depot_size = 0; in zalloc_expand_drain_exhausted_caches_locked()
4674 z->z_depot_cleanup = true; in zalloc_expand_drain_exhausted_caches_locked()
4677 zone_depot_trim(z, 0, &zd); in zalloc_expand_drain_exhausted_caches_locked()
4679 zone_recirc_lock_nopreempt(z); in zalloc_expand_drain_exhausted_caches_locked()
4681 zone_depot_move_full(&z->z_recirc, in zalloc_expand_drain_exhausted_caches_locked()
4685 zone_depot_move_empty(&z->z_recirc, in zalloc_expand_drain_exhausted_caches_locked()
4688 zone_recirc_unlock_nopreempt(z); in zalloc_expand_drain_exhausted_caches_locked()
4691 zone_recirc_lock_nopreempt(z); in zalloc_expand_drain_exhausted_caches_locked()
4692 if (z->z_recirc.zd_full) { in zalloc_expand_drain_exhausted_caches_locked()
4693 mag = zone_depot_pop_head_full(&z->z_recirc, z); in zalloc_expand_drain_exhausted_caches_locked()
4695 zone_recirc_unlock_nopreempt(z); in zalloc_expand_drain_exhausted_caches_locked()
4698 zone_reclaim_elements(z, zc_mag_size(), mag->zm_elems); in zalloc_expand_drain_exhausted_caches_locked()
4723 zone_wakeup_exhausted_waiters(zone_t z) in zone_wakeup_exhausted_waiters() argument
4725 z->z_exhausted_wait = false; in zone_wakeup_exhausted_waiters()
4726 EVENT_INVOKE(ZONE_EXHAUSTED, zone_index(z), z, false); in zone_wakeup_exhausted_waiters()
4727 thread_wakeup(&z->z_expander); in zone_wakeup_exhausted_waiters()
4732 __ZONE_EXHAUSTED_AND_WAITING_HARD__(zone_t z) in __ZONE_EXHAUSTED_AND_WAITING_HARD__() argument
4734 if (z->z_pcpu_cache && z->z_depot_size && in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4735 zalloc_expand_drain_exhausted_caches_locked(z)) { in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4739 if (!z->z_exhausted_wait) { in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4740 zone_recirc_lock_nopreempt(z); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4741 z->z_exhausted_wait = true; in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4742 zone_recirc_unlock_nopreempt(z); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4743 EVENT_INVOKE(ZONE_EXHAUSTED, zone_index(z), z, true); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4746 assert_wait(&z->z_expander, TH_UNINT); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4747 zone_unlock(z); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4749 zone_lock(z); in __ZONE_EXHAUSTED_AND_WAITING_HARD__()
4753 zone_mapping_type(zone_t z) in zone_mapping_type() argument
4755 zone_security_flags_t zsflags = zone_security_config(z); in zone_mapping_type()
4784 zone_expand_locked(zone_t z, zalloc_flags_t flags) in zone_expand_locked() argument
4786 zone_security_flags_t zsflags = zone_security_config(z); in zone_expand_locked()
4791 if (!(ze.ze_thread->options & TH_OPT_VMPRIV) && zone_supports_vm(z)) { in zone_expand_locked()
4812 if (!z->z_permanent && !zalloc_needs_refill(z, flags)) { in zone_expand_locked()
4816 if (z->z_expander == NULL) { in zone_expand_locked()
4817 z->z_expander = &ze; in zone_expand_locked()
4821 if (ze.ze_vm_priv && !z->z_expander->ze_vm_priv) { in zone_expand_locked()
4822 change_sleep_inheritor(&z->z_expander, ze.ze_thread); in zone_expand_locked()
4823 ze.ze_next = z->z_expander; in zone_expand_locked()
4824 z->z_expander = &ze; in zone_expand_locked()
4828 if ((flags & Z_NOPAGEWAIT) && z->z_expander->ze_pg_wait) { in zone_expand_locked()
4832 z->z_expanding_wait = true; in zone_expand_locked()
4833 hw_lck_ticket_sleep_with_inheritor(&z->z_lock, &zone_locks_grp, in zone_expand_locked()
4834 LCK_SLEEP_DEFAULT, &z->z_expander, z->z_expander->ze_thread, in zone_expand_locked()
4845 if ((flags & Z_NOFAIL) && zone_exhausted(z)) { in zone_expand_locked()
4846 __ZONE_EXHAUSTED_AND_WAITING_HARD__(z); in zone_expand_locked()
4859 if (zone_pva_is_null(z->z_pageq_va)) { in zone_expand_locked()
4860 zone_allocate_va_locked(z, flags); in zone_expand_locked()
4863 meta = zone_meta_queue_pop(z, &z->z_pageq_va); in zone_expand_locked()
4869 zone_meta_lock_in_partial(z, meta, cur_pages); in zone_expand_locked()
4871 zone_unlock(z); in zone_expand_locked()
4876 min_pages = z->z_chunk_pages; in zone_expand_locked()
4878 if (!z->z_percpu) { in zone_expand_locked()
4879 min_pages = (uint32_t)atop(round_page(zone_elem_outer_offs(z) + in zone_expand_locked()
4880 zone_elem_outer_size(z))); in zone_expand_locked()
4889 __ZONE_MAP_EXHAUSTED_AND_WAITING_FOR_GC__(z, min_pages); in zone_expand_locked()
4892 ZONE_TRACE_VM_KERN_REQUEST_START(ptoa(z->z_chunk_pages - cur_pages)); in zone_expand_locked()
4894 while (pages < z->z_chunk_pages - cur_pages) { in zone_expand_locked()
4920 zone_lock(z); in zone_expand_locked()
4921 if (z->z_expanding_wait) { in zone_expand_locked()
4922 z->z_expanding_wait = false; in zone_expand_locked()
4923 wakeup_all_with_inheritor(&z->z_expander, in zone_expand_locked()
4927 zone_unlock(z); in zone_expand_locked()
4945 zone_lock(z); in zone_expand_locked()
4947 zone_expand_async_schedule_if_allowed(z); in zone_expand_locked()
4950 zone_meta_unlock_from_partial(z, meta, cur_pages); in zone_expand_locked()
4953 zone_meta_queue_push(z, &z->z_pageq_va, in zone_expand_locked()
4964 zone_kma_flags(z, zsflags, flags), VM_KERN_MEMORY_ZONE, in zone_expand_locked()
4965 zone_page_prot(zsflags), zone_mapping_type(z)); in zone_expand_locked()
4969 zcram_and_lock(z, addr, new_va, cur_pages, cur_pages + pages, 0); in zone_expand_locked()
4975 } while (!z->z_permanent && zalloc_needs_refill(z, flags)); in zone_expand_locked()
4978 if (z->z_expander == &ze) { in zone_expand_locked()
4979 z->z_expander = ze.ze_next; in zone_expand_locked()
4981 assert(z->z_expander->ze_next == &ze); in zone_expand_locked()
4982 z->z_expander->ze_next = NULL; in zone_expand_locked()
4984 if (z->z_expanding_wait) { in zone_expand_locked()
4985 z->z_expanding_wait = false; in zone_expand_locked()
4986 wakeup_all_with_inheritor(&z->z_expander, THREAD_AWAKENED); in zone_expand_locked()
4997 zone_foreach(z) { in zone_expand_async()
4998 if (z->no_callout) { in zone_expand_async()
5003 if (!z->z_async_refilling) { in zone_expand_async()
5014 zone_lock(z); in zone_expand_async()
5015 if (z->z_self && z->z_async_refilling) { in zone_expand_async()
5016 zone_expand_locked(z, Z_WAITOK); in zone_expand_async()
5022 z->z_async_refilling = false; in zone_expand_async()
5024 zone_unlock(z); in zone_expand_async()
5526 pgz_zone_init(zone_t z) in pgz_zone_init() argument
5532 if (zone_elem_inner_size(z) > PAGE_SIZE) { in pgz_zone_init()
5538 z->z_pgz_tracked = true; in pgz_zone_init()
5542 snprintf(zn, sizeof(zn), "%s%s", zone_heap_name(z), zone_name(z)); in pgz_zone_init()
5549 if (track_this_zone(zn, zv) || track_kalloc_zones(z, zv)) { in pgz_zone_init()
5551 z->z_pgz_tracked = true; in pgz_zone_init()
5690 zone_t z; in panic_display_pgz_uaf_info() local
5701 z = &zone_array[meta->zm_index]; in panic_display_pgz_uaf_info()
5709 esize = zone_elem_inner_size(z); in panic_display_pgz_uaf_info()
5711 zone_heap_name(z), zone_name(z)); in panic_display_pgz_uaf_info()
6376 zone_t z = kt_view->kt_zv.zv_zone; in kfree_type_impl_internal() local
6380 zone_security_flags_t zsflags_z = zone_security_config(z); in kfree_type_impl_internal()
6392 zone_has_index(z, zidx_meta)) { in kfree_type_impl_internal()
6397 z = zone_array + zidx_meta; in kfree_type_impl_internal()
6398 return (zfree)(z, ptr); in kfree_type_impl_internal()
6733 zalloc_cached_reuse_smr(zone_t z, zone_cache_t cache, zone_magazine_t mag) in zalloc_cached_reuse_smr() argument
6736 vm_size_t esize = zone_elem_inner_size(z); in zalloc_cached_reuse_smr()
6741 zc_free((void *)elem, zone_elem_inner_size(z)); in zalloc_cached_reuse_smr()
6742 elem = __zcache_mark_invalid(z, elem, in zalloc_cached_reuse_smr()
7454 zone_t z, in zone_reclaim_chunk() argument
7462 zone_security_flags_t zsflags = zone_security_config(z); in zone_reclaim_chunk()
7463 bool sequester = !z->z_destroyed; in zone_reclaim_chunk()
7474 zone_meta_queue_pop(z, &z->z_pageq_empty); in zone_reclaim_chunk()
7481 zone_metadata_corruption(z, meta, "alloc_size"); in zone_reclaim_chunk()
7483 if (z->z_percpu) { in zone_reclaim_chunk()
7485 zone_metadata_corruption(z, meta, "page_count"); in zone_reclaim_chunk()
7487 size_to_free = ptoa(z->z_chunk_pages); in zone_reclaim_chunk()
7488 zone_remove_wired_pages(z, z->z_chunk_pages); in zone_reclaim_chunk()
7490 if (page_count > z->z_chunk_pages) { in zone_reclaim_chunk()
7491 zone_metadata_corruption(z, meta, "page_count"); in zone_reclaim_chunk()
7493 if (page_count < z->z_chunk_pages) { in zone_reclaim_chunk()
7495 zone_meta_remqueue(z, meta + page_count); in zone_reclaim_chunk()
7498 zone_remove_wired_pages(z, page_count); in zone_reclaim_chunk()
7501 zone_counter_sub(z, z_elems_free, free_count); in zone_reclaim_chunk()
7502 zone_counter_sub(z, z_elems_avail, free_count); in zone_reclaim_chunk()
7503 zone_counter_sub(z, z_wired_empty, page_count); in zone_reclaim_chunk()
7504 zone_counter_sub(z, z_wired_cur, page_count); in zone_reclaim_chunk()
7506 if (z->z_pcpu_cache == NULL) { in zone_reclaim_chunk()
7507 if (z->z_elems_free_min < free_count) { in zone_reclaim_chunk()
7508 z->z_elems_free_min = 0; in zone_reclaim_chunk()
7510 z->z_elems_free_min -= free_count; in zone_reclaim_chunk()
7513 if (z->z_elems_free_wma < free_count) { in zone_reclaim_chunk()
7514 z->z_elems_free_wma = 0; in zone_reclaim_chunk()
7516 z->z_elems_free_wma -= free_count; in zone_reclaim_chunk()
7534 zone_counter_sub(z, z_va_cur, z->z_percpu ? 1 : z->z_chunk_pages); in zone_reclaim_chunk()
7535 bzero(meta, sizeof(*meta) * (z->z_chunk_pages + oob_guard)); in zone_reclaim_chunk()
7539 if (__improbable(zleak_should_disable_for_zone(z) && in zone_reclaim_chunk()
7545 zone_unlock(z); in zone_reclaim_chunk()
7553 if (z->z_percpu) { in zone_reclaim_chunk()
7554 for (uint32_t i = 0; i < z->z_chunk_pages; i++) { in zone_reclaim_chunk()
7556 zone_elem_outer_size(z), in zone_reclaim_chunk()
7557 zone_elem_outer_offs(z), in zone_reclaim_chunk()
7558 zone_elem_redzone(z)); in zone_reclaim_chunk()
7562 zone_elem_outer_size(z), in zone_reclaim_chunk()
7563 zone_elem_outer_offs(z), in zone_reclaim_chunk()
7564 zone_elem_redzone(z)); in zone_reclaim_chunk()
7569 kma_flags_t flags = zone_kma_flags(z, zsflags, 0) | KMA_KOBJECT; in zone_reclaim_chunk()
7575 ptoa(z->z_chunk_pages + oob_guard)); in zone_reclaim_chunk()
7583 zone_lock(z); in zone_reclaim_chunk()
7586 zone_meta_queue_push(z, &z->z_pageq_va, meta); in zone_reclaim_chunk()
7591 zone_reclaim_elements(zone_t z, uint16_t n, vm_offset_t *elems) in zone_reclaim_elements() argument
7598 zfree_drop(z, addr); in zone_reclaim_elements()
7601 z->z_elems_free += n; in zone_reclaim_elements()
7621 zone_depot_trim(zone_t z, uint32_t target, struct zone_depot *zd) in zone_depot_trim() argument
7623 zpercpu_foreach(zc, z->z_pcpu_cache) { in zone_depot_trim()
7647 zone_reclaim_pcpu(zone_t z, zone_reclaim_mode_t mode, struct zone_depot *zd) in zone_reclaim_pcpu() argument
7652 if (z->z_depot_cleanup) { in zone_reclaim_pcpu()
7653 z->z_depot_cleanup = false; in zone_reclaim_pcpu()
7654 depot_max = z->z_depot_size; in zone_reclaim_pcpu()
7659 zone_depot_trim(z, depot_max, zd); in zone_reclaim_pcpu()
7663 zpercpu_foreach(zc, z->z_pcpu_cache) { in zone_reclaim_pcpu()
7664 zone_reclaim_elements(z, zc->zc_alloc_cur, in zone_reclaim_pcpu()
7666 zone_reclaim_elements(z, zc->zc_free_cur, in zone_reclaim_pcpu()
7671 z->z_recirc_empty_min = 0; in zone_reclaim_pcpu()
7672 z->z_recirc_empty_wma = 0; in zone_reclaim_pcpu()
7673 z->z_recirc_full_min = 0; in zone_reclaim_pcpu()
7674 z->z_recirc_full_wma = 0; in zone_reclaim_pcpu()
7675 z->z_recirc_cont_cur = 0; in zone_reclaim_pcpu()
7676 z->z_recirc_cont_wma = 0; in zone_reclaim_pcpu()
7681 zone_reclaim_recirc_drain(zone_t z, struct zone_depot *zd) in zone_reclaim_recirc_drain() argument
7686 zone_recirc_lock_nopreempt(z); in zone_reclaim_recirc_drain()
7688 *zd = z->z_recirc; in zone_reclaim_recirc_drain()
7692 zone_depot_init(&z->z_recirc); in zone_reclaim_recirc_drain()
7693 z->z_recirc_empty_min = 0; in zone_reclaim_recirc_drain()
7694 z->z_recirc_empty_wma = 0; in zone_reclaim_recirc_drain()
7695 z->z_recirc_full_min = 0; in zone_reclaim_recirc_drain()
7696 z->z_recirc_full_wma = 0; in zone_reclaim_recirc_drain()
7698 zone_recirc_unlock_nopreempt(z); in zone_reclaim_recirc_drain()
7702 zone_reclaim_recirc_trim(zone_t z, struct zone_depot *zd) in zone_reclaim_recirc_trim() argument
7709 zone_recirc_lock_nopreempt(z); in zone_reclaim_recirc_trim()
7710 count = MIN(z->z_recirc_empty_wma / Z_WMA_UNIT, in zone_reclaim_recirc_trim()
7711 z->z_recirc_empty_min); in zone_reclaim_recirc_trim()
7712 assert(count <= z->z_recirc.zd_empty); in zone_reclaim_recirc_trim()
7720 zone_depot_move_empty(zd, &z->z_recirc, count, NULL); in zone_reclaim_recirc_trim()
7721 z->z_recirc_empty_min -= count; in zone_reclaim_recirc_trim()
7722 z->z_recirc_empty_wma -= count * Z_WMA_UNIT; in zone_reclaim_recirc_trim()
7725 count = MIN(z->z_recirc_full_wma / Z_WMA_UNIT, in zone_reclaim_recirc_trim()
7726 z->z_recirc_full_min); in zone_reclaim_recirc_trim()
7727 assert(count <= z->z_recirc.zd_full); in zone_reclaim_recirc_trim()
7734 zone_depot_move_full(zd, &z->z_recirc, count, NULL); in zone_reclaim_recirc_trim()
7735 z->z_recirc_full_min -= count; in zone_reclaim_recirc_trim()
7736 z->z_recirc_full_wma -= count * Z_WMA_UNIT; in zone_reclaim_recirc_trim()
7739 zone_recirc_unlock_nopreempt(z); in zone_reclaim_recirc_trim()
7751 zone_unlock(z); in zone_reclaim_recirc_trim()
7752 zone_lock(z); in zone_reclaim_recirc_trim()
7772 zone_reclaim(zone_t z, zone_reclaim_mode_t mode) in zone_reclaim() argument
7778 zone_lock(z); in zone_reclaim()
7781 if (!z->z_destructible || z->z_elems_rsv) { in zone_reclaim()
7783 zone_heap_name(z), z->z_name); in zone_reclaim()
7786 if (!z->z_self || z->z_expander || in zone_reclaim()
7787 z->z_async_refilling || z->z_expanding_wait) { in zone_reclaim()
7789 zone_heap_name(z), z->z_name); in zone_reclaim()
7800 z->z_self = NULL; in zone_reclaim()
7802 z->z_destroyed = true; in zone_reclaim()
7803 } else if (z->z_destroyed) { in zone_reclaim()
7804 return zone_unlock(z); in zone_reclaim()
7805 } else if (zone_count_free(z) <= z->z_elems_rsv) { in zone_reclaim()
7807 return zone_unlock(z); in zone_reclaim()
7810 if (z->z_pcpu_cache) { in zone_reclaim()
7825 zone_reclaim_recirc_trim(z, &zd); in zone_reclaim()
7827 zone_reclaim_recirc_drain(z, &zd); in zone_reclaim()
7829 zone_reclaim_pcpu(z, mode, &zd); in zone_reclaim()
7831 if (z->z_chunk_elems) { in zone_reclaim()
7832 zone_cache_t cache = zpercpu_get_cpu(z->z_pcpu_cache, 0); in zone_reclaim()
7839 zalloc_cached_reuse_smr(z, cache, mag); in zone_reclaim()
7842 zone_reclaim_elements(z, zc_mag_size(), in zone_reclaim()
7848 zone_unlock(z); in zone_reclaim()
7851 zone_lock(z); in zone_reclaim()
7856 zone_id_t zid = zone_index(z); in zone_reclaim()
7858 zone_unlock(z); in zone_reclaim()
7873 while (!zone_pva_is_null(z->z_pageq_empty)) { in zone_reclaim()
7875 uint32_t count, limit = z->z_elems_rsv * 5 / 4; in zone_reclaim()
7877 if (mode == ZONE_RECLAIM_TRIM && z->z_pcpu_cache == NULL) { in zone_reclaim()
7878 limit = MAX(limit, z->z_elems_free - in zone_reclaim()
7879 MIN(z->z_elems_free_min, z->z_elems_free_wma)); in zone_reclaim()
7882 meta = zone_pva_to_meta(z->z_pageq_empty); in zone_reclaim()
7883 count = (uint32_t)ptoa(meta->zm_chunk_len) / zone_elem_outer_size(z); in zone_reclaim()
7885 if (zone_count_free(z) - count < limit) { in zone_reclaim()
7889 zone_reclaim_chunk(z, meta, count); in zone_reclaim()
7892 zone_unlock(z); in zone_reclaim()
7925 zone_t z = zone_by_id(zid); in zone_reclaim_all() local
7927 if (z->z_self && z->z_chunk_elems == 0) { in zone_reclaim_all()
7928 zone_reclaim(z, mode); in zone_reclaim_all()
7932 zone_t z = zone_by_id(zid); in zone_reclaim_all() local
7934 if (z == zc_magazine_zone || z->z_chunk_elems == 0) { in zone_reclaim_all()
7938 z->collectable) { in zone_reclaim_all()
7939 zone_reclaim(z, mode); in zone_reclaim_all()
7944 zone_t z = zone_by_id(zid); in zone_reclaim_all() local
7946 if (z == zc_magazine_zone || z->z_chunk_elems == 0) { in zone_reclaim_all()
7950 z->collectable) { in zone_reclaim_all()
7951 zone_reclaim(z, mode); in zone_reclaim_all()
8018 zone_trim_needed(zone_t z) in zone_trim_needed() argument
8020 if (z->z_depot_cleanup) { in zone_trim_needed()
8024 if (z->z_async_refilling) { in zone_trim_needed()
8029 if (z->z_pcpu_cache) { in zone_trim_needed()
8032 e_n = MIN(z->z_recirc_empty_wma, z->z_recirc_empty_min * Z_WMA_UNIT); in zone_trim_needed()
8033 f_n = MIN(z->z_recirc_full_wma, z->z_recirc_full_min * Z_WMA_UNIT); in zone_trim_needed()
8039 if (f_n * zc_mag_size() > z->z_elems_rsv * Z_WMA_UNIT && in zone_trim_needed()
8040 f_n * zc_mag_size() * zone_elem_inner_size(z) > in zone_trim_needed()
8048 if (!zone_pva_is_null(z->z_pageq_empty)) { in zone_trim_needed()
8051 n = MIN(z->z_elems_free_wma, z->z_elems_free_min); in zone_trim_needed()
8053 return n >= z->z_elems_rsv + z->z_chunk_elems; in zone_trim_needed()
8064 zone_foreach(z) { in zone_trim_async()
8065 if (!z->collectable || z == zc_magazine_zone) { in zone_trim_async()
8069 if (zone_trim_needed(z)) { in zone_trim_async()
8071 zone_reclaim(z, ZONE_RECLAIM_TRIM); in zone_trim_async()
8104 zone_foreach(z) { in compute_zone_working_set_size()
8108 if (z->z_self != z) { in compute_zone_working_set_size()
8112 zone_lock(z); in compute_zone_working_set_size()
8114 zone_recirc_lock_nopreempt(z); in compute_zone_working_set_size()
8116 if (z->z_pcpu_cache) { in compute_zone_working_set_size()
8117 wma = Z_WMA_MIX(z->z_recirc_empty_wma, z->z_recirc_empty_min); in compute_zone_working_set_size()
8118 z->z_recirc_empty_min = z->z_recirc.zd_empty; in compute_zone_working_set_size()
8119 z->z_recirc_empty_wma = wma; in compute_zone_working_set_size()
8121 wma = Z_WMA_MIX(z->z_elems_free_wma, z->z_elems_free_min); in compute_zone_working_set_size()
8122 z->z_elems_free_min = z->z_elems_free; in compute_zone_working_set_size()
8123 z->z_elems_free_wma = wma; in compute_zone_working_set_size()
8126 wma = Z_WMA_MIX(z->z_recirc_full_wma, z->z_recirc_full_min); in compute_zone_working_set_size()
8127 z->z_recirc_full_min = z->z_recirc.zd_full; in compute_zone_working_set_size()
8128 z->z_recirc_full_wma = wma; in compute_zone_working_set_size()
8131 old = z->z_recirc_cont_wma; in compute_zone_working_set_size()
8132 cur = z->z_recirc_cont_cur * Z_WMA_UNIT / in compute_zone_working_set_size()
8135 zone_recirc_unlock_nopreempt(z); in compute_zone_working_set_size()
8137 if (z->z_pcpu_cache) { in compute_zone_working_set_size()
8138 uint16_t size = z->z_depot_size; in compute_zone_working_set_size()
8140 if (zone_exhausted(z)) { in compute_zone_working_set_size()
8141 if (z->z_depot_size) { in compute_zone_working_set_size()
8142 z->z_depot_size = 0; in compute_zone_working_set_size()
8143 z->z_depot_cleanup = true; in compute_zone_working_set_size()
8145 } else if (size < z->z_depot_limit && cur > zc_grow_level()) { in compute_zone_working_set_size()
8153 z->z_depot_size = MIN(z->z_depot_limit, size); in compute_zone_working_set_size()
8161 z->z_depot_size = size - 1; in compute_zone_working_set_size()
8162 z->z_depot_cleanup = true; in compute_zone_working_set_size()
8164 } else if (!z->z_nocaching && !zone_exhaustible(z) && zc_auto && in compute_zone_working_set_size()
8169 z->z_recirc_cont_wma = cur; in compute_zone_working_set_size()
8170 z->z_recirc_cont_cur = 0; in compute_zone_working_set_size()
8172 if (!needs_trim && zone_trim_needed(z)) { in compute_zone_working_set_size()
8176 zone_unlock(z); in compute_zone_working_set_size()
8179 zone_enable_caching(z); in compute_zone_working_set_size()
8211 panic_print_types_in_zone(zone_t z, const char* debug_str) in panic_print_types_in_zone() argument
8216 zone_security_flags_t zsflags = zone_security_config(z); in panic_print_types_in_zone()
8220 uint32_t heap_id = KT_VAR_PTR_HEAP0 + ((zone_index(z) - in panic_print_types_in_zone()
8225 kt_cur.ktv_fixed = (kalloc_type_view_t) z->z_views; in panic_print_types_in_zone()
8229 is_kt_var? "type arrays" : "types", debug_str, z->z_name); in panic_print_types_in_zone()
8261 zone_t z = &zone_array[zid]; in zone_find_n_largest() local
8262 vm_offset_t size = zone_size_wired(z); in zone_find_n_largest()
8269 largest_zones[i] = z; in zone_find_n_largest()
8290 zone_t z = largest_zones[i]; in panic_display_largest_zones() local
8292 zone_heap_name(z), z->z_name, in panic_display_largest_zones()
8295 mach_vm_size_pretty(zone_size_free(z)), in panic_display_largest_zones()
8296 mach_vm_size_unit(zone_size_free(z))); in panic_display_largest_zones()
8373 zone_t z = NULL; in panic_display_zone_fault() local
8409 z = &zone_array[meta.zm_index]; in panic_display_zone_fault()
8411 zone_heap_name(z), zone_name(z)); in panic_display_zone_fault()
8416 &z, false, &oob_offs); in panic_display_zone_fault()
8517 zone_t z, in get_zone_info() argument
8524 assert(z != ZONE_NULL); in get_zone_info()
8525 zone_lock(z); in get_zone_info()
8526 if (!z->z_self) { in get_zone_info()
8527 zone_unlock(z); in get_zone_info()
8530 zcopy = *z; in get_zone_info()
8531 if (z->z_pcpu_cache) { in get_zone_info()
8532 zpercpu_foreach(zc, z->z_pcpu_cache) { in get_zone_info()
8537 zone_unlock(z); in get_zone_info()
8545 zone_heap_name(z), z->z_name); in get_zone_info()
8966 zone_foreach(z) { in mach_zone_info_for_zone()
8972 zone_heap_name(z), z->z_name); in mach_zone_info_for_zone()
8976 zone_ptr = z; in mach_zone_info_for_zone()
9024 zone_foreach(z) { in get_zones_collectable_bytes()
9025 if (get_zone_info(z, NULL, &zi)) { in get_zones_collectable_bytes()
9069 zone_t z = &(zone_array[i]); in mach_zone_get_zlog_zones() local
9070 assert(z != ZONE_NULL); in mach_zone_get_zlog_zones()
9073 if (z->z_btlog) { in mach_zone_get_zlog_zones()
9074 get_zone_info(z, &names[logged_zones], NULL); in mach_zone_get_zlog_zones()
9114 zone_foreach(z) { in mach_zone_get_btlog_records()
9120 zone_heap_name(z), z->z_name); in mach_zone_get_btlog_records()
9124 zone_ptr = z; in mach_zone_get_btlog_records()
9232 zone_t z = &zone_array[zid]; in zone_init_defaults() local
9234 z->z_wired_max = ~0u; in zone_init_defaults()
9235 z->collectable = true; in zone_init_defaults()
9237 hw_lck_ticket_init(&z->z_lock, &zone_locks_grp); in zone_init_defaults()
9238 hw_lck_ticket_init(&z->z_recirc_lock, &zone_locks_grp); in zone_init_defaults()
9239 zone_depot_init(&z->z_recirc); in zone_init_defaults()
9240 return z; in zone_init_defaults()
9287 zone_t z; in zone_create_find() local
9318 z = &zone_array[zid]; in zone_create_find()
9327 z = &zone_array[i]; in zone_create_find()
9333 if (strcmp(z->z_name, name) || in zone_create_find()
9334 zone_elem_outer_size(z) != size) { in zone_create_find()
9338 z->z_destroyed = false; in zone_create_find()
9339 z->z_self = z; in zone_create_find()
9346 z = zone_init_defaults(zid); in zone_create_find()
9361 return z; in zone_create_find()
9499 zone_t z; variable
9515 z = zone_create_find(name, size, flags, &zid);
9516 if (__improbable(z->z_self)) {
9518 return z;
9535 z->z_name = buf;
9537 z->z_name = name;
9540 z->z_stats = zalloc_percpu_permanent_type(struct zone_stats);
9547 z->z_stats = __zpcpu_mangle_for_boot(&zone_stats_startup[zone_index(z)]);
9556 z->z_elem_size = (uint16_t)size;
9557 z->z_chunk_pages = 0;
9558 z->z_quo_magic = 0;
9559 z->z_align_magic = 0;
9560 z->z_chunk_elems = 0;
9561 z->z_elem_offs = 0;
9562 z->no_callout = true;
9567 z->z_elem_size = (uint16_t)(size - redzone);
9568 z->z_chunk_pages = (uint16_t)atop(alloc);
9569 z->z_quo_magic = Z_MAGIC_QUO(size);
9570 z->z_align_magic = Z_MAGIC_ALIGNED(size);
9572 z->z_chunk_elems = (uint16_t)(PAGE_SIZE / size);
9573 z->z_elem_offs = (uint16_t)(PAGE_SIZE % size) + redzone;
9575 z->z_chunk_elems = (uint16_t)(alloc / size);
9576 z->z_elem_offs = (uint16_t)(alloc % size) + redzone;
9586 z->z_nocaching = true;
9596 zone_ro_size_params[zid].z_elem_size = z->z_elem_size;
9597 zone_ro_size_params[zid].z_align_magic = z->z_align_magic;
9601 (uint32_t)size, zone_heap_name(z), z->z_name);
9608 z->z_percpu = true;
9611 z->collectable = false;
9620 z->no_callout = true;
9624 z->z_destructible = true;
9638 z->z_pgz_use_guards = true;
9661 z->z_kasan_quarantine = true;
9663 z->z_kasan_redzone = redzone;
9667 z->z_kasan_fakestacks = true;
9675 extra_setup(z);
9683 pgz_zone_init(z);
9692 zone_setup_logging(z);
9695 zone_setup_kasan_logging(z);
9704 z->z_uses_tags = true;
9707 z->z_tags_sizeclass = sizeclass_idx++;
9712 z->z_tags_sizeclass = i;
9723 z->z_tags_sizeclass = sizeclass_idx++;
9726 assert(z->z_tags_sizeclass < VM_TAG_SIZECLASSES);
9745 if ((flags & (ZC_CACHING | ZC_OBJ_CACHE)) && !z->z_nocaching) {
9750 zone_enable_caching(z);
9753 zone_lock(z);
9754 z->z_self = z;
9755 zone_unlock(z);
9757 return z;
9799 zone_t z; in zone_create_startup() local
9801 z = zone_create_ext(spec->z_name, spec->z_size, in zone_create_startup()
9804 *spec->z_var = z; in zone_create_startup()
9826 zone_t z; in zone_view_startup_init() local
9838 z = kalloc_zone_for_size(heap->kh_zstart, spec->zv_size); in zone_view_startup_init()
9840 z = *spec->zv_zone; in zone_view_startup_init()
9841 assert(spec->zv_size <= zone_elem_inner_size(z)); in zone_view_startup_init()
9844 assert(z); in zone_view_startup_init()
9846 zv->zv_zone = z; in zone_view_startup_init()
9848 zv->zv_next = z->z_views; in zone_view_startup_init()
9849 zsflags = zone_security_config(z); in zone_view_startup_init()
9850 if (z->z_views == NULL && zsflags.z_kheap_id == KHEAP_ID_NONE) { in zone_view_startup_init()
9859 z->z_views = zv; in zone_view_startup_init()
9903 zdestroy(zone_t z) in zdestroy() argument
9905 unsigned int zindex = zone_index(z); in zdestroy()
9911 zone_reclaim(z, ZONE_RECLAIM_DESTROY); in zdestroy()
9916 zone_lock(z); in zdestroy()
9919 while (!zone_pva_is_null(z->z_pageq_va)) { in zdestroy()
9922 zone_counter_sub(z, z_va_cur, z->z_percpu ? 1 : z->z_chunk_pages); in zdestroy()
9923 meta = zone_meta_queue_pop(z, &z->z_pageq_va); in zdestroy()
9925 bzero(meta, sizeof(*meta) * z->z_chunk_pages); in zdestroy()
9926 zone_unlock(z); in zdestroy()
9928 ptoa(z->z_chunk_pages)); in zdestroy()
9929 zone_lock(z); in zdestroy()
9935 if (z->z_elems_avail || z->z_elems_free || zone_size_wired(z) || in zdestroy()
9936 (z->z_va_cur && !zone_submap_is_sequestered(zsflags))) { in zdestroy()
9938 zone_heap_name(z), z->z_name); in zdestroy()
9942 assert(zone_pva_is_null(z->z_pageq_empty)); in zdestroy()
9943 assert(zone_pva_is_null(z->z_pageq_partial)); in zdestroy()
9944 assert(zone_pva_is_null(z->z_pageq_full)); in zdestroy()
9946 assert(zone_pva_is_null(z->z_pageq_va)); in zdestroy()
9950 zone_unlock(z); in zdestroy()
10313 zone_t z = &zone_array[zid]; in zone_metadata_init() local
10314 vm_size_t esize = zone_elem_outer_size(z); in zone_metadata_init()
10315 vm_address_t base = reloc_base + ptoa(i) + zone_elem_inner_offs(z); in zone_metadata_init()
10322 for (uint32_t eidx = 0; eidx < z->z_chunk_elems; eidx++) { in zone_metadata_init()
10330 zone_elem_inner_size(z), zone_elem_inner_size(z), in zone_metadata_init()
10331 zone_elem_redzone(z), false, in zone_metadata_init()
10531 ZONE_ID_PERMANENT, ^(zone_t z) { in zone_init()
10532 z->z_permanent = true; in zone_init()
10533 z->z_elem_size = 1; in zone_init()
10536 kma_flags | ZC_PERCPU | ZC_NO_TBI_TAG, ZONE_ID_PERCPU_PERMANENT, ^(zone_t z) { in zone_init()
10537 z->z_permanent = true; in zone_init()
10538 z->z_elem_size = 1; in zone_init()
10638 zone_copy_allocations(zone_t z, uintptr_t *elems, zone_pva_t page_index) in zone_copy_allocations() argument
10640 vm_offset_t elem_size = zone_elem_outer_size(z); in zone_copy_allocations()
10645 base = zone_pva_to_addr(page_index) + zone_elem_inner_offs(z); in zone_copy_allocations()
10689 zone_foreach(z) { in zone_leaks()
10690 if (!z->z_name) { in zone_leaks()
10693 if (!strncmp(zoneName, z->z_name, nameLen)) { in zone_leaks()
10694 zone = z; in zone_leaks()
10950 zone_foreach(z) { in zone_basic_test_run()
10951 zone_lock(z); in zone_basic_test_run()
10952 allva += z->z_wired_cur; in zone_basic_test_run()
10953 if (zone_pva_is_null(z->z_pageq_va)) { in zone_basic_test_run()
10954 zone_unlock(z); in zone_basic_test_run()
10959 zone_pva_t pg = z->z_pageq_va; in zone_basic_test_run()
10963 count += z->z_percpu ? 1 : z->z_chunk_pages; in zone_basic_test_run()
10969 size = zone_size_wired(z); in zone_basic_test_run()
10974 zone_heap_name(z), z->z_name, z->z_va_cur - z->z_wired_cur, in zone_basic_test_run()
10975 z->z_wired_cur, zone_size_allocated(z) * 100ULL / size); in zone_basic_test_run()
10976 zone_unlock(z); in zone_basic_test_run()
11328 zone_t z = vm_map_entry_zone; in zone_alloc_replenish_test() local
11331 if (z == NULL) { in zone_alloc_replenish_test()
11337 for (uint32_t i = 0; i < 10 * z->z_elems_rsv; ++i) { in zone_alloc_replenish_test()
11338 node = zalloc(z); in zone_alloc_replenish_test()
11349 zfree(z, node); in zone_alloc_replenish_test()