324 	void                   *addr,  in kheap_size_from_zone()  argument
339 		    zone_id_for_element(addr, size) != ZONE_ID_INVALID) {  in kheap_size_from_zone()
346 #pragma unused(addr, forcopyin)  in kheap_size_from_zone()
652 kalloc_type_slide_fixed(vm_offset_t addr)  in kalloc_type_slide_fixed()  argument
654 	kalloc_type_view_t ktv = (struct kalloc_type_view *) addr;  in kalloc_type_slide_fixed()
662 kalloc_type_slide_var(vm_offset_t addr)  in kalloc_type_slide_var()  argument
664 	kalloc_type_var_view_t ktv = (struct kalloc_type_var_view *) addr;  in kalloc_type_slide_var()
687 kalloc_type_get_flags_fixed(vm_offset_t addr, uuid_string_t kext_uuid)  in kalloc_type_get_flags_fixed()  argument
689 	kalloc_type_view_t ktv = (kalloc_type_view_t) addr;  in kalloc_type_get_flags_fixed()
695 kalloc_type_get_flags_var(vm_offset_t addr, uuid_string_t kext_uuid)  in kalloc_type_get_flags_var()  argument
697 	kalloc_type_var_view_t ktv = (kalloc_type_var_view_t) addr;  in kalloc_type_get_flags_var()
767 kalloc_type_handle_data_view_fixed(vm_offset_t addr)  in kalloc_type_handle_data_view_fixed()  argument
769 	kalloc_type_view_t cur_data_view = (kalloc_type_view_t) addr;  in kalloc_type_handle_data_view_fixed()
778 kalloc_type_handle_data_view_var(vm_offset_t addr)  in kalloc_type_handle_data_view_var()  argument
780 	kalloc_type_var_view_t ktv = (kalloc_type_var_view_t) addr;  in kalloc_type_handle_data_view_var()
851 kalloc_type_set_type_hash_fixed(vm_offset_t addr)  in kalloc_type_set_type_hash_fixed()  argument
857 	(void) addr;  in kalloc_type_set_type_hash_fixed()
862 kalloc_type_set_type_hash_var(vm_offset_t addr)  in kalloc_type_set_type_hash_var()  argument
864 	kalloc_type_var_view_t ktv = (kalloc_type_var_view_t) addr;  in kalloc_type_set_type_hash_var()
871 kalloc_type_mark_processed_fixed(vm_offset_t addr)  in kalloc_type_mark_processed_fixed()  argument
873 	kalloc_type_view_t ktv = (kalloc_type_view_t) addr;  in kalloc_type_mark_processed_fixed()
879 kalloc_type_mark_processed_var(vm_offset_t addr)  in kalloc_type_mark_processed_var()  argument
881 	kalloc_type_var_view_t ktv = (kalloc_type_var_view_t) addr;  in kalloc_type_mark_processed_var()
887 kalloc_type_update_view_fixed(vm_offset_t addr)  in kalloc_type_update_view_fixed()  argument
889 	kalloc_type_view_t ktv = (kalloc_type_view_t) addr;  in kalloc_type_update_view_fixed()
895 kalloc_type_update_view_var(vm_offset_t addr)  in kalloc_type_update_view_var()  argument
897 	(void) addr;  in kalloc_type_update_view_var()
1001 				kalloc_type_view_copy(type, sect->addr, sect->addr + sect->size,  in kalloc_type_view_parse()
1032 		cur = prelink_sect->addr;  in kalloc_type_view_parse()
1033 		end = prelink_sect->addr + prelink_sect->size;  in kalloc_type_view_parse()
1077 						kalloc_type_view_copy(type, kt_sect->addr + vm_kernel_slide,  in kalloc_type_view_parse()
1078 						    kt_sect->addr + kt_sect->size + vm_kernel_slide, &cur_count,  in kalloc_type_view_parse()
2098 	kr.addr = (void *)ptr;  in __kalloc_array_decode()
2109 __kalloc_array_encode_vm(vm_address_t addr, vm_size_t size)  in __kalloc_array_encode_vm()  argument
2111 	addr &= ~(0x1ull << KALLOC_ARRAY_TYPE_SHIFT);  in __kalloc_array_encode_vm()
2113 	return addr | atop(size);  in __kalloc_array_encode_vm()
2143 	kr.addr = (void *)((long)(ptr << shift) >> shift);  in __kalloc_array_decode()
2151 	vm_address_t addr = (vm_address_t)ptr;  in __kalloc_array_encode_zone()  local
2153 	addr &= (1ull << KALLOC_ARRAY_TYPE_BIT) - 1; /* clear bit */  in __kalloc_array_encode_zone()
2154 	addr |= size << (KALLOC_ARRAY_TYPE_BIT + 1);  in __kalloc_array_encode_zone()
2156 	return (void *)addr;  in __kalloc_array_encode_zone()
2160 __kalloc_array_encode_vm(vm_address_t addr, vm_size_t size)  in __kalloc_array_encode_vm()  argument
2162 	addr &= (2ull << KALLOC_ARRAY_TYPE_BIT) - 1; /* keep bit */  in __kalloc_array_encode_vm()
2163 	addr |= size << (KALLOC_ARRAY_TYPE_BIT + 1 - PAGE_SHIFT);  in __kalloc_array_encode_vm()
2165 	return addr;  in __kalloc_array_encode_vm()
2230 	vm_offset_t addr, size;  in kalloc_large()  local
2297 	addr = kmem_alloc_guard(kernel_map, req_size, 0,  in kalloc_large()
2300 	if (addr != 0) {  in kalloc_large()
2305 			addr = __kalloc_array_encode_vm(addr, req_size);  in kalloc_large()
2308 		addr = 0;  in kalloc_large()
2311 	DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, req_size, void*, addr);  in kalloc_large()
2312 	return (struct kalloc_result){ .addr = (void *)addr, .size = req_size };  in kalloc_large()
2318 kalloc_mark_unused_space(void *addr, vm_size_t size, vm_size_t used)  in kalloc_mark_unused_space()  argument
2325 	kasan_alloc((vm_offset_t)addr, size, used, KASAN_GUARD_SIZE, false,  in kalloc_mark_unused_space()
2330 	kasan_tbi_retag_unused_space((vm_offset_t)addr, size, used ? :1);  in kalloc_mark_unused_space()
2348 	if (__probable(kr.addr)) {  in kalloc_zone()
2355 		kr.addr = zone_element_pgz_oob_adjust(kr.addr, req_size, esize);  in kalloc_zone()
2359 		kalloc_mark_unused_space(kr.addr, esize, kr.size);  in kalloc_zone()
2363 			kr.addr = __kalloc_array_encode_zone(z, kr.addr, kr.size);  in kalloc_zone()
2367 	DTRACE_VM3(kalloc, vm_size_t, req_size, vm_size_t, kr.size, void*, kr.addr);  in kalloc_zone()
2460 kalloc_data_require_panic(void *addr, vm_size_t size)  in kalloc_data_require_panic()  argument
2462 	zone_id_t zid = zone_id_for_element(addr, size);  in kalloc_data_require_panic()
2470 			    addr, zone_heap_name(z), zone_name(z));  in kalloc_data_require_panic()
2474 		    "size too large %zd > %zd", addr,  in kalloc_data_require_panic()
2479 		    addr);  in kalloc_data_require_panic()
2485 kalloc_non_data_require_panic(void *addr, vm_size_t size)  in kalloc_non_data_require_panic()  argument
2487 	zone_id_t zid = zone_id_for_element(addr, size);  in kalloc_non_data_require_panic()
2498 			    addr, zone_heap_name(z), zone_name(z));  in kalloc_non_data_require_panic()
2504 		    "size too large %zd > %zd", addr,  in kalloc_non_data_require_panic()
2509 		    addr);  in kalloc_non_data_require_panic()
2514 kalloc_data_require(void *addr, vm_size_t size)  in kalloc_data_require()  argument
2516 	zone_id_t zid = zone_id_for_element(addr, size);  in kalloc_data_require()
2526 	    (vm_address_t)pgz_decode(addr, size), size)) {  in kalloc_data_require()
2530 	kalloc_data_require_panic(addr, size);  in kalloc_data_require()
2534 kalloc_non_data_require(void *addr, vm_size_t size)  in kalloc_non_data_require()  argument
2536 	zone_id_t zid = zone_id_for_element(addr, size);  in kalloc_non_data_require()
2556 	    (vm_address_t)pgz_decode(addr, size), size)) {  in kalloc_non_data_require()
2560 	kalloc_non_data_require_panic(addr, size);  in kalloc_non_data_require()
2582 		return kalloc_ext(kheap, size, flags, NULL).addr;  in kalloc_type_impl_external()
2666 	void *addr,  in kfree_size_require()  argument
2673 	vm_size_t elem_size = zone_element_size(addr, NULL, false, NULL);  in kfree_size_require()
2675 		kfree_size_require_panic(addr, elem_size, min_size, max_zone_size);  in kfree_size_require()
2681 	vm_offset_t             addr,  in kfree_large()  argument
2686 	size = kmem_free_guard(kernel_map, addr, size,  in kfree_large()
2693 	DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, size, void*, addr);  in kfree_large()
2802 kfree_external(void *addr, vm_size_t size);
2804 kfree_external(void *addr, vm_size_t size)  in kfree_external()  argument
2808 	kfree_ext(kheap, addr, size);  in kfree_external()
2813 (kheap_free_bounded)(kalloc_heap_t kheap, void *addr,
2816 	if (__improbable(addr == NULL)) {
2819 	kfree_size_require(kheap, addr, min_sz, max_sz);
2820 	kfree_addr_ext(kheap, addr);
2832 		return zalloc_ext(z, zs, flags).addr;  in kalloc_type_impl_internal()
2836 	return zalloc_ext(kt_view->kt_zshared, zs, flags | Z_SET_NOTSHARED).addr;  in kalloc_type_impl_internal()
2904 	vm_offset_t           addr,  in krealloc_large()  argument
2977 	kmr = kmem_realloc_guard(kernel_map, addr, old_req_size, new_req_size,  in krealloc_large()
2995 	if (addr != 0 || (flags & Z_REALLOCF)) {  in krealloc_large()
2997 		    void*, addr);  in krealloc_large()
3010 	return (struct kalloc_result){ .addr = kmr.kmr_ptr, .size = new_req_size };  in krealloc_large()
3018 	void                 *addr,  in krealloc_ext()  argument
3035 		krealloc_size_invalid_panic(addr, old_size);  in krealloc_ext()
3038 	if (addr == NULL && new_size == 0) {  in krealloc_ext()
3084 	if (addr == NULL) {  in krealloc_ext()
3087 	} else if (kheap_size_from_zone(addr, old_size, flags)) {  in krealloc_ext()
3088 		old_bucket_size = zone_element_size(addr, &old_z, true, &oob_offs);  in krealloc_ext()
3090 			kfree_size_confusion_panic(old_z, addr,  in krealloc_ext()
3101 		kr.addr = (char *)addr - oob_offs;  in krealloc_ext()
3104 		kr.addr = zone_element_pgz_oob_adjust(kr.addr,  in krealloc_ext()
3106 		if (kr.addr != addr) {  in krealloc_ext()
3107 			memmove(kr.addr, addr, min_size);  in krealloc_ext()
3108 			bzero((char *)kr.addr + min_size,  in krealloc_ext()
3121 		kasan_check_alloc((vm_offset_t)addr, old_bucket_size, old_size);  in krealloc_ext()
3122 		kasan_alloc((vm_offset_t)addr, new_bucket_size, kr.size,  in krealloc_ext()
3130 		__asan_loadN((vm_offset_t)addr, old_size);  in krealloc_ext()
3131 		kr.addr = (void *)vm_memtag_assign_tag((vm_offset_t)kr.addr, kr.size);  in krealloc_ext()
3132 		vm_memtag_set_tag((vm_offset_t)kr.addr, kr.size);  in krealloc_ext()
3133 		kasan_tbi_retag_unused_space((vm_offset_t)kr.addr, new_bucket_size, kr.size);  in krealloc_ext()
3149 		kr.addr = (char *)addr - oob_offs;  in krealloc_ext()
3155 	if (addr && !old_z && new_size && !new_z) {  in krealloc_ext()
3156 		return krealloc_large(kheap, (vm_offset_t)addr,  in krealloc_ext()
3161 		kr.addr = NULL;  in krealloc_ext()
3166 	} else if (old_z || addr == NULL) {  in krealloc_ext()
3171 	if (addr && kr.addr) {  in krealloc_ext()
3172 		__nosan_memcpy(kr.addr, addr, min_size);  in krealloc_ext()
3175 	if (addr && (kr.addr || (flags & Z_REALLOCF) || !new_size)) {  in krealloc_ext()
3178 			    (char *)addr - oob_offs, old_size,  in krealloc_ext()
3181 			kfree_large((vm_offset_t)addr, old_size, KMF_NONE, owner);  in krealloc_ext()
3185 	if (__improbable(kr.addr == NULL)) {  in krealloc_ext()
3195 		kr.addr = __kalloc_array_encode_zone(new_z,  in krealloc_ext()
3196 		    kr.addr, kr.size);  in krealloc_ext()
3198 		kr.addr = (void *)__kalloc_array_encode_vm((vm_offset_t)kr.addr,  in krealloc_ext()
3218 	return krealloc_ext(KHEAP_DATA_BUFFERS, ptr, old_size, new_size, flags, NULL).addr;  in krealloc_data_external()
3294 kern_os_free_external(void *addr);
3296 kern_os_free_external(void *addr)  in kern_os_free_external()  argument
3298 	kheap_free_addr(KERN_OS_MALLOC, addr);  in kern_os_free_external()
3302 kern_os_realloc_external(void *addr, size_t nsize);
3304 kern_os_realloc_external(void *addr, size_t nsize)  in kern_os_realloc_external()  argument
3309 	if (addr == NULL) {  in kern_os_realloc_external()
3313 	osize = zone_element_size(addr, NULL, false, &oob_offs);  in kern_os_realloc_external()
3315 		osize = kmem_size_guard(kernel_map, (vm_offset_t)addr,  in kern_os_realloc_external()
3319 		osize = kasan_user_size((vm_offset_t)addr);  in kern_os_realloc_external()
3322 	return __kheap_realloc(KERN_OS_MALLOC, addr, osize - oob_offs, nsize, flags, NULL);  in kern_os_realloc_external()
3328 kern_os_zfree(zone_t zone, void *addr, vm_size_t size)  in kern_os_zfree()  argument
3332 	zfree(zone, addr);  in kern_os_zfree()
3334 	if (zone_owns(zone, addr)) {  in kern_os_zfree()
3335 		zfree(zone, addr);  in kern_os_zfree()
3343 		kheap_free(KHEAP_DEFAULT, addr, size);  in kern_os_zfree()
3355 kern_os_typed_free(kalloc_type_view_t ktv, void *addr, vm_size_t esize)  in kern_os_typed_free()  argument
3370 	zone_id_t zid = zone_id_for_element(addr, esize);  in kern_os_typed_free()
3374 			return kheap_free(KHEAP_DEFAULT, addr, esize);  in kern_os_typed_free()
3378 	kfree_type_impl_external(ktv, addr);  in kern_os_typed_free()
3573 	    NULL).addr;  in run_kalloc_test()
3581 	    NULL).addr;  in run_kalloc_test()
3583 	data_ptr = krealloc_ext(KHEAP_DATA_BUFFERS, kr.addr, old_alloc_size,  in run_kalloc_test()
3584 	    alloc_size, Z_WAITOK | Z_NOFAIL, NULL).addr;  in run_kalloc_test()
3601 	strippedp_new = (void *)vm_memtag_canonicalize_address((vm_offset_t)kr.addr);  in run_kalloc_test()
3603 	if (!kr.addr || (strippedp_old != strippedp_new) ||  in run_kalloc_test()
3609 	data_ptr = kr.addr;  in run_kalloc_test()
3618 	strippedp_new = (void *)vm_memtag_canonicalize_address((vm_offset_t)kr.addr);  in run_kalloc_test()
3620 	if (!kr.addr || (strippedp_old == strippedp_new) ||  in run_kalloc_test()
3626 	data_ptr = kr.addr;  in run_kalloc_test()
3629 	kheap_free(KHEAP_DATA_BUFFERS, kr.addr, alloc_size);  in run_kalloc_test()
3633 	    Z_WAITOK | Z_FULLSIZE, &data_ptr).addr;  in run_kalloc_test()
3642 	    PAGE_SIZE * 2, Z_REALLOCF | Z_WAITOK, &data_ptr).addr;  in run_kalloc_test()
3650 	    KHEAP_MAX_SIZE * 2, Z_REALLOCF | Z_WAITOK, &data_ptr).addr;  in run_kalloc_test()
3658 	    KHEAP_MAX_SIZE * 4, Z_REALLOCF | Z_WAITOK, &data_ptr).addr;  in run_kalloc_test()