246 	vm_map_entry_t  *entry);
265 	vm_map_entry_t          entry,
270 	vm_map_entry_t          entry,
288 	vm_map_entry_t  entry,
380 	vm_map_entry_t             entry,
1611 		relocate(vm_map_entry_t, store.entry.rbe_left);  in vm_map_relocate_early_elem()
1612 		relocate(vm_map_entry_t, store.entry.rbe_right);  in vm_map_relocate_early_elem()
1613 		relocate(vm_map_entry_t, store.entry.rbe_parent);  in vm_map_relocate_early_elem()
1792 	vm_map_entry_t entry = NULL;  in _vm_map_entry_create()  local
1794 	entry = zalloc_id(ZONE_ID_VM_MAP_ENTRY, Z_WAITOK | Z_ZERO);  in _vm_map_entry_create()
1802 	__builtin_assume(entry->vme_object_value == 0);  in _vm_map_entry_create()
1803 	__builtin_assume(*(uint64_t *)(&entry->vme_object_value + 1) == 0);  in _vm_map_entry_create()
1804 	__builtin_assume(*(uint64_t *)(&entry->vme_object_value + 2) == 0);  in _vm_map_entry_create()
1811 	entry->inheritance = VM_INHERIT_DEFAULT;  in _vm_map_entry_create()
1814 	entry->vme_creation_maphdr = map_header;  in _vm_map_entry_create()
1815 	entry->vme_creation_bt = btref_get(__builtin_frame_address(0),  in _vm_map_entry_create()
1818 	return entry;  in _vm_map_entry_create()
1832 	vm_map_entry_t          entry)  in vm_map_entry_dispose()  argument
1835 	if (entry->vme_kernel_object) {  in vm_map_entry_dispose()
1836 		btref_put(entry->vme_tag_btref);  in vm_map_entry_dispose()
1840 	btref_put(entry->vme_creation_bt);  in vm_map_entry_dispose()
1843 	btref_put(entry->vme_insertion_bt);  in vm_map_entry_dispose()
1845 	zfree(vm_map_entry_zone, entry);  in vm_map_entry_dispose()
1869 	vm_map_entry_t          entry)  in vm_map_zap_append()  argument
1871 	entry->vme_next = VM_MAP_ENTRY_NULL;  in vm_map_zap_append()
1872 	*list->vmz_tail = entry;  in vm_map_zap_append()
1873 	list->vmz_tail = &entry->vme_next;  in vm_map_zap_append()
1894 	vm_map_entry_t          entry;  in vm_map_zap_dispose()  local
1896 	while ((entry = vm_map_zap_pop(list))) {  in vm_map_zap_dispose()
1897 		if (entry->is_sub_map) {  in vm_map_zap_dispose()
1898 			vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_zap_dispose()
1900 			vm_object_deallocate(VME_OBJECT(entry));  in vm_map_zap_dispose()
1903 		vm_map_entry_dispose(entry);  in vm_map_zap_dispose()
1922 #define vm_map_copy_entry_link(copy, after_where, entry)                \  argument
1923 	_vm_map_store_entry_link(&(copy)->cpy_hdr, after_where, (entry))
1925 #define vm_map_copy_entry_unlink(copy, entry)                           \  argument
1926 	_vm_map_store_entry_unlink(&(copy)->cpy_hdr, (entry), false)
2023 	vm_map_entry_t  *entry)         /* OUT */  in vm_map_lookup_entry()  argument
2039 	result = vm_map_store_lookup_entry( map, address, entry );  in vm_map_lookup_entry()
2048 	vm_map_entry_t  *entry)         /* OUT */  in vm_map_lookup_entry_or_next()  argument
2050 	if (vm_map_lookup_entry(map, address, entry)) {  in vm_map_lookup_entry_or_next()
2054 	*entry = (*entry)->vme_next;  in vm_map_lookup_entry_or_next()
2063 	vm_map_entry_t  *entry)         /* OUT */  in vm_map_lookup_entry_allow_pgz()  argument
2071 	return vm_map_store_lookup_entry( map, address, entry );  in vm_map_lookup_entry_allow_pgz()
2202 	vm_map_entry_t  entry;  in vm_map_locate_space_anywhere()  local
2303 	entry = vm_map_store_find_space(map,  in vm_map_locate_space_anywhere()
2308 	if (__improbable(entry == NULL)) {  in vm_map_locate_space_anywhere()
2322 		*entry_out = entry;  in vm_map_locate_space_anywhere()
2366 	vm_map_entry_t  entry;  in vm_map_locate_space_fixed()  local
2478 	if (vm_map_lookup_entry(map, start, &entry)) {  in vm_map_locate_space_fixed()
2479 		*entry_out = entry;  in vm_map_locate_space_fixed()
2488 	if ((entry->vme_next != vm_map_to_entry(map)) &&  in vm_map_locate_space_fixed()
2489 	    (entry->vme_next->vme_start < end)) {  in vm_map_locate_space_fixed()
2493 	*entry_out = entry;  in vm_map_locate_space_fixed()
2519 	vm_map_entry_t          new_entry, entry;  in vm_map_find_space()  local
2541 	    &hint_address, &entry);  in vm_map_find_space()
2571 	vm_map_store_entry_link(map, entry, new_entry,  in vm_map_find_space()
2792 	vm_map_entry_t          entry, new_entry;  in vm_map_enter()  local
3113 		    address, &entry);  in vm_map_enter()
3118 		    vmk_flags, &entry, &zap_old_list);  in vm_map_enter()
3135 		if (entry->vme_start < start) {  in vm_map_enter()
3136 			tmp_start -= start - entry->vme_start;  in vm_map_enter()
3137 			tmp_offset -= start - entry->vme_start;  in vm_map_enter()
3139 		for (; entry->vme_start < end;  in vm_map_enter()
3140 		    entry = entry->vme_next) {  in vm_map_enter()
3145 			if (entry == vm_map_to_entry(map) ||  in vm_map_enter()
3146 			    entry->vme_start != tmp_start ||  in vm_map_enter()
3147 			    entry->is_sub_map != is_submap ||  in vm_map_enter()
3148 			    VME_OFFSET(entry) != tmp_offset ||  in vm_map_enter()
3149 			    entry->needs_copy != needs_copy ||  in vm_map_enter()
3150 			    entry->protection != cur_protection ||  in vm_map_enter()
3151 			    entry->max_protection != max_protection ||  in vm_map_enter()
3152 			    entry->inheritance != inheritance ||  in vm_map_enter()
3153 			    entry->iokit_acct != iokit_acct ||  in vm_map_enter()
3154 			    VME_ALIAS(entry) != alias) {  in vm_map_enter()
3162 				if (VME_SUBMAP(entry) !=  in vm_map_enter()
3168 				if (VME_OBJECT(entry) != object) {  in vm_map_enter()
3172 					obj2 = VME_OBJECT(entry);  in vm_map_enter()
3186 			tmp_offset += entry->vme_end - entry->vme_start;  in vm_map_enter()
3187 			tmp_start += entry->vme_end - entry->vme_start;  in vm_map_enter()
3188 			if (entry->vme_end >= end) {  in vm_map_enter()
3281 	    (entry != vm_map_to_entry(map)) &&  in vm_map_enter()
3282 	    (entry->vme_end == start) &&  in vm_map_enter()
3283 	    (!entry->is_shared) &&  in vm_map_enter()
3284 	    (!entry->is_sub_map) &&  in vm_map_enter()
3285 	    (!entry->in_transition) &&  in vm_map_enter()
3286 	    (!entry->needs_wakeup) &&  in vm_map_enter()
3287 	    (entry->behavior == VM_BEHAVIOR_DEFAULT) &&  in vm_map_enter()
3288 	    (entry->protection == cur_protection) &&  in vm_map_enter()
3289 	    (entry->max_protection == max_protection) &&  in vm_map_enter()
3290 	    (entry->inheritance == inheritance) &&  in vm_map_enter()
3292 	    (VME_ALIAS(entry) == alias)) &&  in vm_map_enter()
3293 	    (entry->no_cache == no_cache) &&  in vm_map_enter()
3294 	    (entry->vme_permanent == permanent) &&  in vm_map_enter()
3296 	    !((entry->protection & VM_PROT_EXECUTE) &&  in vm_map_enter()
3297 	    entry->vme_permanent) &&  in vm_map_enter()
3298 	    (!entry->superpage_size && !superpage_size) &&  in vm_map_enter()
3303 	    (!entry->map_aligned || !clear_map_aligned) &&  in vm_map_enter()
3304 	    (!entry->zero_wired_pages) &&  in vm_map_enter()
3305 	    (!entry->used_for_jit && !entry_for_jit) &&  in vm_map_enter()
3307 	    (!entry->used_for_tpro && !entry_for_tpro) &&  in vm_map_enter()
3309 	    (!entry->csm_associated) &&  in vm_map_enter()
3310 	    (entry->iokit_acct == iokit_acct) &&  in vm_map_enter()
3311 	    (!entry->vme_resilient_codesign) &&  in vm_map_enter()
3312 	    (!entry->vme_resilient_media) &&  in vm_map_enter()
3313 	    (!entry->vme_atomic) &&  in vm_map_enter()
3314 	    (entry->vme_no_copy_on_read == no_copy_on_read) &&  in vm_map_enter()
3316 	    ((entry->vme_end - entry->vme_start) + size <=  in vm_map_enter()
3321 	    (entry->wired_count == 0)) {        /* implies user_wired_count == 0 */  in vm_map_enter()
3322 		if (vm_object_coalesce(VME_OBJECT(entry),  in vm_map_enter()
3324 		    VME_OFFSET(entry),  in vm_map_enter()
3326 		    (vm_map_size_t)(entry->vme_end - entry->vme_start),  in vm_map_enter()
3327 		    (vm_map_size_t)(end - entry->vme_end))) {  in vm_map_enter()
3333 			map->size += (end - entry->vme_end);  in vm_map_enter()
3334 			assert(entry->vme_start < end);  in vm_map_enter()
3338 …p_entry_extend, vm_map_t, map, vm_map_entry_t, entry, vm_address_t, entry->vme_start, vm_address_t…  in vm_map_enter()
3340 			entry->vme_end = end;  in vm_map_enter()
3342 				vm_map_store_update_first_free(map, entry, TRUE);  in vm_map_enter()
3397 			    entry, tmp_start, tmp_end,  in vm_map_enter()
3509 			entry = new_entry;  in vm_map_enter()
3517 				VME_OFFSET_SET(entry, 0);  in vm_map_enter()
3530 				sp_object = vm_object_allocate((vm_map_size_t)(entry->vme_end - entry->vme_start));  in vm_map_enter()
3535 				VME_OBJECT_SET(entry, sp_object, false, 0);  in vm_map_enter()
3536 				assert(entry->use_pmap);  in vm_map_enter()
3692 		entry = VM_MAP_ENTRY_NULL;  in vm_map_enter()
5146 	vm_map_entry_t  entry,  in vm_map_clip_unnest()  argument
5153 	assert(entry->is_sub_map);  in vm_map_clip_unnest()
5154 	assert(VME_SUBMAP(entry) != NULL);  in vm_map_clip_unnest()
5155 	assert(entry->use_pmap);  in vm_map_clip_unnest()
5166 		assert(VME_SUBMAP(entry)->is_nested_map);  in vm_map_clip_unnest()
5167 		assert(!VME_SUBMAP(entry)->disable_vmentry_reuse);  in vm_map_clip_unnest()
5171 		    VME_SUBMAP(entry)->is_nested_map,  in vm_map_clip_unnest()
5172 		    (entry->vme_start +  in vm_map_clip_unnest()
5173 		    VME_SUBMAP(entry)->lowest_unnestable_start -  in vm_map_clip_unnest()
5174 		    VME_OFFSET(entry)));  in vm_map_clip_unnest()
5177 	if (entry->vme_start > start_unnest ||  in vm_map_clip_unnest()
5178 	    entry->vme_end < end_unnest) {  in vm_map_clip_unnest()
5182 		    (long long)entry->vme_start, (long long)entry->vme_end);  in vm_map_clip_unnest()
5185 	if (start_unnest > entry->vme_start) {  in vm_map_clip_unnest()
5187 		    entry,  in vm_map_clip_unnest()
5195 	if (entry->vme_end > end_unnest) {  in vm_map_clip_unnest()
5197 		    entry,  in vm_map_clip_unnest()
5207 	    entry->vme_start,  in vm_map_clip_unnest()
5208 	    entry->vme_end - entry->vme_start);  in vm_map_clip_unnest()
5212 			map, entry->vme_start,  in vm_map_clip_unnest()
5213 			entry->vme_end,  in vm_map_clip_unnest()
5214 			VME_SUBMAP(entry),  in vm_map_clip_unnest()
5215 			VME_OFFSET(entry));  in vm_map_clip_unnest()
5217 	entry->use_pmap = FALSE;  in vm_map_clip_unnest()
5219 	    (VME_ALIAS(entry) == VM_MEMORY_SHARED_PMAP)) {  in vm_map_clip_unnest()
5220 		VME_ALIAS_SET(entry, VM_MEMORY_UNSHARED_PMAP);  in vm_map_clip_unnest()
5229 	vm_map_entry_t  entry,  in __vm_map_clip_atomic_entry_panic()  argument
5233 	    "%p [0x%llx:0x%llx] at 0x%llx", map, entry,  in __vm_map_clip_atomic_entry_panic()
5234 	    (uint64_t)entry->vme_start,  in __vm_map_clip_atomic_entry_panic()
5235 	    (uint64_t)entry->vme_end,  in __vm_map_clip_atomic_entry_panic()
5249 	vm_map_entry_t  entry,  in vm_map_clip_start()  argument
5253 	if (entry->is_sub_map &&  in vm_map_clip_start()
5254 	    entry->use_pmap &&  in vm_map_clip_start()
5255 	    startaddr >= entry->vme_start) {  in vm_map_clip_start()
5267 		vm_map_clip_unnest(map, entry, start_unnest, end_unnest);  in vm_map_clip_start()
5270 	if (startaddr > entry->vme_start) {  in vm_map_clip_start()
5271 		if (!entry->is_sub_map &&  in vm_map_clip_start()
5272 		    VME_OBJECT(entry) &&  in vm_map_clip_start()
5273 		    VME_OBJECT(entry)->phys_contiguous) {  in vm_map_clip_start()
5275 			    (addr64_t)(entry->vme_start),  in vm_map_clip_start()
5276 			    (addr64_t)(entry->vme_end));  in vm_map_clip_start()
5278 		if (entry->vme_atomic) {  in vm_map_clip_start()
5279 			__vm_map_clip_atomic_entry_panic(map, entry, startaddr);  in vm_map_clip_start()
5285 			vm_map_offset_t, entry->vme_start,  in vm_map_clip_start()
5286 			vm_map_offset_t, entry->vme_end,  in vm_map_clip_start()
5288 			int, VME_ALIAS(entry));  in vm_map_clip_start()
5290 		_vm_map_clip_start(&map->hdr, entry, startaddr);  in vm_map_clip_start()
5300 #define vm_map_copy_clip_start(copy, entry, startaddr) \  argument
5302 	if ((startaddr) > (entry)->vme_start) \
5303 	        _vm_map_clip_start(&(copy)->cpy_hdr,(entry),(startaddr)); \
5313 	vm_map_entry_t          entry,  in _vm_map_clip_start()  argument
5326 	if (entry->map_aligned) {  in _vm_map_clip_start()
5332 	vm_map_entry_copy_full(new_entry, entry);  in _vm_map_clip_start()
5336 	VME_OFFSET_SET(entry, VME_OFFSET(entry) + (start - entry->vme_start));  in _vm_map_clip_start()
5337 	if (__improbable(start >= entry->vme_end)) {  in _vm_map_clip_start()
5338 …ry %p start 0x%llx end 0x%llx new start 0x%llx", map_header, entry, entry->vme_start, entry->vme_e…  in _vm_map_clip_start()
5340 	assert(start < entry->vme_end);  in _vm_map_clip_start()
5341 	entry->vme_start = start;  in _vm_map_clip_start()
5349 	_vm_map_store_entry_link(map_header, entry->vme_prev, new_entry);  in _vm_map_clip_start()
5351 	if (entry->is_sub_map) {  in _vm_map_clip_start()
5369 	vm_map_entry_t  entry,  in vm_map_clip_end()  argument
5372 	if (endaddr > entry->vme_end) {  in vm_map_clip_end()
5377 		endaddr = entry->vme_end;  in vm_map_clip_end()
5380 	if (entry->is_sub_map && entry->use_pmap) {  in vm_map_clip_end()
5390 		start_unnest = entry->vme_start;  in vm_map_clip_end()
5394 		vm_map_clip_unnest(map, entry, start_unnest, end_unnest);  in vm_map_clip_end()
5397 	if (endaddr < entry->vme_end) {  in vm_map_clip_end()
5398 		if (!entry->is_sub_map &&  in vm_map_clip_end()
5399 		    VME_OBJECT(entry) &&  in vm_map_clip_end()
5400 		    VME_OBJECT(entry)->phys_contiguous) {  in vm_map_clip_end()
5402 			    (addr64_t)(entry->vme_start),  in vm_map_clip_end()
5403 			    (addr64_t)(entry->vme_end));  in vm_map_clip_end()
5405 		if (entry->vme_atomic) {  in vm_map_clip_end()
5406 			__vm_map_clip_atomic_entry_panic(map, entry, endaddr);  in vm_map_clip_end()
5411 			vm_map_offset_t, entry->vme_start,  in vm_map_clip_end()
5412 			vm_map_offset_t, entry->vme_end,  in vm_map_clip_end()
5414 			int, VME_ALIAS(entry));  in vm_map_clip_end()
5416 		_vm_map_clip_end(&map->hdr, entry, endaddr);  in vm_map_clip_end()
5426 #define vm_map_copy_clip_end(copy, entry, endaddr) \  argument
5428 	if ((endaddr) < (entry)->vme_end) \
5429 	        _vm_map_clip_end(&(copy)->cpy_hdr,(entry),(endaddr)); \
5439 	vm_map_entry_t          entry,  in _vm_map_clip_end()  argument
5449 	if (entry->map_aligned) {  in _vm_map_clip_end()
5455 	vm_map_entry_copy_full(new_entry, entry);  in _vm_map_clip_end()
5457 	if (__improbable(end <= entry->vme_start)) {  in _vm_map_clip_end()
5458 …ntry %p start 0x%llx end 0x%llx new end 0x%llx", map_header, entry, entry->vme_start, entry->vme_e…  in _vm_map_clip_end()
5460 	assert(entry->vme_start < end);  in _vm_map_clip_end()
5461 	new_entry->vme_start = entry->vme_end = end;  in _vm_map_clip_end()
5463 	    VME_OFFSET(new_entry) + (end - entry->vme_start));  in _vm_map_clip_end()
5472 	_vm_map_store_entry_link(map_header, entry, new_entry);  in _vm_map_clip_end()
5474 	if (entry->is_sub_map) {  in _vm_map_clip_end()
5517 	vm_map_entry_t          *entry)  in vm_map_range_check()  argument
5541 	if (entry != (vm_map_entry_t *) NULL) {  in vm_map_range_check()
5542 		*entry = cur;  in vm_map_range_check()
5611 	vm_map_entry_t                  entry;  in vm_map_protect()  local
5721 		if (!vm_map_lookup_entry(map, start, &entry)) {  in vm_map_protect()
5726 		if (entry->superpage_size && (start & (SUPERPAGE_SIZE - 1))) { /* extend request to whole entry */  in vm_map_protect()
5732 	if (entry->superpage_size) {  in vm_map_protect()
5741 	current = entry;  in vm_map_protect()
5875 	current = entry;  in vm_map_protect()
5903 			entry = NULL;  in vm_map_protect()
6079 	if (entry == VM_MAP_ENTRY_NULL) {  in vm_map_protect()
6084 		vm_map_lookup_entry_or_next(map, original_start, &entry);  in vm_map_protect()
6086 	current = entry;  in vm_map_protect()
6141 	vm_map_entry_t  entry;  in vm_map_inherit()  local
6164 		entry = temp_entry;  in vm_map_inherit()
6167 		entry = temp_entry;  in vm_map_inherit()
6172 	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {  in vm_map_inherit()
6173 		if (entry->is_sub_map) {  in vm_map_inherit()
6180 		entry = entry->vme_next;  in vm_map_inherit()
6183 	entry = temp_entry;  in vm_map_inherit()
6184 	if (entry != vm_map_to_entry(map)) {  in vm_map_inherit()
6186 		vm_map_clip_start(map, entry, start);  in vm_map_inherit()
6189 	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {  in vm_map_inherit()
6190 		vm_map_clip_end(map, entry, end);  in vm_map_inherit()
6191 		if (entry->is_sub_map) {  in vm_map_inherit()
6193 			assert(!entry->use_pmap);  in vm_map_inherit()
6196 		entry->inheritance = new_inheritance;  in vm_map_inherit()
6198 		entry = entry->vme_next;  in vm_map_inherit()
6213 	vm_map_entry_t  entry,  in add_wire_counts()  argument
6218 	bool first_wire = entry->wired_count == 0 && entry->user_wired_count == 0;  in add_wire_counts()
6228 		if (entry->user_wired_count == 0) {  in add_wire_counts()
6229 			size = entry->vme_end - entry->vme_start;  in add_wire_counts()
6264 			if (entry->wired_count >= MAX_WIRE_COUNT) {  in add_wire_counts()
6268 			entry->wired_count++;  in add_wire_counts()
6272 		if (entry->user_wired_count >= MAX_WIRE_COUNT) {  in add_wire_counts()
6276 		entry->user_wired_count++;  in add_wire_counts()
6282 		if (entry->wired_count >= MAX_WIRE_COUNT) {  in add_wire_counts()
6286 		entry->wired_count++;  in add_wire_counts()
6290 		vme_btref_consider_and_set(entry, __builtin_frame_address(0));  in add_wire_counts()
6303 	vm_map_entry_t  entry,  in subtract_wire_counts()  argument
6311 		if (entry->user_wired_count == 1) {  in subtract_wire_counts()
6317 			assert(entry->wired_count >= 1);  in subtract_wire_counts()
6318 			entry->wired_count--;  in subtract_wire_counts()
6319 			map->user_wire_size -= entry->vme_end - entry->vme_start;  in subtract_wire_counts()
6322 		assert(entry->user_wired_count >= 1);  in subtract_wire_counts()
6323 		entry->user_wired_count--;  in subtract_wire_counts()
6329 		assert(entry->wired_count >= 1);  in subtract_wire_counts()
6330 		entry->wired_count--;  in subtract_wire_counts()
6333 	vme_btref_consider_and_put(entry);  in subtract_wire_counts()
6350 	vm_map_entry_t          entry;  in vm_map_wire_nested()  local
6417 		entry = first_entry;  in vm_map_wire_nested()
6428 	while ((entry != vm_map_to_entry(map)) && (s < end)) {  in vm_map_wire_nested()
6438 		e = entry->vme_end;  in vm_map_wire_nested()
6447 		if (entry->in_transition) {  in vm_map_wire_nested()
6459 			entry->needs_wakeup = TRUE;  in vm_map_wire_nested()
6504 			entry = first_entry;  in vm_map_wire_nested()
6508 		if (entry->is_sub_map) {  in vm_map_wire_nested()
6526 			vm_map_clip_start(map, entry, s);  in vm_map_wire_nested()
6527 			vm_map_clip_end(map, entry, end);  in vm_map_wire_nested()
6529 			sub_start = VME_OFFSET(entry);  in vm_map_wire_nested()
6530 			sub_end = entry->vme_end;  in vm_map_wire_nested()
6531 			sub_end += VME_OFFSET(entry) - entry->vme_start;  in vm_map_wire_nested()
6533 			local_end = entry->vme_end;  in vm_map_wire_nested()
6543 				if (entry->use_pmap) {  in vm_map_wire_nested()
6544 					pmap = VME_SUBMAP(entry)->pmap;  in vm_map_wire_nested()
6557 				if (entry->wired_count) {  in vm_map_wire_nested()
6558 					if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) {  in vm_map_wire_nested()
6567 					entry = entry->vme_next;  in vm_map_wire_nested()
6568 					s = entry->vme_start;  in vm_map_wire_nested()
6575 				local_start = entry->vme_start;  in vm_map_wire_nested()
6612 				entry = local_entry;  in vm_map_wire_nested()
6614 				vm_map_clip_start(map, entry, s);  in vm_map_wire_nested()
6615 				vm_map_clip_end(map, entry, end);  in vm_map_wire_nested()
6617 				e = entry->vme_end;  in vm_map_wire_nested()
6623 				if (!entry->is_sub_map) {  in vm_map_wire_nested()
6628 				local_start = entry->vme_start;  in vm_map_wire_nested()
6632 			if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) {  in vm_map_wire_nested()
6636 			entry->in_transition = TRUE;  in vm_map_wire_nested()
6639 			rc = vm_map_wire_nested(VME_SUBMAP(entry),  in vm_map_wire_nested()
6654 			entry = first_entry;  in vm_map_wire_nested()
6658 			e = entry->vme_end;  in vm_map_wire_nested()
6664 			while ((entry != vm_map_to_entry(map)) &&  in vm_map_wire_nested()
6665 			    (entry->vme_start < e)) {  in vm_map_wire_nested()
6666 				assert(entry->in_transition);  in vm_map_wire_nested()
6667 				entry->in_transition = FALSE;  in vm_map_wire_nested()
6668 				if (entry->needs_wakeup) {  in vm_map_wire_nested()
6669 					entry->needs_wakeup = FALSE;  in vm_map_wire_nested()
6673 					subtract_wire_counts(map, entry, user_wire);  in vm_map_wire_nested()
6675 				entry = entry->vme_next;  in vm_map_wire_nested()
6682 			s = entry->vme_start;  in vm_map_wire_nested()
6690 		if (entry->wired_count) {  in vm_map_wire_nested()
6691 			if ((entry->protection & access_type) != access_type) {  in vm_map_wire_nested()
6712 			vm_map_clip_start(map, entry, s);  in vm_map_wire_nested()
6713 			vm_map_clip_end(map, entry, end);  in vm_map_wire_nested()
6715 			if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) {  in vm_map_wire_nested()
6730 				assert((entry->vme_end - entry->vme_start)  in vm_map_wire_nested()
6732 				assert(!entry->needs_copy);  in vm_map_wire_nested()
6733 				assert(!entry->is_sub_map);  in vm_map_wire_nested()
6734 				assert(VME_OBJECT(entry));  in vm_map_wire_nested()
6735 				if (((entry->vme_end - entry->vme_start)  in vm_map_wire_nested()
6737 				    entry->needs_copy ||  in vm_map_wire_nested()
6738 				    entry->is_sub_map ||  in vm_map_wire_nested()
6739 				    VME_OBJECT(entry) == VM_OBJECT_NULL) {  in vm_map_wire_nested()
6744 				object = VME_OBJECT(entry);  in vm_map_wire_nested()
6745 				offset = VME_OFFSET(entry);  in vm_map_wire_nested()
6747 				if (entry->protection & VM_PROT_WRITE) {  in vm_map_wire_nested()
6757 					if (entry->protection & VM_PROT_WRITE) {  in vm_map_wire_nested()
6770 			entry = entry->vme_next;  in vm_map_wire_nested()
6771 			s = entry->vme_start;  in vm_map_wire_nested()
6786 		if ((entry->protection & VM_PROT_EXECUTE)  in vm_map_wire_nested()
6809 			    (uint64_t) entry->vme_start,  in vm_map_wire_nested()
6810 			    (uint64_t) entry->vme_end);  in vm_map_wire_nested()
6813 			    uint64_t, (uint64_t)entry->vme_start,  in vm_map_wire_nested()
6814 			    uint64_t, (uint64_t)entry->vme_end);  in vm_map_wire_nested()
6826 		size = entry->vme_end - entry->vme_start;  in vm_map_wire_nested()
6832 		if (entry->needs_copy) {  in vm_map_wire_nested()
6842 			VME_OBJECT_SHADOW(entry, size,  in vm_map_wire_nested()
6844 			entry->needs_copy = FALSE;  in vm_map_wire_nested()
6845 		} else if (VME_OBJECT(entry) == VM_OBJECT_NULL) {  in vm_map_wire_nested()
6854 			VME_OBJECT_SET(entry, vm_object_allocate(size), false, 0);  in vm_map_wire_nested()
6855 			VME_OFFSET_SET(entry, (vm_object_offset_t)0);  in vm_map_wire_nested()
6856 			assert(entry->use_pmap);  in vm_map_wire_nested()
6857 		} else if (VME_OBJECT(entry)->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {  in vm_map_wire_nested()
6871 			vm_map_clip_start(map, entry, s);  in vm_map_wire_nested()
6872 			vm_map_clip_end(map, entry, end);  in vm_map_wire_nested()
6874 			size = entry->vme_end - entry->vme_start;  in vm_map_wire_nested()
6878 			orig_object = VME_OBJECT(entry);  in vm_map_wire_nested()
6879 			orig_offset = VME_OFFSET(entry);  in vm_map_wire_nested()
6880 			VME_OBJECT_SHADOW(entry, size, vm_map_always_shadow(map));  in vm_map_wire_nested()
6881 			if (VME_OBJECT(entry) != orig_object) {  in vm_map_wire_nested()
6901 				    entry->vme_start,  in vm_map_wire_nested()
6902 				    entry->protection & ~VM_PROT_WRITE);  in vm_map_wire_nested()
6905 		if (VME_OBJECT(entry)->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {  in vm_map_wire_nested()
6917 			object = VME_OBJECT(entry);  in vm_map_wire_nested()
6920 …assertf(vm_object_round_page(VME_OFFSET(entry) + size) - vm_object_trunc_page(VME_OFFSET(entry)) =…  in vm_map_wire_nested()
6923 				    entry,  in vm_map_wire_nested()
6924 				    (uint64_t)entry->vme_start,  in vm_map_wire_nested()
6925 				    (uint64_t)entry->vme_end,  in vm_map_wire_nested()
6926 				    (uint64_t)VME_OFFSET(entry),  in vm_map_wire_nested()
6931 				assertf(!entry->needs_copy,  in vm_map_wire_nested()
6932 				    "entry %p\n", entry);  in vm_map_wire_nested()
6939 		vm_map_clip_start(map, entry, s);  in vm_map_wire_nested()
6940 		vm_map_clip_end(map, entry, end);  in vm_map_wire_nested()
6943 		e = entry->vme_end;  in vm_map_wire_nested()
6955 		if ((entry->vme_end < end) &&  in vm_map_wire_nested()
6956 		    ((entry->vme_next == vm_map_to_entry(map)) ||  in vm_map_wire_nested()
6957 		    (entry->vme_next->vme_start > entry->vme_end))) {  in vm_map_wire_nested()
6962 		if ((entry->protection & access_type) != access_type) {  in vm_map_wire_nested()
6968 		assert(entry->wired_count == 0 && entry->user_wired_count == 0);  in vm_map_wire_nested()
6970 		if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) {  in vm_map_wire_nested()
6974 		entry->in_transition = TRUE;  in vm_map_wire_nested()
6985 		tmp_entry = *entry;  in vm_map_wire_nested()
7027 			entry = first_entry;  in vm_map_wire_nested()
7032 		while ((entry != vm_map_to_entry(map)) &&  in vm_map_wire_nested()
7033 		    (entry->vme_start < tmp_entry.vme_end)) {  in vm_map_wire_nested()
7034 			assert(entry->in_transition);  in vm_map_wire_nested()
7035 			entry->in_transition = FALSE;  in vm_map_wire_nested()
7036 			if (entry->needs_wakeup) {  in vm_map_wire_nested()
7037 				entry->needs_wakeup = FALSE;  in vm_map_wire_nested()
7041 				subtract_wire_counts(map, entry, user_wire);  in vm_map_wire_nested()
7043 			entry = entry->vme_next;  in vm_map_wire_nested()
7050 		if ((entry != vm_map_to_entry(map)) && /* we still have entries in the map */  in vm_map_wire_nested()
7052 		    (entry->vme_start != tmp_entry.vme_end)) { /* AND, the next entry is not contiguous. */  in vm_map_wire_nested()
7059 		s = entry->vme_start;  in vm_map_wire_nested()
7216 	vm_map_entry_t          entry;  in vm_map_unwire_nested()  local
7238 		entry = first_entry;  in vm_map_unwire_nested()
7252 	if (entry->superpage_size) {  in vm_map_unwire_nested()
7259 	while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {  in vm_map_unwire_nested()
7260 		if (entry->in_transition) {  in vm_map_unwire_nested()
7299 			entry = entry->vme_next;  in vm_map_unwire_nested()
7303 		if (entry->is_sub_map) {  in vm_map_unwire_nested()
7309 			vm_map_clip_start(map, entry, start);  in vm_map_unwire_nested()
7310 			vm_map_clip_end(map, entry, end);  in vm_map_unwire_nested()
7312 			sub_start = VME_OFFSET(entry);  in vm_map_unwire_nested()
7313 			sub_end = entry->vme_end - entry->vme_start;  in vm_map_unwire_nested()
7314 			sub_end += VME_OFFSET(entry);  in vm_map_unwire_nested()
7315 			local_end = entry->vme_end;  in vm_map_unwire_nested()
7317 				if (entry->use_pmap) {  in vm_map_unwire_nested()
7318 					pmap = VME_SUBMAP(entry)->pmap;  in vm_map_unwire_nested()
7324 				if (entry->wired_count == 0 ||  in vm_map_unwire_nested()
7325 				    (user_wire && entry->user_wired_count == 0)) {  in vm_map_unwire_nested()
7329 					entry = entry->vme_next;  in vm_map_unwire_nested()
7338 				if (((entry->vme_end < end) &&  in vm_map_unwire_nested()
7339 				    ((entry->vme_next == vm_map_to_entry(map)) ||  in vm_map_unwire_nested()
7340 				    (entry->vme_next->vme_start  in vm_map_unwire_nested()
7341 				    > entry->vme_end)))) {  in vm_map_unwire_nested()
7351 				subtract_wire_counts(map, entry, user_wire);  in vm_map_unwire_nested()
7353 				if (entry->wired_count != 0) {  in vm_map_unwire_nested()
7354 					entry = entry->vme_next;  in vm_map_unwire_nested()
7358 				entry->in_transition = TRUE;  in vm_map_unwire_nested()
7359 				tmp_entry = *entry;/* see comment in vm_map_wire() */  in vm_map_unwire_nested()
7366 				vm_map_unwire_nested(VME_SUBMAP(entry),  in vm_map_unwire_nested()
7381 						entry = first_entry->vme_next;  in vm_map_unwire_nested()
7383 						entry = first_entry;  in vm_map_unwire_nested()
7393 				while ((entry != vm_map_to_entry(map)) &&  in vm_map_unwire_nested()
7394 				    (entry->vme_start < tmp_entry.vme_end)) {  in vm_map_unwire_nested()
7395 					assert(entry->in_transition);  in vm_map_unwire_nested()
7396 					entry->in_transition = FALSE;  in vm_map_unwire_nested()
7397 					if (entry->needs_wakeup) {  in vm_map_unwire_nested()
7398 						entry->needs_wakeup = FALSE;  in vm_map_unwire_nested()
7401 					entry = entry->vme_next;  in vm_map_unwire_nested()
7405 				tmp_entry = *entry;  in vm_map_unwire_nested()
7407 				vm_map_unwire_nested(VME_SUBMAP(entry),  in vm_map_unwire_nested()
7423 						entry = first_entry->vme_next;  in vm_map_unwire_nested()
7425 						entry = first_entry;  in vm_map_unwire_nested()
7433 		if ((entry->wired_count == 0) ||  in vm_map_unwire_nested()
7434 		    (user_wire && entry->user_wired_count == 0)) {  in vm_map_unwire_nested()
7439 			entry = entry->vme_next;  in vm_map_unwire_nested()
7443 		assert(entry->wired_count > 0 &&  in vm_map_unwire_nested()
7444 		    (!user_wire || entry->user_wired_count > 0));  in vm_map_unwire_nested()
7446 		vm_map_clip_start(map, entry, start);  in vm_map_unwire_nested()
7447 		vm_map_clip_end(map, entry, end);  in vm_map_unwire_nested()
7454 		if (((entry->vme_end < end) &&  in vm_map_unwire_nested()
7455 		    ((entry->vme_next == vm_map_to_entry(map)) ||  in vm_map_unwire_nested()
7456 		    (entry->vme_next->vme_start > entry->vme_end)))) {  in vm_map_unwire_nested()
7460 			entry = entry->vme_next;  in vm_map_unwire_nested()
7464 		subtract_wire_counts(map, entry, user_wire);  in vm_map_unwire_nested()
7466 		if (entry->wired_count != 0) {  in vm_map_unwire_nested()
7467 			entry = entry->vme_next;  in vm_map_unwire_nested()
7471 		if (entry->zero_wired_pages) {  in vm_map_unwire_nested()
7472 			entry->zero_wired_pages = FALSE;  in vm_map_unwire_nested()
7475 		entry->in_transition = TRUE;  in vm_map_unwire_nested()
7476 		tmp_entry = *entry;     /* see comment in vm_map_wire() */  in vm_map_unwire_nested()
7502 				entry = first_entry->vme_next;  in vm_map_unwire_nested()
7504 				entry = first_entry;  in vm_map_unwire_nested()
7514 		while ((entry != vm_map_to_entry(map)) &&  in vm_map_unwire_nested()
7515 		    (entry->vme_start < tmp_entry.vme_end)) {  in vm_map_unwire_nested()
7516 			assert(entry->in_transition);  in vm_map_unwire_nested()
7517 			entry->in_transition = FALSE;  in vm_map_unwire_nested()
7518 			if (entry->needs_wakeup) {  in vm_map_unwire_nested()
7519 				entry->needs_wakeup = FALSE;  in vm_map_unwire_nested()
7522 			entry = entry->vme_next;  in vm_map_unwire_nested()
7614 	vm_map_entry_t          entry,  in vm_map_entry_zap()  argument
7619 	s = entry->vme_start;  in vm_map_entry_zap()
7620 	e = entry->vme_end;  in vm_map_entry_zap()
7627 	if (entry->map_aligned == TRUE) {  in vm_map_entry_zap()
7631 	assert(entry->wired_count == 0);  in vm_map_entry_zap()
7632 	assert(entry->user_wired_count == 0);  in vm_map_entry_zap()
7633 	assert(!entry->vme_permanent);  in vm_map_entry_zap()
7635 	vm_map_store_entry_unlink(map, entry, false);  in vm_map_entry_zap()
7638 	vm_map_zap_append(zap, entry);  in vm_map_entry_zap()
7652 	vm_map_entry_t  entry;  in vm_map_submap_pmap_clean()  local
7658 	if (vm_map_lookup_entry(sub_map, offset, &entry)) {  in vm_map_submap_pmap_clean()
7659 		remove_size = (entry->vme_end - entry->vme_start);  in vm_map_submap_pmap_clean()
7660 		if (offset > entry->vme_start) {  in vm_map_submap_pmap_clean()
7661 			remove_size -= offset - entry->vme_start;  in vm_map_submap_pmap_clean()
7665 		if (submap_end < entry->vme_end) {  in vm_map_submap_pmap_clean()
7667 			    entry->vme_end - submap_end;  in vm_map_submap_pmap_clean()
7669 		if (entry->is_sub_map) {  in vm_map_submap_pmap_clean()
7674 				VME_SUBMAP(entry),  in vm_map_submap_pmap_clean()
7675 				VME_OFFSET(entry));  in vm_map_submap_pmap_clean()
7679 			    VME_OBJECT(entry) != NULL) {  in vm_map_submap_pmap_clean()
7681 					VME_OBJECT(entry),  in vm_map_submap_pmap_clean()
7682 					(VME_OFFSET(entry) +  in vm_map_submap_pmap_clean()
7684 					entry->vme_start),  in vm_map_submap_pmap_clean()
7688 					entry->vme_start,  in vm_map_submap_pmap_clean()
7699 	entry = entry->vme_next;  in vm_map_submap_pmap_clean()
7701 	while ((entry != vm_map_to_entry(sub_map))  in vm_map_submap_pmap_clean()
7702 	    && (entry->vme_start < submap_end)) {  in vm_map_submap_pmap_clean()
7703 		remove_size = (entry->vme_end - entry->vme_start);  in vm_map_submap_pmap_clean()
7704 		if (submap_end < entry->vme_end) {  in vm_map_submap_pmap_clean()
7705 			remove_size -= entry->vme_end - submap_end;  in vm_map_submap_pmap_clean()
7707 		if (entry->is_sub_map) {  in vm_map_submap_pmap_clean()
7710 				(start + entry->vme_start) - offset,  in vm_map_submap_pmap_clean()
7711 				((start + entry->vme_start) - offset) + remove_size,  in vm_map_submap_pmap_clean()
7712 				VME_SUBMAP(entry),  in vm_map_submap_pmap_clean()
7713 				VME_OFFSET(entry));  in vm_map_submap_pmap_clean()
7717 			    VME_OBJECT(entry) != NULL) {  in vm_map_submap_pmap_clean()
7719 					VME_OBJECT(entry),  in vm_map_submap_pmap_clean()
7720 					VME_OFFSET(entry),  in vm_map_submap_pmap_clean()
7724 					entry->vme_start,  in vm_map_submap_pmap_clean()
7729 				    (addr64_t)((start + entry->vme_start)  in vm_map_submap_pmap_clean()
7731 				    (addr64_t)(((start + entry->vme_start)  in vm_map_submap_pmap_clean()
7735 		entry = entry->vme_next;  in vm_map_submap_pmap_clean()
7936 	vm_map_entry_t          entry)  in __vm_map_delete_permanent_panic()  argument
7940 	    map, (uint64_t)start, (uint64_t)end, entry,  in __vm_map_delete_permanent_panic()
7941 	    (uint64_t)entry->vme_start,  in __vm_map_delete_permanent_panic()
7942 	    (uint64_t)entry->vme_end);  in __vm_map_delete_permanent_panic()
8023 	vm_map_entry_t          entry, next;  in vm_map_delete()  local
8136 	while (vm_map_lookup_entry_or_next(map, start, &entry)) {  in vm_map_delete()
8137 		if (entry->superpage_size && (start & ~SUPERPAGE_MASK)) {  in vm_map_delete()
8140 			SAVE_HINT_MAP_WRITE(map, entry->vme_prev);  in vm_map_delete()
8145 	if (entry->superpage_size) {  in vm_map_delete()
8182 			if (vm_map_lookup_entry_or_next(map, s, &entry)) {  in vm_map_delete()
8183 				SAVE_HINT_MAP_WRITE(map, entry->vme_prev);  in vm_map_delete()
8192 			for (vm_map_entry_t it = entry;  in vm_map_delete()
8213 		if (entry == vm_map_to_entry(map) || s < entry->vme_start) {  in vm_map_delete()
8227 			if (entry == vm_map_to_entry(map) ||  in vm_map_delete()
8228 			    end <= entry->vme_start) {  in vm_map_delete()
8232 			s = entry->vme_start;  in vm_map_delete()
8241 			if (entry->vme_permanent) {  in vm_map_delete()
8242 				__vm_map_delete_permanent_panic(map, start, end, entry);  in vm_map_delete()
8246 				end = entry->vme_end;  in vm_map_delete()
8260 			kmem_entry_validate_guard(map, entry,  in vm_map_delete()
8269 		if (entry->vme_permanent && entry->is_sub_map) {  in vm_map_delete()
8300 				if (entry->in_transition) {  in vm_map_delete()
8305 				submap_start = s - entry->vme_start;  in vm_map_delete()
8306 				submap_start += VME_OFFSET(entry);  in vm_map_delete()
8307 				submap_end = end - entry->vme_start;  in vm_map_delete()
8308 				submap_end += VME_OFFSET(entry);  in vm_map_delete()
8310 					VME_SUBMAP(entry),  in vm_map_delete()
8324 					    : "?"), entry,  in vm_map_delete()
8325 					    (uint64_t)entry->vme_start,  in vm_map_delete()
8326 					    (uint64_t)entry->vme_end,  in vm_map_delete()
8327 					    entry->protection,  in vm_map_delete()
8328 					    entry->max_protection);  in vm_map_delete()
8330 					    vm_map_entry_t, entry,  in vm_map_delete()
8331 					    vm_map_offset_t, entry->vme_start,  in vm_map_delete()
8332 					    vm_map_offset_t, entry->vme_end,  in vm_map_delete()
8333 					    vm_prot_t, entry->protection,  in vm_map_delete()
8334 					    vm_prot_t, entry->max_protection,  in vm_map_delete()
8335 					    int, VME_ALIAS(entry));  in vm_map_delete()
8349 		if (entry->vme_start < s) {  in vm_map_delete()
8351 			    entry->map_aligned &&  in vm_map_delete()
8357 				entry->map_aligned = FALSE;  in vm_map_delete()
8359 			vm_map_clip_start(map, entry, s);  in vm_map_delete()
8360 			SAVE_HINT_MAP_WRITE(map, entry->vme_prev);  in vm_map_delete()
8363 		if (end < entry->vme_end) {  in vm_map_delete()
8365 			    entry->map_aligned &&  in vm_map_delete()
8371 				entry->map_aligned = FALSE;  in vm_map_delete()
8373 			vm_map_clip_end(map, entry, end);  in vm_map_delete()
8376 		if (entry->vme_permanent && entry->is_sub_map) {  in vm_map_delete()
8386 			    vm_map_entry_t, entry,  in vm_map_delete()
8387 			    vm_map_offset_t, entry->vme_start,  in vm_map_delete()
8388 			    vm_map_offset_t, entry->vme_end,  in vm_map_delete()
8389 			    vm_prot_t, entry->protection,  in vm_map_delete()
8390 			    vm_prot_t, entry->max_protection,  in vm_map_delete()
8391 			    int, VME_ALIAS(entry));  in vm_map_delete()
8392 			entry->vme_permanent = false;  in vm_map_delete()
8395 		assert(s == entry->vme_start);  in vm_map_delete()
8396 		assert(entry->vme_end <= end);  in vm_map_delete()
8403 		if (entry->in_transition) {  in vm_map_delete()
8412 			entry->needs_wakeup = TRUE;  in vm_map_delete()
8448 		if (entry->wired_count) {  in vm_map_delete()
8453 			user_wire = entry->user_wired_count > 0;  in vm_map_delete()
8459 				entry->wired_count--;  in vm_map_delete()
8460 				vme_btref_consider_and_put(entry);  in vm_map_delete()
8466 			while (entry->user_wired_count) {  in vm_map_delete()
8467 				subtract_wire_counts(map, entry, user_wire);  in vm_map_delete()
8483 			if (entry->wired_count != 0) {  in vm_map_delete()
8494 				entry->needs_wakeup = TRUE;  in vm_map_delete()
8532 			entry->in_transition = TRUE;  in vm_map_delete()
8533 			tmp_entry = *entry;  in vm_map_delete()
8596 		assert(entry->wired_count == 0);  in vm_map_delete()
8597 		assert(entry->user_wired_count == 0);  in vm_map_delete()
8604 		if (!entry->vme_permanent) {  in vm_map_delete()
8609 		    (entry->protection & VM_PROT_EXECUTE) &&  in vm_map_delete()
8621 			    : "?"), entry,  in vm_map_delete()
8622 			    (uint64_t)entry->vme_start,  in vm_map_delete()
8623 			    (uint64_t)entry->vme_end,  in vm_map_delete()
8624 			    entry->protection,  in vm_map_delete()
8625 			    entry->max_protection);  in vm_map_delete()
8627 			entry->vme_permanent = FALSE;  in vm_map_delete()
8635 			    : "?"), entry,  in vm_map_delete()
8636 			    (uint64_t)entry->vme_start,  in vm_map_delete()
8637 			    (uint64_t)entry->vme_end,  in vm_map_delete()
8638 			    entry->protection,  in vm_map_delete()
8639 			    entry->max_protection);  in vm_map_delete()
8641 			entry->vme_permanent = FALSE;  in vm_map_delete()
8643 		} else if ((entry->protection & VM_PROT_EXECUTE) && !csm_enabled()) {  in vm_map_delete()
8644 			entry->vme_permanent = FALSE;  in vm_map_delete()
8656 			    (uint64_t)entry->vme_start,  in vm_map_delete()
8657 			    (uint64_t)entry->vme_end,  in vm_map_delete()
8658 			    entry->protection,  in vm_map_delete()
8659 			    entry->max_protection);  in vm_map_delete()
8663 			    vm_map_entry_t, entry,  in vm_map_delete()
8664 			    vm_map_offset_t, entry->vme_start,  in vm_map_delete()
8665 			    vm_map_offset_t, entry->vme_end,  in vm_map_delete()
8666 			    vm_prot_t, entry->protection,  in vm_map_delete()
8667 			    vm_prot_t, entry->max_protection,  in vm_map_delete()
8668 			    int, VME_ALIAS(entry));  in vm_map_delete()
8671 		if (entry->is_sub_map) {  in vm_map_delete()
8672 			assertf(VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)) >= VM_MAP_PAGE_SHIFT(map),  in vm_map_delete()
8674 			    map, VM_MAP_PAGE_SHIFT(map), entry,  in vm_map_delete()
8675 			    VME_SUBMAP(entry),  in vm_map_delete()
8676 			    VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)));  in vm_map_delete()
8677 			if (entry->use_pmap) {  in vm_map_delete()
8704 					(addr64_t)entry->vme_start,  in vm_map_delete()
8705 					entry->vme_end - entry->vme_start,  in vm_map_delete()
8712 						map, entry->vme_start,  in vm_map_delete()
8713 						entry->vme_end,  in vm_map_delete()
8714 						VME_SUBMAP(entry),  in vm_map_delete()
8715 						VME_OFFSET(entry));  in vm_map_delete()
8719 					map, entry->vme_start, entry->vme_end,  in vm_map_delete()
8720 					VME_SUBMAP(entry),  in vm_map_delete()
8721 					VME_OFFSET(entry));  in vm_map_delete()
8723 		} else if (entry->vme_kernel_object ||  in vm_map_delete()
8724 		    VME_OBJECT(entry) == compressor_object) {  in vm_map_delete()
8731 				VME_OBJECT(entry), VME_OFFSET(entry),  in vm_map_delete()
8732 				entry->vme_end - entry->vme_start,  in vm_map_delete()
8735 				entry->vme_start,  in vm_map_delete()
8738 		} else if ((VME_OBJECT(entry) != VM_OBJECT_NULL) ||  in vm_map_delete()
8751 			vm_map_address_t remove_start = entry->vme_start;  in vm_map_delete()
8752 			vm_map_address_t remove_end = entry->vme_end;  in vm_map_delete()
8771 		    entry->vme_start,  in vm_map_delete()
8772 		    entry->vme_end));  in vm_map_delete()
8775 		if (entry->iokit_acct) {  in vm_map_delete()
8779 			    vm_map_offset_t, entry->vme_start,  in vm_map_delete()
8780 			    vm_map_offset_t, entry->vme_end,  in vm_map_delete()
8781 			    int, VME_ALIAS(entry));  in vm_map_delete()
8783 			    (entry->vme_end -  in vm_map_delete()
8784 			    entry->vme_start));  in vm_map_delete()
8785 			entry->iokit_acct = FALSE;  in vm_map_delete()
8786 			entry->use_pmap = FALSE;  in vm_map_delete()
8790 		s    = entry->vme_end;  in vm_map_delete()
8791 		next = entry->vme_next;  in vm_map_delete()
8792 		if (!entry->map_aligned) {  in vm_map_delete()
8807 		ret.kmr_size += s - entry->vme_start;  in vm_map_delete()
8809 		if (entry->vme_permanent) {  in vm_map_delete()
8822 				    entry,  in vm_map_delete()
8823 				    (uint64_t)entry->vme_start,  in vm_map_delete()
8824 				    (uint64_t)entry->vme_end,  in vm_map_delete()
8825 				    entry->is_sub_map,  in vm_map_delete()
8826 				    entry->protection,  in vm_map_delete()
8827 				    entry->max_protection);  in vm_map_delete()
8830 			    vm_map_entry_t, entry,  in vm_map_delete()
8831 			    vm_map_offset_t, entry->vme_start,  in vm_map_delete()
8832 			    vm_map_offset_t, entry->vme_end,  in vm_map_delete()
8833 			    vm_prot_t, entry->protection,  in vm_map_delete()
8834 			    vm_prot_t, entry->max_protection,  in vm_map_delete()
8835 			    int, VME_ALIAS(entry));  in vm_map_delete()
8836 			entry->protection = VM_PROT_NONE;  in vm_map_delete()
8837 			entry->max_protection = VM_PROT_NONE;  in vm_map_delete()
8839 			entry->used_for_tpro = FALSE;  in vm_map_delete()
8842 			vm_map_entry_zap(map, entry, zap_list);  in vm_map_delete()
8845 		entry = next;  in vm_map_delete()
8994 			vm_map_entry_t  entry = vm_map_copy_first_entry(copy);  in vm_map_copy_discard()  local
8996 			vm_map_copy_entry_unlink(copy, entry);  in vm_map_copy_discard()
8997 			if (entry->is_sub_map) {  in vm_map_copy_discard()
8998 				vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_copy_discard()
9000 				vm_object_deallocate(VME_OBJECT(entry));  in vm_map_copy_discard()
9002 			vm_map_copy_entry_dispose(entry);  in vm_map_copy_discard()
9101 	vm_map_entry_t  entry)  in vm_map_entry_is_overwritable()  argument
9103 	if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_entry_is_overwritable()
9108 	if (entry->used_for_jit &&  in vm_map_entry_is_overwritable()
9120 	if (entry->used_for_tpro) {  in vm_map_entry_is_overwritable()
9125 	if (entry->vme_permanent) {  in vm_map_entry_is_overwritable()
9126 		if (entry->is_sub_map) {  in vm_map_entry_is_overwritable()
9139 			    vm_map_entry_t, entry,  in vm_map_entry_is_overwritable()
9140 			    vm_map_offset_t, entry->vme_start,  in vm_map_entry_is_overwritable()
9141 			    vm_map_offset_t, entry->vme_end,  in vm_map_entry_is_overwritable()
9142 			    vm_prot_t, entry->protection,  in vm_map_entry_is_overwritable()
9143 			    vm_prot_t, entry->max_protection,  in vm_map_entry_is_overwritable()
9144 			    int, VME_ALIAS(entry));  in vm_map_entry_is_overwritable()
9150 	if (entry->is_sub_map) {  in vm_map_entry_is_overwritable()
9165 	vm_map_entry_t  entry;  in vm_map_overwrite_submap_recurse()  local
9197 	for (entry = tmp_entry;;) {  in vm_map_overwrite_submap_recurse()
9200 		next = entry->vme_next;  in vm_map_overwrite_submap_recurse()
9201 		while (entry->is_sub_map) {  in vm_map_overwrite_submap_recurse()
9206 			if (entry->in_transition) {  in vm_map_overwrite_submap_recurse()
9210 				entry->needs_wakeup = TRUE;  in vm_map_overwrite_submap_recurse()
9217 			sub_start = VME_OFFSET(entry);  in vm_map_overwrite_submap_recurse()
9219 			if (entry->vme_end < dst_end) {  in vm_map_overwrite_submap_recurse()
9220 				sub_end = entry->vme_end;  in vm_map_overwrite_submap_recurse()
9224 			sub_end -= entry->vme_start;  in vm_map_overwrite_submap_recurse()
9225 			sub_end += VME_OFFSET(entry);  in vm_map_overwrite_submap_recurse()
9226 			local_end = entry->vme_end;  in vm_map_overwrite_submap_recurse()
9230 				VME_SUBMAP(entry),  in vm_map_overwrite_submap_recurse()
9237 			if (dst_end <= entry->vme_end) {  in vm_map_overwrite_submap_recurse()
9246 			entry = tmp_entry;  in vm_map_overwrite_submap_recurse()
9247 			next = entry->vme_next;  in vm_map_overwrite_submap_recurse()
9249 		assert(!entry->is_sub_map);  in vm_map_overwrite_submap_recurse()
9251 		if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_overwrite_submap_recurse()
9256 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_overwrite_submap_recurse()
9266 		if (entry->in_transition) {  in vm_map_overwrite_submap_recurse()
9270 			entry->needs_wakeup = TRUE;  in vm_map_overwrite_submap_recurse()
9279 		if (dst_end <= entry->vme_end) {  in vm_map_overwrite_submap_recurse()
9287 		    (next->vme_start != entry->vme_end)) {  in vm_map_overwrite_submap_recurse()
9295 		assert(!entry->is_sub_map);  in vm_map_overwrite_submap_recurse()
9296 		if ((VME_OBJECT(entry) != VM_OBJECT_NULL) &&  in vm_map_overwrite_submap_recurse()
9297 		    ((!VME_OBJECT(entry)->internal) ||  in vm_map_overwrite_submap_recurse()
9298 		    (VME_OBJECT(entry)->true_share))) {  in vm_map_overwrite_submap_recurse()
9306 		entry = next;  in vm_map_overwrite_submap_recurse()
9377 	vm_map_entry_t          entry;  in vm_map_copy_overwrite_nested()  local
9456 	for (entry = tmp_entry;;) {  in vm_map_copy_overwrite_nested()
9457 		vm_map_entry_t  next = entry->vme_next;  in vm_map_copy_overwrite_nested()
9459 		while (entry->is_sub_map) {  in vm_map_copy_overwrite_nested()
9464 			if (entry->in_transition) {  in vm_map_copy_overwrite_nested()
9468 				entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_nested()
9474 			local_end = entry->vme_end;  in vm_map_copy_overwrite_nested()
9475 			if (!(entry->needs_copy)) {  in vm_map_copy_overwrite_nested()
9481 				sub_start = VME_OFFSET(entry);  in vm_map_copy_overwrite_nested()
9483 				if (entry->vme_end < dst_end) {  in vm_map_copy_overwrite_nested()
9484 					sub_end = entry->vme_end;  in vm_map_copy_overwrite_nested()
9488 				sub_end -= entry->vme_start;  in vm_map_copy_overwrite_nested()
9489 				sub_end += VME_OFFSET(entry);  in vm_map_copy_overwrite_nested()
9493 					VME_SUBMAP(entry),  in vm_map_copy_overwrite_nested()
9502 			if (dst_end <= entry->vme_end) {  in vm_map_copy_overwrite_nested()
9506 			    &entry)) {  in vm_map_copy_overwrite_nested()
9510 			next = entry->vme_next;  in vm_map_copy_overwrite_nested()
9512 		assert(!entry->is_sub_map);  in vm_map_copy_overwrite_nested()
9514 		if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_copy_overwrite_nested()
9519 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_nested()
9529 		if (entry->in_transition) {  in vm_map_copy_overwrite_nested()
9533 			entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_nested()
9542 		if (dst_end <= entry->vme_end) {  in vm_map_copy_overwrite_nested()
9549 		    (next->vme_start != entry->vme_end)) {  in vm_map_copy_overwrite_nested()
9558 		assert(!entry->is_sub_map);  in vm_map_copy_overwrite_nested()
9559 		if ((VME_OBJECT(entry) != VM_OBJECT_NULL) &&  in vm_map_copy_overwrite_nested()
9560 		    ((!VME_OBJECT(entry)->internal) ||  in vm_map_copy_overwrite_nested()
9561 		    (VME_OBJECT(entry)->true_share))) {  in vm_map_copy_overwrite_nested()
9565 		entry = next;  in vm_map_copy_overwrite_nested()
9613 		for (entry = tmp_entry; copy_size == 0;) {  in vm_map_copy_overwrite_nested()
9616 			next = entry->vme_next;  in vm_map_copy_overwrite_nested()
9626 			if (entry->in_transition) {  in vm_map_copy_overwrite_nested()
9630 				entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_nested()
9639 				entry = tmp_entry;  in vm_map_copy_overwrite_nested()
9642 			if (entry->is_sub_map) {  in vm_map_copy_overwrite_nested()
9647 				if (entry->needs_copy) {  in vm_map_copy_overwrite_nested()
9651 					assert(!entry->vme_permanent);  in vm_map_copy_overwrite_nested()
9652 					if (entry->vme_end < dst_end) {  in vm_map_copy_overwrite_nested()
9653 						sub_end = entry->vme_end;  in vm_map_copy_overwrite_nested()
9657 					if (entry->vme_start < base_addr) {  in vm_map_copy_overwrite_nested()
9660 						sub_start = entry->vme_start;  in vm_map_copy_overwrite_nested()
9663 						dst_map, entry, sub_end);  in vm_map_copy_overwrite_nested()
9665 						dst_map, entry, sub_start);  in vm_map_copy_overwrite_nested()
9666 					assert(!entry->use_pmap);  in vm_map_copy_overwrite_nested()
9667 					assert(!entry->iokit_acct);  in vm_map_copy_overwrite_nested()
9668 					entry->use_pmap = TRUE;  in vm_map_copy_overwrite_nested()
9669 					vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_copy_overwrite_nested()
9670 					assert(!entry->vme_permanent);  in vm_map_copy_overwrite_nested()
9671 					VME_OBJECT_SET(entry, VM_OBJECT_NULL, false, 0);  in vm_map_copy_overwrite_nested()
9672 					VME_OFFSET_SET(entry, 0);  in vm_map_copy_overwrite_nested()
9673 					entry->is_shared = FALSE;  in vm_map_copy_overwrite_nested()
9674 					entry->needs_copy = FALSE;  in vm_map_copy_overwrite_nested()
9675 					entry->protection = VM_PROT_DEFAULT;  in vm_map_copy_overwrite_nested()
9676 					entry->max_protection = VM_PROT_ALL;  in vm_map_copy_overwrite_nested()
9677 					entry->wired_count = 0;  in vm_map_copy_overwrite_nested()
9678 					entry->user_wired_count = 0;  in vm_map_copy_overwrite_nested()
9679 					if (entry->inheritance  in vm_map_copy_overwrite_nested()
9681 						entry->inheritance = VM_INHERIT_COPY;  in vm_map_copy_overwrite_nested()
9687 				if (base_addr < entry->vme_start) {  in vm_map_copy_overwrite_nested()
9690 					    entry->vme_start - base_addr;  in vm_map_copy_overwrite_nested()
9693 				sub_start = VME_OFFSET(entry);  in vm_map_copy_overwrite_nested()
9695 				if (entry->vme_end < dst_end) {  in vm_map_copy_overwrite_nested()
9696 					sub_end = entry->vme_end;  in vm_map_copy_overwrite_nested()
9700 				sub_end -= entry->vme_start;  in vm_map_copy_overwrite_nested()
9701 				sub_end += VME_OFFSET(entry);  in vm_map_copy_overwrite_nested()
9702 				local_end = entry->vme_end;  in vm_map_copy_overwrite_nested()
9752 				if ((entry->use_pmap) && (pmap == NULL)) {  in vm_map_copy_overwrite_nested()
9754 						VME_SUBMAP(entry),  in vm_map_copy_overwrite_nested()
9758 						VME_SUBMAP(entry)->pmap,  in vm_map_copy_overwrite_nested()
9762 						VME_SUBMAP(entry),  in vm_map_copy_overwrite_nested()
9769 						VME_SUBMAP(entry),  in vm_map_copy_overwrite_nested()
9816 				entry = tmp_entry;  in vm_map_copy_overwrite_nested()
9819 			assert(!entry->is_sub_map);  in vm_map_copy_overwrite_nested()
9821 			if (dst_end <= entry->vme_end) {  in vm_map_copy_overwrite_nested()
9827 			    (next->vme_start != entry->vme_end)) {  in vm_map_copy_overwrite_nested()
9832 			entry = next;  in vm_map_copy_overwrite_nested()
9958 				entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_nested()
9969 		entry = tmp_entry;  in vm_map_copy_overwrite_nested()
10016 	vm_map_entry_t  entry;  in vm_map_copy_overwrite()  local
10145 	if (!vm_map_lookup_entry(dst_map, dst_addr, &entry)) {  in vm_map_copy_overwrite()
10150 	    (entry != vm_map_to_entry(dst_map) &&  in vm_map_copy_overwrite()
10151 	    entry->vme_start < dst_addr + copy_size);  in vm_map_copy_overwrite()
10152 	    entry = entry->vme_next) {  in vm_map_copy_overwrite()
10153 		if (entry->is_sub_map) {  in vm_map_copy_overwrite()
10174 		entry = vm_map_copy_first_entry(copy);  in vm_map_copy_overwrite()
10175 		if (entry->vme_end < copy->offset + head_size) {  in vm_map_copy_overwrite()
10176 			head_size = entry->vme_end - copy->offset;  in vm_map_copy_overwrite()
10186 		vm_map_copy_clip_end(copy, entry, copy->offset);  in vm_map_copy_overwrite()
10187 		vm_map_copy_entry_unlink(copy, entry);  in vm_map_copy_overwrite()
10190 		    entry);  in vm_map_copy_overwrite()
10223 		entry = vm_map_copy_last_entry(copy);  in vm_map_copy_overwrite()
10224 		vm_map_copy_clip_start(copy, entry, tail_copy->offset);  in vm_map_copy_overwrite()
10225 		entry = vm_map_copy_last_entry(copy);  in vm_map_copy_overwrite()
10226 		vm_map_copy_entry_unlink(copy, entry);  in vm_map_copy_overwrite()
10229 		    entry);  in vm_map_copy_overwrite()
10296 			entry = vm_map_copy_first_entry(head_copy);  in vm_map_copy_overwrite()
10297 			vm_map_copy_entry_unlink(head_copy, entry);  in vm_map_copy_overwrite()
10300 			    entry);  in vm_map_copy_overwrite()
10307 			entry = vm_map_copy_last_entry(tail_copy);  in vm_map_copy_overwrite()
10308 			vm_map_copy_entry_unlink(tail_copy, entry);  in vm_map_copy_overwrite()
10311 			    entry);  in vm_map_copy_overwrite()
10351 	vm_map_entry_t  entry,  in vm_map_copy_overwrite_unaligned()  argument
10382 		if (entry == vm_map_to_entry(dst_map)) {  in vm_map_copy_overwrite_unaligned()
10388 		assert((start >= entry->vme_start) && (start < entry->vme_end));  in vm_map_copy_overwrite_unaligned()
10393 		if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_copy_overwrite_unaligned()
10397 		if (entry->is_sub_map) {  in vm_map_copy_overwrite_unaligned()
10402 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_unaligned()
10411 		if (entry->in_transition) {  in vm_map_copy_overwrite_unaligned()
10415 			entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_unaligned()
10421 		dst_offset = start - entry->vme_start;  in vm_map_copy_overwrite_unaligned()
10423 		dst_size = entry->vme_end - start;  in vm_map_copy_overwrite_unaligned()
10449 		assert(!entry->is_sub_map);  in vm_map_copy_overwrite_unaligned()
10450 		if (entry->needs_copy) {  in vm_map_copy_overwrite_unaligned()
10455 			VME_OBJECT_SHADOW(entry,  in vm_map_copy_overwrite_unaligned()
10456 			    (vm_map_size_t)(entry->vme_end  in vm_map_copy_overwrite_unaligned()
10457 			    - entry->vme_start),  in vm_map_copy_overwrite_unaligned()
10459 			entry->needs_copy = FALSE;  in vm_map_copy_overwrite_unaligned()
10462 		dst_object = VME_OBJECT(entry);  in vm_map_copy_overwrite_unaligned()
10473 			    entry->vme_end - entry->vme_start);  in vm_map_copy_overwrite_unaligned()
10474 			VME_OBJECT_SET(entry, dst_object, false, 0);  in vm_map_copy_overwrite_unaligned()
10475 			VME_OFFSET_SET(entry, 0);  in vm_map_copy_overwrite_unaligned()
10476 			assert(entry->use_pmap);  in vm_map_copy_overwrite_unaligned()
10485 		entry_offset = VME_OFFSET(entry);  in vm_map_copy_overwrite_unaligned()
10486 		entry_end = entry->vme_end;  in vm_map_copy_overwrite_unaligned()
10554 				entry = entry->vme_next;  in vm_map_copy_overwrite_unaligned()
10559 				if (start != entry->vme_start) {  in vm_map_copy_overwrite_unaligned()
10571 			if (!vm_map_lookup_entry(dst_map, start, &entry)) {  in vm_map_copy_overwrite_unaligned()
10620 	vm_map_entry_t  entry;  in vm_map_copy_overwrite_aligned()  local
10626 		entry = tmp_entry;  in vm_map_copy_overwrite_aligned()
10628 		if (entry->is_sub_map) {  in vm_map_copy_overwrite_aligned()
10630 			assert(!entry->use_pmap);  in vm_map_copy_overwrite_aligned()
10632 		if (entry == vm_map_to_entry(dst_map)) {  in vm_map_copy_overwrite_aligned()
10636 		size = (entry->vme_end - entry->vme_start);  in vm_map_copy_overwrite_aligned()
10644 		if ((entry->vme_start != start) || ((entry->is_sub_map)  in vm_map_copy_overwrite_aligned()
10645 		    && !entry->needs_copy)) {  in vm_map_copy_overwrite_aligned()
10649 		assert(entry != vm_map_to_entry(dst_map));  in vm_map_copy_overwrite_aligned()
10655 		if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_copy_overwrite_aligned()
10660 		if (entry->is_sub_map) {  in vm_map_copy_overwrite_aligned()
10666 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_aligned()
10676 		if (entry->in_transition) {  in vm_map_copy_overwrite_aligned()
10680 			entry->needs_wakeup = TRUE;  in vm_map_copy_overwrite_aligned()
10691 			if (entry->map_aligned &&  in vm_map_copy_overwrite_aligned()
10692 			    !VM_MAP_PAGE_ALIGNED(entry->vme_start + copy_size,  in vm_map_copy_overwrite_aligned()
10695 				entry->map_aligned = FALSE;  in vm_map_copy_overwrite_aligned()
10697 			vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);  in vm_map_copy_overwrite_aligned()
10711 		assert((entry->vme_end - entry->vme_start) == size);  in vm_map_copy_overwrite_aligned()
10728 		object = VME_OBJECT(entry);  in vm_map_copy_overwrite_aligned()
10729 		if ((!entry->is_shared &&  in vm_map_copy_overwrite_aligned()
10730 		    !entry->vme_permanent &&  in vm_map_copy_overwrite_aligned()
10731 		    !entry->used_for_jit &&  in vm_map_copy_overwrite_aligned()
10733 		    !entry->used_for_tpro &&  in vm_map_copy_overwrite_aligned()
10735 		    !(entry->protection & VM_PROT_EXECUTE) &&  in vm_map_copy_overwrite_aligned()
10736 		    !pmap_has_prot_policy(dst_map->pmap, entry->translated_allow_execute, entry->protection) &&  in vm_map_copy_overwrite_aligned()
10741 		    entry->needs_copy) {  in vm_map_copy_overwrite_aligned()
10742 			vm_object_t     old_object = VME_OBJECT(entry);  in vm_map_copy_overwrite_aligned()
10743 			vm_object_offset_t      old_offset = VME_OFFSET(entry);  in vm_map_copy_overwrite_aligned()
10746 			assert(!entry->is_sub_map);  in vm_map_copy_overwrite_aligned()
10786 			    (VME_ALIAS(entry) >= VM_MEMORY_MALLOC) &&  in vm_map_copy_overwrite_aligned()
10787 			    (VME_ALIAS(entry) <= VM_MEMORY_MALLOC_MEDIUM)) {  in vm_map_copy_overwrite_aligned()
10852 				assert(!entry->vme_permanent);  in vm_map_copy_overwrite_aligned()
10853 				if (entry->is_sub_map) {  in vm_map_copy_overwrite_aligned()
10854 					if (entry->use_pmap) {  in vm_map_copy_overwrite_aligned()
10857 						    (addr64_t)entry->vme_start,  in vm_map_copy_overwrite_aligned()
10858 						    entry->vme_end - entry->vme_start);  in vm_map_copy_overwrite_aligned()
10864 								dst_map, entry->vme_start,  in vm_map_copy_overwrite_aligned()
10865 								entry->vme_end,  in vm_map_copy_overwrite_aligned()
10866 								VME_SUBMAP(entry),  in vm_map_copy_overwrite_aligned()
10867 								VME_OFFSET(entry));  in vm_map_copy_overwrite_aligned()
10871 							dst_map, entry->vme_start,  in vm_map_copy_overwrite_aligned()
10872 							entry->vme_end,  in vm_map_copy_overwrite_aligned()
10873 							VME_SUBMAP(entry),  in vm_map_copy_overwrite_aligned()
10874 							VME_OFFSET(entry));  in vm_map_copy_overwrite_aligned()
10876 					vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_copy_overwrite_aligned()
10880 							VME_OBJECT(entry),  in vm_map_copy_overwrite_aligned()
10881 							VME_OFFSET(entry),  in vm_map_copy_overwrite_aligned()
10882 							entry->vme_end  in vm_map_copy_overwrite_aligned()
10883 							- entry->vme_start,  in vm_map_copy_overwrite_aligned()
10886 							entry->vme_start,  in vm_map_copy_overwrite_aligned()
10892 							(addr64_t)(entry->vme_start),  in vm_map_copy_overwrite_aligned()
10893 							(addr64_t)(entry->vme_end),  in vm_map_copy_overwrite_aligned()
10900 			if (entry->iokit_acct) {  in vm_map_copy_overwrite_aligned()
10902 				entry->use_pmap = FALSE;  in vm_map_copy_overwrite_aligned()
10905 				entry->use_pmap = TRUE;  in vm_map_copy_overwrite_aligned()
10907 			assert(!entry->vme_permanent);  in vm_map_copy_overwrite_aligned()
10908 			VME_OBJECT_SET(entry, VME_OBJECT(copy_entry), false, 0);  in vm_map_copy_overwrite_aligned()
10909 			object = VME_OBJECT(entry);  in vm_map_copy_overwrite_aligned()
10910 			entry->needs_copy = copy_entry->needs_copy;  in vm_map_copy_overwrite_aligned()
10911 			entry->wired_count = 0;  in vm_map_copy_overwrite_aligned()
10912 			entry->user_wired_count = 0;  in vm_map_copy_overwrite_aligned()
10914 			VME_OFFSET_SET(entry, offset);  in vm_map_copy_overwrite_aligned()
10945 			if (entry->needs_copy) {  in vm_map_copy_overwrite_aligned()
10946 				VME_OBJECT_SHADOW(entry,  in vm_map_copy_overwrite_aligned()
10947 				    (entry->vme_end -  in vm_map_copy_overwrite_aligned()
10948 				    entry->vme_start),  in vm_map_copy_overwrite_aligned()
10950 				entry->needs_copy = FALSE;  in vm_map_copy_overwrite_aligned()
10953 			dst_object = VME_OBJECT(entry);  in vm_map_copy_overwrite_aligned()
10954 			dst_offset = VME_OFFSET(entry);  in vm_map_copy_overwrite_aligned()
10976 					entry->vme_end - entry->vme_start);  in vm_map_copy_overwrite_aligned()
10978 				VME_OBJECT_SET(entry, dst_object, false, 0);  in vm_map_copy_overwrite_aligned()
10979 				VME_OFFSET_SET(entry, dst_offset);  in vm_map_copy_overwrite_aligned()
10980 				assert(entry->use_pmap);  in vm_map_copy_overwrite_aligned()
11283 	vm_map_entry_t  entry;  in vm_map_copy_insert()  local
11286 		entry = vm_map_copy_first_entry(copy);  in vm_map_copy_insert()
11287 		vm_map_copy_entry_unlink(copy, entry);  in vm_map_copy_insert()
11288 		vm_map_store_entry_link(map, after_where, entry,  in vm_map_copy_insert()
11290 		after_where = entry;  in vm_map_copy_insert()
11406 	vm_map_entry_t          entry;  in vm_map_copyout_internal()  local
11537 		entry = vm_map_copy_first_entry(copy);  in vm_map_copyout_internal()
11543 		vm_map_store_copy_reset(copy, entry);  in vm_map_copyout_internal()
11549 		while (entry != vm_map_copy_to_entry(copy)) {  in vm_map_copyout_internal()
11551 			vm_map_entry_copy_full(new, entry);  in vm_map_copyout_internal()
11561 			next = entry->vme_next;  in vm_map_copyout_internal()
11562 			vm_map_entry_dispose(entry);  in vm_map_copyout_internal()
11563 			entry = next;  in vm_map_copyout_internal()
11572 	for (entry = vm_map_copy_first_entry(copy);  in vm_map_copyout_internal()
11573 	    entry != vm_map_copy_to_entry(copy);  in vm_map_copyout_internal()
11574 	    entry = entry->vme_next) {  in vm_map_copyout_internal()
11582 			entry->map_aligned = FALSE;  in vm_map_copyout_internal()
11585 		entry->vme_start += adjustment;  in vm_map_copyout_internal()
11586 		entry->vme_end += adjustment;  in vm_map_copyout_internal()
11588 		if (entry->map_aligned) {  in vm_map_copyout_internal()
11589 			assert(VM_MAP_PAGE_ALIGNED(entry->vme_start,  in vm_map_copyout_internal()
11591 			assert(VM_MAP_PAGE_ALIGNED(entry->vme_end,  in vm_map_copyout_internal()
11595 		entry->inheritance = VM_INHERIT_DEFAULT;  in vm_map_copyout_internal()
11596 		entry->protection = VM_PROT_DEFAULT;  in vm_map_copyout_internal()
11597 		entry->max_protection = VM_PROT_ALL;  in vm_map_copyout_internal()
11598 		entry->behavior = VM_BEHAVIOR_DEFAULT;  in vm_map_copyout_internal()
11604 		if (entry->wired_count != 0) {  in vm_map_copyout_internal()
11615 			object = VME_OBJECT(entry);  in vm_map_copyout_internal()
11616 			offset = VME_OFFSET(entry);  in vm_map_copyout_internal()
11617 			va = entry->vme_start;  in vm_map_copyout_internal()
11620 			    entry->vme_start,  in vm_map_copyout_internal()
11621 			    entry->vme_end,  in vm_map_copyout_internal()
11624 			while (va < entry->vme_end) {  in vm_map_copyout_internal()
11653 				prot = entry->protection;  in vm_map_copyout_internal()
11655 				if (override_nx(dst_map, VME_ALIAS(entry)) &&  in vm_map_copyout_internal()
11662 				fault_info.user_tag = VME_ALIAS(entry);  in vm_map_copyout_internal()
11664 				if (entry->iokit_acct ||  in vm_map_copyout_internal()
11665 				    (!entry->is_sub_map && !entry->use_pmap)) {  in vm_map_copyout_internal()
11668 				if (entry->vme_xnu_user_debug &&  in vm_map_copyout_internal()
13883 	vm_map_entry_t                  entry;  in vm_map_lookup_and_lock_object()  local
13924 	entry = map->hint;  in vm_map_lookup_and_lock_object()
13926 	if ((entry == vm_map_to_entry(map)) ||  in vm_map_lookup_and_lock_object()
13927 	    (vaddr < entry->vme_start) || (vaddr >= entry->vme_end)) {  in vm_map_lookup_and_lock_object()
13945 		entry = tmp_entry;  in vm_map_lookup_and_lock_object()
13948 		old_start = entry->vme_start;  in vm_map_lookup_and_lock_object()
13949 		old_end = entry->vme_end;  in vm_map_lookup_and_lock_object()
13959 	if (entry->is_sub_map) {  in vm_map_lookup_and_lock_object()
13979 		assertf(VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)) >= VM_MAP_PAGE_SHIFT(map),  in vm_map_lookup_and_lock_object()
13981 		    map, VM_MAP_PAGE_SHIFT(map), entry,  in vm_map_lookup_and_lock_object()
13982 		    VME_SUBMAP(entry), VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)));  in vm_map_lookup_and_lock_object()
13985 		top_entry_saved_start = entry->vme_start;  in vm_map_lookup_and_lock_object()
13986 		top_entry_saved_offset = VME_OFFSET(entry);  in vm_map_lookup_and_lock_object()
13988 		if ((entry->use_pmap &&  in vm_map_lookup_and_lock_object()
13996 			*real_map = VME_SUBMAP(entry);  in vm_map_lookup_and_lock_object()
13999 		if (entry->needs_copy &&  in vm_map_lookup_and_lock_object()
14008 				vm_map_lock_read(VME_SUBMAP(entry));  in vm_map_lookup_and_lock_object()
14009 				*var_map = VME_SUBMAP(entry);  in vm_map_lookup_and_lock_object()
14014 				old_start = entry->vme_start;  in vm_map_lookup_and_lock_object()
14015 				old_end = entry->vme_end;  in vm_map_lookup_and_lock_object()
14019 				vm_map_lock_read(VME_SUBMAP(entry));  in vm_map_lookup_and_lock_object()
14020 				*var_map = VME_SUBMAP(entry);  in vm_map_lookup_and_lock_object()
14027 			if (entry->needs_copy) {  in vm_map_lookup_and_lock_object()
14030 			vm_map_lock_read(VME_SUBMAP(entry));  in vm_map_lookup_and_lock_object()
14031 			*var_map = VME_SUBMAP(entry);  in vm_map_lookup_and_lock_object()
14042 		entry = NULL;  in vm_map_lookup_and_lock_object()
14088 			entry = submap_entry;  in vm_map_lookup_and_lock_object()
14389 			    vaddr, &entry)) {  in vm_map_lookup_and_lock_object()
14403 				    uint64_t, (uint64_t)entry->vme_start,  in vm_map_lookup_and_lock_object()
14404 				    uint64_t, (uint64_t)entry->vme_end,  in vm_map_lookup_and_lock_object()
14437 			vm_map_clip_start(map, entry, local_start);  in vm_map_lookup_and_lock_object()
14438 			vm_map_clip_end(map, entry, local_end);  in vm_map_lookup_and_lock_object()
14439 			if (entry->is_sub_map) {  in vm_map_lookup_and_lock_object()
14441 				assert(!entry->use_pmap);  in vm_map_lookup_and_lock_object()
14446 			vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_lookup_and_lock_object()
14447 			assert(!entry->iokit_acct);  in vm_map_lookup_and_lock_object()
14448 			entry->use_pmap = TRUE;  in vm_map_lookup_and_lock_object()
14449 			VME_OBJECT_SET(entry, copy_object, false, 0);  in vm_map_lookup_and_lock_object()
14452 			if (entry->protection != VM_PROT_READ) {  in vm_map_lookup_and_lock_object()
14464 				entry->protection |= subentry_protection;  in vm_map_lookup_and_lock_object()
14466 			entry->max_protection |= subentry_max_protection;  in vm_map_lookup_and_lock_object()
14468 			entry->vme_no_copy_on_read = subentry_no_copy_on_read;  in vm_map_lookup_and_lock_object()
14469 			entry->vme_permanent = subentry_permanent;  in vm_map_lookup_and_lock_object()
14470 			entry->csm_associated = subentry_csm_associated;  in vm_map_lookup_and_lock_object()
14475 					entry->used_for_tpro = subentry_used_for_tpro;  in vm_map_lookup_and_lock_object()
14478 					entry->vme_permanent = FALSE;  in vm_map_lookup_and_lock_object()
14482 			if ((entry->protection & VM_PROT_WRITE) &&  in vm_map_lookup_and_lock_object()
14483 			    (entry->protection & VM_PROT_EXECUTE) &&  in vm_map_lookup_and_lock_object()
14495 			    !(entry->used_for_jit) &&  in vm_map_lookup_and_lock_object()
14498 				    uint64_t, (uint64_t)entry->vme_start,  in vm_map_lookup_and_lock_object()
14499 				    uint64_t, (uint64_t)entry->vme_end,  in vm_map_lookup_and_lock_object()
14500 				    vm_prot_t, entry->protection);  in vm_map_lookup_and_lock_object()
14508 				    (uint64_t)entry->vme_start,  in vm_map_lookup_and_lock_object()
14509 				    (uint64_t)entry->vme_end,  in vm_map_lookup_and_lock_object()
14514 				    entry->protection);  in vm_map_lookup_and_lock_object()
14515 				entry->protection &= ~VM_PROT_EXECUTE;  in vm_map_lookup_and_lock_object()
14519 				VME_OFFSET_SET(entry, local_start - old_start + object_copied_offset);  in vm_map_lookup_and_lock_object()
14520 				entry->needs_copy = object_copied_needs_copy;  in vm_map_lookup_and_lock_object()
14521 				entry->is_shared = FALSE;  in vm_map_lookup_and_lock_object()
14523 				assert(VME_OBJECT(entry) != VM_OBJECT_NULL);  in vm_map_lookup_and_lock_object()
14524 				assert(VME_OBJECT(entry)->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC);  in vm_map_lookup_and_lock_object()
14525 				assert(entry->wired_count == 0);  in vm_map_lookup_and_lock_object()
14526 				VME_OFFSET_SET(entry, copy_offset);  in vm_map_lookup_and_lock_object()
14527 				entry->needs_copy = TRUE;  in vm_map_lookup_and_lock_object()
14529 					entry->is_shared = TRUE;  in vm_map_lookup_and_lock_object()
14532 			if (entry->inheritance == VM_INHERIT_SHARE) {  in vm_map_lookup_and_lock_object()
14533 				entry->inheritance = VM_INHERIT_COPY;  in vm_map_lookup_and_lock_object()
14543 			entry = submap_entry;  in vm_map_lookup_and_lock_object()
14553 	prot = entry->protection;  in vm_map_lookup_and_lock_object()
14555 	if (override_nx(old_map, VME_ALIAS(entry)) && prot) {  in vm_map_lookup_and_lock_object()
14568 	if (entry->used_for_tpro && (fault_type & VM_PROT_WRITE)) {  in vm_map_lookup_and_lock_object()
14612 	*wired = (entry->wired_count != 0);  in vm_map_lookup_and_lock_object()
14621 	if (entry->needs_copy) {  in vm_map_lookup_and_lock_object()
14643 			if (VME_OBJECT(entry)->shadowed == FALSE) {  in vm_map_lookup_and_lock_object()
14644 				vm_object_lock(VME_OBJECT(entry));  in vm_map_lookup_and_lock_object()
14645 				VM_OBJECT_SET_SHADOWED(VME_OBJECT(entry), TRUE);  in vm_map_lookup_and_lock_object()
14646 				vm_object_unlock(VME_OBJECT(entry));  in vm_map_lookup_and_lock_object()
14648 			VME_OBJECT_SHADOW(entry,  in vm_map_lookup_and_lock_object()
14649 			    (vm_map_size_t) (entry->vme_end -  in vm_map_lookup_and_lock_object()
14650 			    entry->vme_start),  in vm_map_lookup_and_lock_object()
14652 			entry->needs_copy = FALSE;  in vm_map_lookup_and_lock_object()
14682 	if (VME_OBJECT(entry) == VM_OBJECT_NULL) {  in vm_map_lookup_and_lock_object()
14688 		VME_OBJECT_SET(entry,  in vm_map_lookup_and_lock_object()
14690 			    (vm_map_size_t)(entry->vme_end -  in vm_map_lookup_and_lock_object()
14691 			    entry->vme_start)), false, 0);  in vm_map_lookup_and_lock_object()
14692 		VME_OFFSET_SET(entry, 0);  in vm_map_lookup_and_lock_object()
14693 		assert(entry->use_pmap);  in vm_map_lookup_and_lock_object()
14703 	*offset = (vaddr - entry->vme_start) + VME_OFFSET(entry);  in vm_map_lookup_and_lock_object()
14704 	*object = VME_OBJECT(entry);  in vm_map_lookup_and_lock_object()
14706 …VM_MAP_LOOKUP_OBJECT), VM_KERNEL_UNSLIDE_OR_PERM(*object), (unsigned long) VME_ALIAS(entry), 0, 0);  in vm_map_lookup_and_lock_object()
14710 		fault_info->user_tag = VME_ALIAS(entry);  in vm_map_lookup_and_lock_object()
14712 		if (entry->iokit_acct ||  in vm_map_lookup_and_lock_object()
14713 		    (!entry->is_sub_map && !entry->use_pmap)) {  in vm_map_lookup_and_lock_object()
14717 			fault_info->behavior = entry->behavior;  in vm_map_lookup_and_lock_object()
14719 		fault_info->lo_offset = VME_OFFSET(entry);  in vm_map_lookup_and_lock_object()
14721 		    (entry->vme_end - entry->vme_start) + VME_OFFSET(entry);  in vm_map_lookup_and_lock_object()
14722 		fault_info->no_cache  = entry->no_cache;  in vm_map_lookup_and_lock_object()
14725 		if (entry->used_for_jit ||  in vm_map_lookup_and_lock_object()
14729 		    entry->vme_resilient_codesign) {  in vm_map_lookup_and_lock_object()
14736 		if (entry->csm_associated) {  in vm_map_lookup_and_lock_object()
14746 		fault_info->resilient_media = entry->vme_resilient_media;  in vm_map_lookup_and_lock_object()
14747 		fault_info->fi_xnu_user_debug = entry->vme_xnu_user_debug;  in vm_map_lookup_and_lock_object()
14748 		fault_info->no_copy_on_read = entry->vme_no_copy_on_read;  in vm_map_lookup_and_lock_object()
14750 		fault_info->fi_used_for_tpro = entry->used_for_tpro;  in vm_map_lookup_and_lock_object()
14754 		if (entry->translated_allow_execute) {  in vm_map_lookup_and_lock_object()
15339 	vm_map_entry_t          entry;  in vm_map_region()  local
15364 			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {  in vm_map_region()
15369 			entry = tmp_entry;  in vm_map_region()
15372 		start = entry->vme_start;  in vm_map_region()
15374 		basic->offset = (uint32_t)VME_OFFSET(entry);  in vm_map_region()
15375 		basic->protection = entry->protection;  in vm_map_region()
15376 		basic->inheritance = entry->inheritance;  in vm_map_region()
15377 		basic->max_protection = entry->max_protection;  in vm_map_region()
15378 		basic->behavior = entry->behavior;  in vm_map_region()
15379 		basic->user_wired_count = entry->user_wired_count;  in vm_map_region()
15380 		basic->reserved = entry->is_sub_map;  in vm_map_region()
15383 		*size_u    = vm_sanitize_wrap_size(entry->vme_end - start);  in vm_map_region()
15388 		if (entry->is_sub_map) {  in vm_map_region()
15391 			basic->shared = entry->is_shared;  in vm_map_region()
15412 			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {  in vm_map_region()
15417 			entry = tmp_entry;  in vm_map_region()
15420 		start = entry->vme_start;  in vm_map_region()
15422 		basic->offset = VME_OFFSET(entry);  in vm_map_region()
15423 		basic->protection = entry->protection;  in vm_map_region()
15424 		basic->inheritance = entry->inheritance;  in vm_map_region()
15425 		basic->max_protection = entry->max_protection;  in vm_map_region()
15426 		basic->behavior = entry->behavior;  in vm_map_region()
15427 		basic->user_wired_count = entry->user_wired_count;  in vm_map_region()
15428 		basic->reserved = entry->is_sub_map;  in vm_map_region()
15431 		*size_u    = vm_sanitize_wrap_size(entry->vme_end - start);  in vm_map_region()
15436 		if (entry->is_sub_map) {  in vm_map_region()
15439 			basic->shared = entry->is_shared;  in vm_map_region()
15468 			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {  in vm_map_region()
15473 			entry = tmp_entry;  in vm_map_region()
15475 		start = entry->vme_start;  in vm_map_region()
15477 		extended->protection = entry->protection;  in vm_map_region()
15478 		extended->user_tag = VME_ALIAS(entry);  in vm_map_region()
15494 …vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, extended, TRUE, *…  in vm_map_region()
15501 		*size_u    = vm_sanitize_wrap_size(entry->vme_end - start);  in vm_map_region()
15520 			if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) {  in vm_map_region()
15525 			entry = tmp_entry;  in vm_map_region()
15527 		start = entry->vme_start;  in vm_map_region()
15532 		vm_map_region_top_walk(entry, top);  in vm_map_region()
15539 		*size_u    = vm_sanitize_wrap_size(entry->vme_end - start);  in vm_map_region()
15557 	vm_map_entry_t             entry,  in vm_map_region_top_walk()  argument
15560 	if (entry->is_sub_map || VME_OBJECT(entry) == 0) {  in vm_map_region_top_walk()
15572 		entry_size = (uint32_t) ((entry->vme_end - entry->vme_start) / PAGE_SIZE_64);  in vm_map_region_top_walk()
15574 		obj = VME_OBJECT(entry);  in vm_map_region_top_walk()
15611 			if (entry->superpage_size) {  in vm_map_region_top_walk()
15615 			} else if (entry->needs_copy) {  in vm_map_region_top_walk()
15647 	vm_map_entry_t                  entry,  in vm_map_region_walk()  argument
15666 	if ((entry->is_sub_map) ||  in vm_map_region_walk()
15667 	    (VME_OBJECT(entry) == 0) ||  in vm_map_region_walk()
15668 	    (VME_OBJECT(entry)->phys_contiguous &&  in vm_map_region_walk()
15669 	    !entry->superpage_size)) {  in vm_map_region_walk()
15675 	if (entry->superpage_size) {  in vm_map_region_walk()
15693 	obj = VME_OBJECT(entry);  in vm_map_region_walk()
15725 						entry,  in vm_map_region_walk()
15783 	if (extended->shadow_depth || entry->needs_copy) {  in vm_map_region_walk()
15822 		obj = VME_OBJECT(entry);  in vm_map_region_walk()
15943 	vm_map_entry_t    entry,  in vm_map_region_has_obj_ref()  argument
15949 	if (entry->is_sub_map) {  in vm_map_region_has_obj_ref()
15953 	cur_obj = VME_OBJECT(entry);  in vm_map_region_has_obj_ref()
16121 	vm_map_entry_t  entry;  in vm_map_simplify_range()  local
16137 	if (!vm_map_lookup_entry(map, start, &entry)) {  in vm_map_simplify_range()
16139 		if (entry == vm_map_to_entry(map)) {  in vm_map_simplify_range()
16141 			entry = vm_map_first_entry(map);  in vm_map_simplify_range()
16144 			entry = entry->vme_next;  in vm_map_simplify_range()
16148 	while (entry != vm_map_to_entry(map) &&  in vm_map_simplify_range()
16149 	    entry->vme_start <= end) {  in vm_map_simplify_range()
16151 		vm_map_simplify_entry(map, entry);  in vm_map_simplify_range()
16152 		entry = entry->vme_next;  in vm_map_simplify_range()
16198 	vm_map_entry_t   entry;  in vm_map_machine_attribute()  local
16228 		if (vm_map_lookup_entry(map, start, &entry)) {  in vm_map_machine_attribute()
16230 			if ((entry->vme_end - start) > sync_size) {  in vm_map_machine_attribute()
16234 				sub_size = entry->vme_end - start;  in vm_map_machine_attribute()
16237 			if (entry->is_sub_map) {  in vm_map_machine_attribute()
16241 				sub_start = (start - entry->vme_start)  in vm_map_machine_attribute()
16242 				    + VME_OFFSET(entry);  in vm_map_machine_attribute()
16245 					VME_SUBMAP(entry),  in vm_map_machine_attribute()
16249 			} else if (VME_OBJECT(entry)) {  in vm_map_machine_attribute()
16258 				offset = (start - entry->vme_start)  in vm_map_machine_attribute()
16259 				    + VME_OFFSET(entry);  in vm_map_machine_attribute()
16262 				object = VME_OBJECT(entry);  in vm_map_machine_attribute()
16330 	vm_map_entry_t  entry;  in vm_map_behavior_set()  local
16364 			entry = temp_entry;  in vm_map_behavior_set()
16365 			vm_map_clip_start(map, entry, start);  in vm_map_behavior_set()
16371 		while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) {  in vm_map_behavior_set()
16372 			vm_map_clip_end(map, entry, end);  in vm_map_behavior_set()
16373 			if (entry->is_sub_map) {  in vm_map_behavior_set()
16374 				assert(!entry->use_pmap);  in vm_map_behavior_set()
16378 				entry->zero_wired_pages = TRUE;  in vm_map_behavior_set()
16380 				entry->behavior = new_behavior;  in vm_map_behavior_set()
16382 			entry = entry->vme_next;  in vm_map_behavior_set()
16444 	vm_map_entry_t                  entry;  in vm_map_willneed()  local
16471 	if (!vm_map_range_check(map, start, end, &entry)) {  in vm_map_willneed()
16481 	for (; entry != vm_map_to_entry(map) && start < end;) {  in vm_map_willneed()
16488 		offset = (start - entry->vme_start) + VME_OFFSET(entry);  in vm_map_willneed()
16500 		len = MIN(entry->vme_end - start, end - start);  in vm_map_willneed()
16509 		fault_info.user_tag     = VME_ALIAS(entry);  in vm_map_willneed()
16511 		if (entry->iokit_acct ||  in vm_map_willneed()
16512 		    (!entry->is_sub_map && !entry->use_pmap)) {  in vm_map_willneed()
16515 		fault_info.fi_xnu_user_debug = entry->vme_xnu_user_debug;  in vm_map_willneed()
16521 		if ((entry->is_sub_map) || (entry->protection & VM_PROT_READ) == 0) {  in vm_map_willneed()
16522 			entry = entry->vme_next;  in vm_map_willneed()
16523 			start = entry->vme_start;  in vm_map_willneed()
16527 		object = VME_OBJECT(entry);  in vm_map_willneed()
16559 			if ((object = find_vnode_object(entry)) == VM_OBJECT_NULL) {  in vm_map_willneed()
16560 				entry = entry->vme_next;  in vm_map_willneed()
16561 				start = entry->vme_start;  in vm_map_willneed()
16624 		if (!vm_map_lookup_entry(map, start, &entry)) {  in vm_map_willneed()
16643 	vm_map_entry_t entry)  in vm_map_entry_is_reusable()  argument
16649 	if (entry->is_sub_map) {  in vm_map_entry_is_reusable()
16653 	switch (VME_ALIAS(entry)) {  in vm_map_entry_is_reusable()
16676 		entry->is_sub_map ||  in vm_map_entry_is_reusable()
16677 		entry->in_transition ||  in vm_map_entry_is_reusable()
16678 		entry->protection != VM_PROT_DEFAULT ||  in vm_map_entry_is_reusable()
16679 		entry->max_protection != VM_PROT_ALL ||  in vm_map_entry_is_reusable()
16680 		entry->inheritance != VM_INHERIT_DEFAULT ||  in vm_map_entry_is_reusable()
16681 		entry->no_cache ||  in vm_map_entry_is_reusable()
16682 		entry->vme_permanent ||  in vm_map_entry_is_reusable()
16683 		entry->superpage_size != FALSE ||  in vm_map_entry_is_reusable()
16684 		entry->zero_wired_pages ||  in vm_map_entry_is_reusable()
16685 		entry->wired_count != 0 ||  in vm_map_entry_is_reusable()
16686 		entry->user_wired_count != 0) {  in vm_map_entry_is_reusable()
16690 	object = VME_OBJECT(entry);  in vm_map_entry_is_reusable()
16727 	vm_map_entry_t                  entry;  in vm_map_reuse_pages()  local
16754 	if (!vm_map_range_check(map, start, end, &entry)) {  in vm_map_reuse_pages()
16763 	for (; entry != vm_map_to_entry(map) && entry->vme_start < end;  in vm_map_reuse_pages()
16764 	    entry = entry->vme_next) {  in vm_map_reuse_pages()
16768 		if (!vm_map_entry_is_reusable(entry)) {  in vm_map_reuse_pages()
16779 		if (entry->vme_start < start) {  in vm_map_reuse_pages()
16780 			start_offset = start - entry->vme_start;  in vm_map_reuse_pages()
16784 		end_offset = MIN(end, entry->vme_end) - entry->vme_start;  in vm_map_reuse_pages()
16785 		start_offset += VME_OFFSET(entry);  in vm_map_reuse_pages()
16786 		end_offset += VME_OFFSET(entry);  in vm_map_reuse_pages()
16788 		object = VME_OBJECT(entry);  in vm_map_reuse_pages()
16796 		if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSABLE) {  in vm_map_reuse_pages()
16805 			VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSED);  in vm_map_reuse_pages()
16821 	vm_map_entry_t                  entry;  in vm_map_reusable_pages()  local
16849 	if (!vm_map_range_check(map, start, end, &entry)) {  in vm_map_reusable_pages()
16858 	for (; entry != vm_map_to_entry(map) && entry->vme_start < end;  in vm_map_reusable_pages()
16859 	    entry = entry->vme_next) {  in vm_map_reusable_pages()
16866 		if (!vm_map_entry_is_reusable(entry)) {  in vm_map_reusable_pages()
16872 		if (!(entry->protection & VM_PROT_WRITE) && !entry->used_for_jit  in vm_map_reusable_pages()
16874 		    && !entry->used_for_tpro  in vm_map_reusable_pages()
16889 		if (entry->vme_start < start) {  in vm_map_reusable_pages()
16890 			start_offset = start - entry->vme_start;  in vm_map_reusable_pages()
16894 			pmap_offset = entry->vme_start;  in vm_map_reusable_pages()
16896 		end_offset = MIN(end, entry->vme_end) - entry->vme_start;  in vm_map_reusable_pages()
16897 		start_offset += VME_OFFSET(entry);  in vm_map_reusable_pages()
16898 		end_offset += VME_OFFSET(entry);  in vm_map_reusable_pages()
16900 		object = VME_OBJECT(entry);  in vm_map_reusable_pages()
16905 		if (entry->protection & VM_PROT_EXECUTE) {  in vm_map_reusable_pages()
16913 		if (entry->vme_xnu_user_debug) {  in vm_map_reusable_pages()
16934 		    !(entry->iokit_acct ||  in vm_map_reusable_pages()
16935 		    (!entry->is_sub_map && !entry->use_pmap))) {  in vm_map_reusable_pages()
16955 			    unsigned int, VME_ALIAS(entry),  in vm_map_reusable_pages()
16957 			    vm_map_entry_t, entry,  in vm_map_reusable_pages()
16962 		if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE ||  in vm_map_reusable_pages()
16963 		    VME_ALIAS(entry) == VM_MEMORY_MALLOC_LARGE_REUSED) {  in vm_map_reusable_pages()
16972 			VME_ALIAS_SET(entry, VM_MEMORY_MALLOC_LARGE_REUSABLE);  in vm_map_reusable_pages()
16988 	vm_map_entry_t                  entry;  in vm_map_can_reuse()  local
17004 	if (!vm_map_range_check(map, start, end, &entry)) {  in vm_map_can_reuse()
17013 	for (; entry != vm_map_to_entry(map) && entry->vme_start < end;  in vm_map_can_reuse()
17014 	    entry = entry->vme_next) {  in vm_map_can_reuse()
17018 		if (!vm_map_entry_is_reusable(entry)) {  in vm_map_can_reuse()
17038 	vm_map_entry_t                  entry;  in vm_map_pageout()  local
17053 	if (!vm_map_range_check(map, start, end, &entry)) {  in vm_map_pageout()
17061 	for (; entry != vm_map_to_entry(map) && entry->vme_start < end;  in vm_map_pageout()
17062 	    entry = entry->vme_next) {  in vm_map_pageout()
17068 		if (entry->is_sub_map) {  in vm_map_pageout()
17074 			submap = VME_SUBMAP(entry);  in vm_map_pageout()
17075 			submap_start = VME_OFFSET(entry);  in vm_map_pageout()
17076 			submap_end = submap_start + (entry->vme_end -  in vm_map_pageout()
17077 			    entry->vme_start);  in vm_map_pageout()
17109 		object = VME_OBJECT(entry);  in vm_map_pageout()
17128 vm_map_zero_entry_preflight(vm_map_entry_t entry)  in vm_map_zero_entry_preflight()  argument
17134 	if (!(entry->protection & VM_PROT_WRITE) ||  in vm_map_zero_entry_preflight()
17135 	    (entry->protection & VM_PROT_EXECUTE) ||  in vm_map_zero_entry_preflight()
17136 	    entry->used_for_jit ||  in vm_map_zero_entry_preflight()
17137 	    entry->vme_xnu_user_debug) {  in vm_map_zero_entry_preflight()
17145 	if (entry->needs_copy || entry->is_sub_map) {  in vm_map_zero_entry_preflight()
17157 	vm_map_entry_t      entry,  in vm_map_get_bounds_in_object()  argument
17163 	if (entry->vme_start < start) {  in vm_map_get_bounds_in_object()
17164 		*start_offset = start - entry->vme_start;  in vm_map_get_bounds_in_object()
17168 	*end_offset = MIN(end, entry->vme_end) - entry->vme_start;  in vm_map_get_bounds_in_object()
17169 	*start_offset += VME_OFFSET(entry);  in vm_map_get_bounds_in_object()
17170 	*end_offset += VME_OFFSET(entry);  in vm_map_get_bounds_in_object()
17184 	vm_map_entry_t                  entry;  in vm_map_zero()  local
17209 	if (!vm_map_range_check(map, cur, end, &entry)) {  in vm_map_zero()
17217 	while (entry != vm_map_to_entry(map) && entry->vme_start < end) {  in vm_map_zero()
17221 		vm_object_t object = VME_OBJECT(entry);  in vm_map_zero()
17223 		ret = vm_map_zero_entry_preflight(entry);  in vm_map_zero()
17230 			entry = entry->vme_next;  in vm_map_zero()
17234 		vm_map_get_bounds_in_object(entry, cur, end, &cur_offset, &end_offset);  in vm_map_zero()
17266 			if (!vm_map_range_check(map, cur, end, &entry)) {  in vm_map_zero()
17275 		entry = entry->vme_next;  in vm_map_zero()
18480 	vm_map_entry_t          entry;  in vm_map_copy_to_physcopy()  local
18608 		entry = vm_map_copy_first_entry(copy_map);  in vm_map_copy_to_physcopy()
18609 		vm_map_copy_entry_unlink(copy_map, entry);  in vm_map_copy_to_physcopy()
18610 		if (entry->is_sub_map) {  in vm_map_copy_to_physcopy()
18611 			vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_copy_to_physcopy()
18613 			vm_object_deallocate(VME_OBJECT(entry));  in vm_map_copy_to_physcopy()
18615 		vm_map_copy_entry_dispose(entry);  in vm_map_copy_to_physcopy()
18641 	vm_map_entry_t  entry, target_entry;  in vm_map_copy_adjust_get_target_copy_map()  local
18654 	for (entry = vm_map_copy_first_entry(copy_map);  in vm_map_copy_adjust_get_target_copy_map()
18655 	    entry != vm_map_copy_to_entry(copy_map);  in vm_map_copy_adjust_get_target_copy_map()
18656 	    entry = entry->vme_next) {  in vm_map_copy_adjust_get_target_copy_map()
18658 		vm_map_entry_copy_full(target_entry, entry);  in vm_map_copy_adjust_get_target_copy_map()
18669 	entry = VM_MAP_ENTRY_NULL;  in vm_map_copy_adjust_get_target_copy_map()
18686 	vm_map_entry_t  entry, next_entry;  in vm_map_copy_trim()  local
18698 	for (entry = vm_map_copy_first_entry(copy_map);  in vm_map_copy_trim()
18699 	    entry != vm_map_copy_to_entry(copy_map);  in vm_map_copy_trim()
18700 	    entry = next_entry) {  in vm_map_copy_trim()
18701 		next_entry = entry->vme_next;  in vm_map_copy_trim()
18702 		if (entry->vme_end <= trim_start) {  in vm_map_copy_trim()
18706 		if (entry->vme_start >= trim_end) {  in vm_map_copy_trim()
18711 		vm_map_copy_clip_start(copy_map, entry, trim_start);  in vm_map_copy_trim()
18712 		vm_map_copy_clip_end(copy_map, entry, trim_end);  in vm_map_copy_trim()
18714 		copy_map->size -= entry->vme_end - entry->vme_start;  in vm_map_copy_trim()
18715 		vm_map_copy_entry_unlink(copy_map, entry);  in vm_map_copy_trim()
18716 		if (entry->is_sub_map) {  in vm_map_copy_trim()
18717 			vm_map_deallocate(VME_SUBMAP(entry));  in vm_map_copy_trim()
18719 			vm_object_deallocate(VME_OBJECT(entry));  in vm_map_copy_trim()
18721 		vm_map_copy_entry_dispose(entry);  in vm_map_copy_trim()
18722 		entry = VM_MAP_ENTRY_NULL;  in vm_map_copy_trim()
18758 	vm_map_entry_t          entry, target_entry;  in vm_map_copy_adjust_to_target()  local
18864 		for (entry = vm_map_copy_first_entry(copy_map);  in vm_map_copy_adjust_to_target()
18865 		    entry != vm_map_copy_to_entry(copy_map);  in vm_map_copy_adjust_to_target()
18866 		    entry = entry->vme_next) {  in vm_map_copy_adjust_to_target()
18869 			object_offset_start = VME_OFFSET(entry);  in vm_map_copy_adjust_to_target()
18871 			object_offset_end += entry->vme_end - entry->vme_start;  in vm_map_copy_adjust_to_target()
18873 				if (entry == vm_map_copy_first_entry(copy_map) && !copy) {  in vm_map_copy_adjust_to_target()
18880 				if (entry->vme_next == vm_map_copy_to_entry(copy_map) && !copy) {  in vm_map_copy_adjust_to_target()
18888 	entry = VM_MAP_ENTRY_NULL;  in vm_map_copy_adjust_to_target()
19423 			vm_map_entry_t entry = vm_map_copy_first_entry(copy_map);  in vm_map_remap()  local
19425 			vm_map_copy_entry_unlink(copy_map, entry);  in vm_map_remap()
19438 				if (entry->vme_permanent) {  in vm_map_remap()
19440 					if ((entry->max_protection & VM_PROT_EXECUTE) &&  in vm_map_remap()
19459 						    entry->protection,  in vm_map_remap()
19460 						    entry->max_protection,  in vm_map_remap()
19463 						    vm_map_entry_t, entry,  in vm_map_remap()
19464 						    vm_map_offset_t, entry->vme_start,  in vm_map_remap()
19465 						    vm_map_offset_t, entry->vme_end,  in vm_map_remap()
19466 						    vm_prot_t, entry->protection,  in vm_map_remap()
19467 						    vm_prot_t, entry->max_protection,  in vm_map_remap()
19468 						    int, VME_ALIAS(entry));  in vm_map_remap()
19479 					entry->max_protection |= VM_PROT_WRITE;  in vm_map_remap()
19484 				entry->max_protection = VM_PROT_READ;  in vm_map_remap()
19485 				entry->protection = VM_PROT_READ;  in vm_map_remap()
19486 				entry->vme_resilient_codesign = TRUE;  in vm_map_remap()
19488 			entry->vme_start += target_addr;  in vm_map_remap()
19489 			entry->vme_end += target_addr;  in vm_map_remap()
19490 			assert(!entry->map_aligned);  in vm_map_remap()
19492 			    !entry->is_sub_map &&  in vm_map_remap()
19493 			    (VME_OBJECT(entry) == VM_OBJECT_NULL ||  in vm_map_remap()
19494 			    VME_OBJECT(entry)->internal)) {  in vm_map_remap()
19495 				entry->vme_resilient_media = TRUE;  in vm_map_remap()
19497 			assert(VM_MAP_PAGE_ALIGNED(entry->vme_start, MIN(target_page_mask, PAGE_MASK)));  in vm_map_remap()
19498 			assert(VM_MAP_PAGE_ALIGNED(entry->vme_end, MIN(target_page_mask, PAGE_MASK)));  in vm_map_remap()
19499 			assert(VM_MAP_PAGE_ALIGNED(VME_OFFSET(entry), MIN(target_page_mask, PAGE_MASK)));  in vm_map_remap()
19500 			vm_map_store_entry_link(target_map, insp_entry, entry,  in vm_map_remap()
19502 			insp_entry = entry;  in vm_map_remap()
19770 	vm_map_entry_t entry;  in vm_map_check_protection()  local
19805 	entry = tmp_entry;  in vm_map_check_protection()
19808 		if (entry == vm_map_to_entry(map)) {  in vm_map_check_protection()
19817 		if (start < entry->vme_start) {  in vm_map_check_protection()
19826 		if ((entry->protection & protection) != protection) {  in vm_map_check_protection()
19833 		start = entry->vme_end;  in vm_map_check_protection()
19834 		entry = entry->vme_next;  in vm_map_check_protection()
19848 	vm_map_entry_t          entry;  in vm_map_purgable_control()  local
19884 	if (!vm_map_lookup_entry(map, address, &entry) || entry->is_sub_map) {  in vm_map_purgable_control()
19892 	if ((entry->protection & VM_PROT_WRITE) == 0 &&  in vm_map_purgable_control()
19901 	object = VME_OBJECT(entry);  in vm_map_purgable_control()
19914 	if (VME_OFFSET(entry) != 0 ||  in vm_map_purgable_control()
19915 	    entry->vme_end - entry->vme_start != object->vo_size) {  in vm_map_purgable_control()
19926 	assert(!entry->is_sub_map);  in vm_map_purgable_control()
19927 	assert(!entry->use_pmap); /* purgeable has its own accounting */  in vm_map_purgable_control()
20719 	vm_map_entry_t          entry;  in vm_map_msync()  local
20758 		    &entry)) {  in vm_map_msync()
20781 			if (entry == vm_map_to_entry(map) &&  in vm_map_msync()
20782 			    entry->vme_next == entry) {  in vm_map_msync()
20791 			    (entry->vme_next->vme_start < address)) {  in vm_map_msync()
20798 			skip = (entry->vme_next->vme_start - address);  in vm_map_msync()
20804 			address = entry->vme_next->vme_start;  in vm_map_msync()
20809 		offset = address - entry->vme_start;  in vm_map_msync()
20816 		if (amount_left + entry->vme_start + offset > entry->vme_end) {  in vm_map_msync()
20817 			flush_size = entry->vme_end -  in vm_map_msync()
20818 			    (entry->vme_start + offset);  in vm_map_msync()
20825 		if (entry->is_sub_map == TRUE) {  in vm_map_msync()
20829 			local_map = VME_SUBMAP(entry);  in vm_map_msync()
20830 			local_offset = VME_OFFSET(entry);  in vm_map_msync()
20843 		object = VME_OBJECT(entry);  in vm_map_msync()
20853 		offset += VME_OFFSET(entry);  in vm_map_msync()
21717 	vm_map_entry_t entry;  in vm_map_sign()  local
21735 	if (!vm_map_lookup_entry(map, start, &entry) || entry->is_sub_map) {  in vm_map_sign()
21743 	if ((entry->vme_start > start) || (entry->vme_end < end)) {  in vm_map_sign()
21752 	object = VME_OBJECT(entry);  in vm_map_sign()
21768 		    start - entry->vme_start + VME_OFFSET(entry));  in vm_map_sign()
21813 	vm_map_entry_t  entry = VM_MAP_ENTRY_NULL;  in vm_map_partial_reap()  local
21820 	for (entry = vm_map_first_entry(map);  in vm_map_partial_reap()
21821 	    entry != vm_map_to_entry(map);  in vm_map_partial_reap()
21822 	    entry = next_entry) {  in vm_map_partial_reap()
21823 		next_entry = entry->vme_next;  in vm_map_partial_reap()
21825 		if (!entry->is_sub_map &&  in vm_map_partial_reap()
21826 		    VME_OBJECT(entry) &&  in vm_map_partial_reap()
21827 		    (VME_OBJECT(entry)->internal == TRUE) &&  in vm_map_partial_reap()
21828 		    (os_ref_get_count_raw(&VME_OBJECT(entry)->ref_count) == 1)) {  in vm_map_partial_reap()
21829 			*reclaimed_resident += VME_OBJECT(entry)->resident_page_count;  in vm_map_partial_reap()
21830 			*reclaimed_compressed += vm_compressor_pager_get_count(VME_OBJECT(entry)->pager);  in vm_map_partial_reap()
21832 			(void)vm_map_delete(map, entry->vme_start,  in vm_map_partial_reap()
21833 			    entry->vme_end, VM_MAP_REMOVE_NO_YIELD,  in vm_map_partial_reap()
21853 	vm_map_entry_t entry;  in vm_map_disconnect_page_mappings()  local
21860 		for (entry = vm_map_first_entry(map);  in vm_map_disconnect_page_mappings()
21861 		    entry != vm_map_to_entry(map);  in vm_map_disconnect_page_mappings()
21862 		    entry = entry->vme_next) {  in vm_map_disconnect_page_mappings()
21863 			if (entry->is_sub_map && entry->use_pmap) {  in vm_map_disconnect_page_mappings()
21870 				vm_map_clip_unnest(map, entry, entry->vme_start, entry->vme_end);  in vm_map_disconnect_page_mappings()
21880 	for (entry = vm_map_first_entry(map);  in vm_map_disconnect_page_mappings()
21881 	    entry != vm_map_to_entry(map);  in vm_map_disconnect_page_mappings()
21882 	    entry = entry->vme_next) {  in vm_map_disconnect_page_mappings()
21883 		if (!entry->is_sub_map && ((VME_OBJECT(entry) == 0) ||  in vm_map_disconnect_page_mappings()
21884 		    (VME_OBJECT(entry)->phys_contiguous))) {  in vm_map_disconnect_page_mappings()
21887 		if (entry->is_sub_map) {  in vm_map_disconnect_page_mappings()
21888 			assert(!entry->use_pmap);  in vm_map_disconnect_page_mappings()
21891 		pmap_remove_options(map->pmap, entry->vme_start, entry->vme_end, 0);  in vm_map_disconnect_page_mappings()
21944     kern_return_t (^entry_handler)(void* entry))
21954 	vm_map_entry_t entry = vm_map_first_entry(map);  variable
21956 	while (entry != vm_map_to_entry(map)) {
21957 		error = entry_handler(entry);
21961 		entry = entry->vme_next;
21976     kern_return_t (^entry_handler)(void* entry))
22001 	vm_map_entry_t entry = (vm_map_entry_t)pentry;  in vm_map_dump_entry_and_compressor_pager()  local
22003 	out_entry->vmei_start = entry->vme_start;  in vm_map_dump_entry_and_compressor_pager()
22004 	out_entry->vmei_end = entry->vme_end;  in vm_map_dump_entry_and_compressor_pager()
22005 	out_entry->vmei_alias = VME_ALIAS(entry);  in vm_map_dump_entry_and_compressor_pager()
22006 	out_entry->vmei_offset = VME_OFFSET(entry);  in vm_map_dump_entry_and_compressor_pager()
22007 	out_entry->vmei_is_sub_map = entry->is_sub_map;  in vm_map_dump_entry_and_compressor_pager()
22008 	out_entry->vmei_protection = entry->protection;  in vm_map_dump_entry_and_compressor_pager()
22018 	vm_object_t object = VME_OBJECT(entry);  in vm_map_dump_entry_and_compressor_pager()
22320 	vm_map_entry_t  entry)  in vm_map_entry_should_cow_for_true_share()  argument
22324 	if (entry->is_sub_map) {  in vm_map_entry_should_cow_for_true_share()
22329 	if (entry->needs_copy) {  in vm_map_entry_should_cow_for_true_share()
22334 	if (VME_ALIAS(entry) != VM_MEMORY_MALLOC &&  in vm_map_entry_should_cow_for_true_share()
22335 	    VME_ALIAS(entry) != VM_MEMORY_MALLOC_SMALL) {  in vm_map_entry_should_cow_for_true_share()
22340 	if (entry->wired_count) {  in vm_map_entry_should_cow_for_true_share()
22346 	object = VME_OBJECT(entry);  in vm_map_entry_should_cow_for_true_share()
22368 	if (VME_ALIAS(entry) == VM_MEMORY_MALLOC &&  in vm_map_entry_should_cow_for_true_share()
22374 	if (VME_ALIAS(entry) == VM_MEMORY_MALLOC_SMALL &&  in vm_map_entry_should_cow_for_true_share()
22502 	vm_map_entry_t  entry;  in vm_map_query_volatile()  local
22513 	for (entry = vm_map_first_entry(map);  in vm_map_query_volatile()
22514 	    entry != vm_map_to_entry(map);  in vm_map_query_volatile()
22515 	    entry = entry->vme_next) {  in vm_map_query_volatile()
22518 		if (entry->is_sub_map) {  in vm_map_query_volatile()
22521 		if (!(entry->protection & VM_PROT_WRITE)) {  in vm_map_query_volatile()
22524 		object = VME_OBJECT(entry);  in vm_map_query_volatile()
22532 		if (VME_OFFSET(entry)) {  in vm_map_query_volatile()
22543 		if ((VME_OFFSET(entry) / PAGE_SIZE) >= resident_count) {  in vm_map_query_volatile()
22546 			resident_count -= (VME_OFFSET(entry) / PAGE_SIZE);  in vm_map_query_volatile()
22549 		volatile_virtual_size += entry->vme_end - entry->vme_start;  in vm_map_query_volatile()
22558 		    entry->vme_start,  in vm_map_query_volatile()
22559 		    entry->vme_end,  in vm_map_query_volatile()
22583 	vm_map_entry_t  entry;  in vm_map_sizes()  local
22600 	for (entry = vm_map_first_entry(map);; entry = entry->vme_next) {  in vm_map_sizes()
22601 		end = (entry == vm_map_to_entry(map));  in vm_map_sizes()
22604 			free = entry->vme_end   - prev;  in vm_map_sizes()
22606 			free = entry->vme_start - prev;  in vm_map_sizes()
22617 		prev = entry->vme_end;  in vm_map_sizes()
22634 	vm_map_entry_t  entry;  in vm_map_shadow_max()  local
22645 	for (entry = vm_map_first_entry(map);  in vm_map_shadow_max()
22646 	    entry != vm_map_to_entry(map);  in vm_map_shadow_max()
22647 	    entry = entry->vme_next) {  in vm_map_shadow_max()
22648 		if (entry->is_sub_map) {  in vm_map_shadow_max()
22651 		object = VME_OBJECT(entry);  in vm_map_shadow_max()
22696 	vm_map_entry_t          entry,  in vm_map_entry_cs_associate()  argument
22706 	    entry->is_sub_map || /* XXX FBDP: recurse on sub-range? */  in vm_map_entry_cs_associate()
22708 	    VME_OBJECT(entry) == VM_OBJECT_NULL) {  in vm_map_entry_cs_associate()
22712 	if (!(entry->protection & VM_PROT_EXECUTE)) {  in vm_map_entry_cs_associate()
22718 		    (entry->max_protection & VM_PROT_EXECUTE)) {  in vm_map_entry_cs_associate()
22767 			entry->vme_start,  in vm_map_entry_cs_associate()
22768 			entry->vme_end - entry->vme_start);  in vm_map_entry_cs_associate()
22781 			entry->vme_xnu_user_debug = TRUE;  in vm_map_entry_cs_associate()
22789 			    map, entry,  in vm_map_entry_cs_associate()
22790 			    (uint64_t)entry->vme_start, (uint64_t)entry->vme_end,  in vm_map_entry_cs_associate()
22791 			    entry->vme_xnu_user_debug,  in vm_map_entry_cs_associate()
22798 	if (entry->used_for_jit) {  in vm_map_entry_cs_associate()
22801 			entry->vme_start,  in vm_map_entry_cs_associate()
22802 			entry->vme_end - entry->vme_start);  in vm_map_entry_cs_associate()
22806 	cs_object = VME_OBJECT(entry);  in vm_map_entry_cs_associate()
22808 	cs_offset = VME_OFFSET(entry);  in vm_map_entry_cs_associate()
22861 		    entry->vme_start,  in vm_map_entry_cs_associate()
22862 		    (entry->vme_end - entry->vme_start),  in vm_map_entry_cs_associate()
22872 		    vm_map_offset_t, entry->vme_start,  in vm_map_entry_cs_associate()
22873 		    vm_map_offset_t, entry->vme_end);  in vm_map_entry_cs_associate()
22880 			entry->vme_permanent = TRUE;  in vm_map_entry_cs_associate()
22891 		entry->csm_associated = TRUE;  in vm_map_entry_cs_associate()
22900 		    vm_map_offset_t, entry->vme_start,  in vm_map_entry_cs_associate()
22901 		    vm_map_offset_t, entry->vme_end,  in vm_map_entry_cs_associate()
22909 		    vm_map_offset_t, entry->vme_start,  in vm_map_entry_cs_associate()
22910 		    vm_map_offset_t, entry->vme_end,  in vm_map_entry_cs_associate()
22920 			entry->protection &= ~VM_PROT_ALLEXEC;  in vm_map_entry_cs_associate()
22921 			entry->max_protection &= ~VM_PROT_ALLEXEC;  in vm_map_entry_cs_associate()
23739 	vm_map_entry_t          entry;  in vm_map_get_phys_page()  local
23745 	while (vm_map_lookup_entry(map, map_offset, &entry)) {  in vm_map_get_phys_page()
23746 		if (entry->is_sub_map) {  in vm_map_get_phys_page()
23748 			vm_map_lock(VME_SUBMAP(entry));  in vm_map_get_phys_page()
23750 			map = VME_SUBMAP(entry);  in vm_map_get_phys_page()
23751 			map_offset = (VME_OFFSET(entry) +  in vm_map_get_phys_page()
23752 			    (map_offset - entry->vme_start));  in vm_map_get_phys_page()
23756 		if (VME_OBJECT(entry) == VM_OBJECT_NULL) {  in vm_map_get_phys_page()
23760 		if (VME_OBJECT(entry)->phys_contiguous) {  in vm_map_get_phys_page()
23765 			if (VME_OBJECT(entry)->vo_shadow_offset == 0) {  in vm_map_get_phys_page()
23774 			offset = (VME_OFFSET(entry) +  in vm_map_get_phys_page()
23775 			    (map_offset - entry->vme_start));  in vm_map_get_phys_page()
23777 			    ((VME_OBJECT(entry)->vo_shadow_offset  in vm_map_get_phys_page()
23781 		offset = (VME_OFFSET(entry) + (map_offset - entry->vme_start));  in vm_map_get_phys_page()
23782 		object = VME_OBJECT(entry);  in vm_map_get_phys_page()
24107 		vm_map_entry_t entry;  in vm_map_range_configure()  local
24115 		vm_map_lookup_entry_or_next(map, data_range.max_address, &entry);  in vm_map_range_configure()
24116 		if (entry == vm_map_to_entry(map) || entry->vme_start > vm_map_max(map)) {  in vm_map_range_configure()
24119 			size = entry->vme_start - data_range.max_address;  in vm_map_range_configure()
24323 	vm_map_entry_t entry;  in vm_map_entry_has_device_pager()  local
24333 		if (!vm_map_lookup_entry(map, vaddr, &entry)) {  in vm_map_entry_has_device_pager()
24337 		if (entry->is_sub_map) {  in vm_map_entry_has_device_pager()
24339 			vm_map_t submap = VME_SUBMAP(entry);  in vm_map_entry_has_device_pager()
24346 		object = VME_OBJECT(entry);  in vm_map_entry_has_device_pager()