287 	vm_map_t        dst_map,
294 	vm_map_t        dst_map,
346 	vm_map_t                   dst_map,
351 	vm_map_t                   dst_map,
9100 	vm_map_t        dst_map __unused,  in vm_map_entry_is_overwritable()
9109 	    vm_map_cs_enforcement(dst_map) &&  in vm_map_entry_is_overwritable()
9110 	    !dst_map->cs_debugged) {  in vm_map_entry_is_overwritable()
9159 	vm_map_t        dst_map,  in vm_map_overwrite_submap_recurse()  argument
9179 	    VM_MAP_PAGE_MASK(dst_map));  in vm_map_overwrite_submap_recurse()
9180 	vm_map_lock(dst_map);  in vm_map_overwrite_submap_recurse()
9183 	if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {  in vm_map_overwrite_submap_recurse()
9184 		vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9188 	vm_map_clip_start(dst_map,  in vm_map_overwrite_submap_recurse()
9191 	    VM_MAP_PAGE_MASK(dst_map)));  in vm_map_overwrite_submap_recurse()
9211 				vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_overwrite_submap_recurse()
9227 			vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9240 			vm_map_lock(dst_map);  in vm_map_overwrite_submap_recurse()
9241 			if (!vm_map_lookup_entry(dst_map, local_end,  in vm_map_overwrite_submap_recurse()
9243 				vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9252 			vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9256 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_overwrite_submap_recurse()
9257 			vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9271 			vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_overwrite_submap_recurse()
9280 			vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9286 		if ((next == vm_map_to_entry(dst_map)) ||  in vm_map_overwrite_submap_recurse()
9288 			vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9300 				vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9308 	vm_map_unlock(dst_map);  in vm_map_overwrite_submap_recurse()
9368 	vm_map_t                dst_map,  in vm_map_copy_overwrite_nested()  argument
9394 			dst_map, &dst_addr,  in vm_map_copy_overwrite_nested()
9423 	    VM_MAP_PAGE_MASK(dst_map)) ||  in vm_map_copy_overwrite_nested()
9425 	    VM_MAP_PAGE_MASK(dst_map)) ||  in vm_map_copy_overwrite_nested()
9427 	    VM_MAP_PAGE_MASK(dst_map)) ||  in vm_map_copy_overwrite_nested()
9428 	    copy_page_shift != VM_MAP_PAGE_SHIFT(dst_map)) {  in vm_map_copy_overwrite_nested()
9431 		    VM_MAP_PAGE_MASK(dst_map));  in vm_map_copy_overwrite_nested()
9436 	vm_map_lock(dst_map);  in vm_map_copy_overwrite_nested()
9442 	if (dst_addr >= dst_map->max_offset) {  in vm_map_copy_overwrite_nested()
9443 		vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9448 	if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {  in vm_map_copy_overwrite_nested()
9449 		vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9452 	vm_map_clip_start(dst_map,  in vm_map_copy_overwrite_nested()
9455 	    VM_MAP_PAGE_MASK(dst_map)));  in vm_map_copy_overwrite_nested()
9469 				vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_nested()
9490 				vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9499 				vm_map_lock(dst_map);  in vm_map_copy_overwrite_nested()
9505 			if (!vm_map_lookup_entry(dst_map, local_end,  in vm_map_copy_overwrite_nested()
9507 				vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9515 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9519 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_nested()
9520 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9534 			vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_nested()
9548 		if ((next == vm_map_to_entry(dst_map)) ||  in vm_map_copy_overwrite_nested()
9550 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9575 		vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9594 		if (!vm_map_lookup_entry( dst_map, dst_addr, &tmp_entry)) {  in vm_map_copy_overwrite_nested()
9595 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9631 				vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_nested()
9633 				if (!vm_map_lookup_entry(dst_map, base_addr,  in vm_map_copy_overwrite_nested()
9635 					vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9663 						dst_map, entry, sub_end);  in vm_map_copy_overwrite_nested()
9665 						dst_map, entry, sub_start);  in vm_map_copy_overwrite_nested()
9703 				vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9773 						dst_map->pmap,  in vm_map_copy_overwrite_nested()
9810 				vm_map_lock(dst_map);  in vm_map_copy_overwrite_nested()
9811 				if (!vm_map_lookup_entry(dst_map,  in vm_map_copy_overwrite_nested()
9813 					vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9826 			if ((next == vm_map_to_entry(dst_map)) ||  in vm_map_copy_overwrite_nested()
9828 				vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9887 				local_pmap = dst_map->pmap;  in vm_map_copy_overwrite_nested()
9891 				    dst_map, tmp_entry, copy,  in vm_map_copy_overwrite_nested()
9904 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9918 				dst_map,  in vm_map_copy_overwrite_nested()
9950 		vm_map_lock(dst_map);  in vm_map_copy_overwrite_nested()
9952 			if (!vm_map_lookup_entry(dst_map,  in vm_map_copy_overwrite_nested()
9954 				vm_map_unlock(dst_map);  in vm_map_copy_overwrite_nested()
9959 				vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_nested()
9964 		vm_map_clip_start(dst_map,  in vm_map_copy_overwrite_nested()
9967 		    VM_MAP_PAGE_MASK(dst_map)));  in vm_map_copy_overwrite_nested()
10005 	vm_map_t                dst_map,  in vm_map_copy_overwrite()  argument
10039 		dst_map,  in vm_map_copy_overwrite()
10063 		kr = vm_map_copy_overwrite_nested(dst_map,  in vm_map_copy_overwrite()
10077 	    VM_MAP_PAGE_SHIFT(dst_map) < PAGE_SHIFT) {  in vm_map_copy_overwrite()
10081 	if (VM_MAP_PAGE_SHIFT(dst_map) < PAGE_SHIFT) {  in vm_map_copy_overwrite()
10082 		effective_page_mask = VM_MAP_PAGE_MASK(dst_map);  in vm_map_copy_overwrite()
10084 		effective_page_mask = MAX(VM_MAP_PAGE_MASK(dst_map), PAGE_MASK);  in vm_map_copy_overwrite()
10144 	vm_map_lock_read(dst_map);  in vm_map_copy_overwrite()
10145 	if (!vm_map_lookup_entry(dst_map, dst_addr, &entry)) {  in vm_map_copy_overwrite()
10146 		vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite()
10150 	    (entry != vm_map_to_entry(dst_map) &&  in vm_map_copy_overwrite()
10154 			vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite()
10158 	vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite()
10195 		kr = vm_map_copy_overwrite_nested(dst_map,  in vm_map_copy_overwrite()
10253 	kr = vm_map_copy_overwrite_nested(dst_map,  in vm_map_copy_overwrite()
10265 		kr = vm_map_copy_overwrite_nested(dst_map,  in vm_map_copy_overwrite()
10350 	vm_map_t        dst_map,  in vm_map_copy_overwrite_unaligned()  argument
10373 	vm_map_lock_write_to_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10382 		if (entry == vm_map_to_entry(dst_map)) {  in vm_map_copy_overwrite_unaligned()
10383 			vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10394 			vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10399 			vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10402 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_unaligned()
10403 			vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10416 			vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_unaligned()
10451 			if (vm_map_lock_read_to_write(dst_map)) {  in vm_map_copy_overwrite_unaligned()
10452 				vm_map_lock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10458 			    vm_map_always_shadow(dst_map));  in vm_map_copy_overwrite_unaligned()
10460 			vm_map_lock_write_to_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10468 			if (vm_map_lock_read_to_write(dst_map)) {  in vm_map_copy_overwrite_unaligned()
10469 				vm_map_lock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10477 			vm_map_lock_write_to_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10484 		version.main_timestamp = dst_map->timestamp;  in vm_map_copy_overwrite_unaligned()
10487 		vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10497 			dst_map,  in vm_map_copy_overwrite_unaligned()
10546 		vm_map_lock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10547 		if (version.main_timestamp == dst_map->timestamp) {  in vm_map_copy_overwrite_unaligned()
10560 					vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10571 			if (!vm_map_lookup_entry(dst_map, start, &entry)) {  in vm_map_copy_overwrite_unaligned()
10572 				vm_map_unlock_read(dst_map);  in vm_map_copy_overwrite_unaligned()
10610 	vm_map_t        dst_map,  in vm_map_copy_overwrite_aligned()  argument
10632 		if (entry == vm_map_to_entry(dst_map)) {  in vm_map_copy_overwrite_aligned()
10633 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
10646 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
10649 		assert(entry != vm_map_to_entry(dst_map));  in vm_map_copy_overwrite_aligned()
10656 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
10662 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
10666 		if (!vm_map_entry_is_overwritable(dst_map, entry)) {  in vm_map_copy_overwrite_aligned()
10667 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
10681 			vm_map_entry_wait(dst_map, THREAD_UNINT);  in vm_map_copy_overwrite_aligned()
10693 			    VM_MAP_PAGE_MASK(dst_map))) {  in vm_map_copy_overwrite_aligned()
10697 			vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size);  in vm_map_copy_overwrite_aligned()
10736 		    !pmap_has_prot_policy(dst_map->pmap, entry->translated_allow_execute, entry->protection) &&  in vm_map_copy_overwrite_aligned()
10785 			if ((dst_map->pmap != kernel_pmap) &&  in vm_map_copy_overwrite_aligned()
10856 						pmap_unnest(dst_map->pmap,  in vm_map_copy_overwrite_aligned()
10860 						if (dst_map->mapped_in_other_pmaps) {  in vm_map_copy_overwrite_aligned()
10864 								dst_map, entry->vme_start,  in vm_map_copy_overwrite_aligned()
10871 							dst_map, entry->vme_start,  in vm_map_copy_overwrite_aligned()
10878 					if (dst_map->mapped_in_other_pmaps) {  in vm_map_copy_overwrite_aligned()
10891 							dst_map->pmap,  in vm_map_copy_overwrite_aligned()
10949 				    vm_map_always_shadow(dst_map));  in vm_map_copy_overwrite_aligned()
10986 			version.main_timestamp = dst_map->timestamp + 1;  in vm_map_copy_overwrite_aligned()
10988 			vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
11001 				dst_map,  in vm_map_copy_overwrite_aligned()
11039 			vm_map_lock(dst_map);  in vm_map_copy_overwrite_aligned()
11040 			if (version.main_timestamp == dst_map->timestamp &&  in vm_map_copy_overwrite_aligned()
11047 					    VM_MAP_PAGE_MASK(dst_map))) {  in vm_map_copy_overwrite_aligned()
11051 				vm_map_clip_end(dst_map, tmp_entry, start);  in vm_map_copy_overwrite_aligned()
11057 				if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {  in vm_map_copy_overwrite_aligned()
11058 					vm_map_unlock(dst_map);  in vm_map_copy_overwrite_aligned()
11064 					    VM_MAP_PAGE_MASK(dst_map))) {  in vm_map_copy_overwrite_aligned()
11068 				vm_map_clip_start(dst_map, tmp_entry, start);  in vm_map_copy_overwrite_aligned()
11350 	vm_map_t                dst_map,  in vm_map_copy_validate_size()  argument
11379 		    copy_sz <= vm_map_round_page(sz, VM_MAP_PAGE_MASK(dst_map))) {  in vm_map_copy_validate_size()
11392 	vm_map_t                dst_map,  in vm_map_copyout_internal()  argument
11443 		kr = vm_map_copyout_kernel_buffer(dst_map, dst_addr,  in vm_map_copyout_internal()
11456 	if (copy->cpy_hdr.page_shift != VM_MAP_PAGE_SHIFT(dst_map)) {  in vm_map_copyout_internal()
11466 			dst_map,  in vm_map_copyout_internal()
11477 …y->cpy_hdr.page_shift, copy->offset, (uint64_t)copy->size, dst_map, VM_MAP_PAGE_SHIFT(dst_map), ta…  in vm_map_copyout_internal()
11494 	vm_map_kernel_flags_update_range_id(&vmk_flags, dst_map, size);  in vm_map_copyout_internal()
11496 	vm_map_lock(dst_map);  in vm_map_copyout_internal()
11497 	kr = vm_map_locate_space_anywhere(dst_map, size, 0, vmk_flags,  in vm_map_copyout_internal()
11500 		vm_map_unlock(dst_map);  in vm_map_copyout_internal()
11526 	if (copy->cpy_hdr.entries_pageable != dst_map->hdr.entries_pageable) {  in vm_map_copyout_internal()
11544 		copy->cpy_hdr.entries_pageable = dst_map->hdr.entries_pageable;  in vm_map_copyout_internal()
11575 		if (VM_MAP_PAGE_SHIFT(dst_map) == PAGE_SHIFT) {  in vm_map_copyout_internal()
11590 			    VM_MAP_PAGE_MASK(dst_map)));  in vm_map_copyout_internal()
11592 			    VM_MAP_PAGE_MASK(dst_map)));  in vm_map_copyout_internal()
11613 			assert(VM_MAP_PAGE_SHIFT(dst_map) == PAGE_SHIFT);  in vm_map_copyout_internal()
11619 			pmap_pageable(dst_map->pmap,  in vm_map_copyout_internal()
11655 				if (override_nx(dst_map, VME_ALIAS(entry)) &&  in vm_map_copyout_internal()
11682 				    dst_map->pmap,  in vm_map_copyout_internal()
11720 		SAVE_HINT_MAP_WRITE(dst_map, vm_map_copy_last_entry(copy));  in vm_map_copyout_internal()
11722 		SAVE_HINT_MAP_WRITE(dst_map, last);  in vm_map_copyout_internal()
11725 	dst_map->size += size;  in vm_map_copyout_internal()
11732 		vm_map_copy_insert(dst_map, last, copy);  in vm_map_copyout_internal()
11738 		vm_map_copy_remap(dst_map, last, copy, adjustment,  in vm_map_copyout_internal()
11748 	vm_map_unlock(dst_map);  in vm_map_copyout_internal()
11770 	vm_map_t                dst_map,  in vm_map_copyout_size()  argument
11775 	return vm_map_copyout_internal(dst_map, dst_addr, copy, copy_size,  in vm_map_copyout_size()
11794 	vm_map_t                dst_map,  in vm_map_copyout()  argument
11798 	return vm_map_copyout_internal(dst_map, dst_addr, copy, copy ? copy->size : 0,  in vm_map_copyout()