Lines Matching refs:map
203 vm_map_t map,
224 vm_map_t map,
238 vm_map_t map,
243 vm_map_t map,
249 vm_map_t map,
256 vm_map_t map,
274 vm_map_t map,
282 vm_map_t map,
308 vm_map_t map,
327 vm_map_t map,
338 vm_map_t map,
359 vm_map_t map,
370 vm_map_t map,
385 vm_map_t map,
390 vm_map_t map,
395 vm_map_t map,
400 vm_map_t map,
405 vm_map_t map,
410 vm_map_t map,
419 vm_map_t map,
427 vm_map_t map,
439 vm_map_t map);
441 vm_map_t map,
445 vm_map_t map,
582 vm_map_t map __unused, in vm_map_entry_copy_csm_assoc()
601 map, new, new->vme_start, new->vme_end); in vm_map_entry_copy_csm_assoc()
622 vm_map_t map, in vm_map_entry_copy_code_signing() argument
626 if (VM_MAP_POLICY_ALLOW_JIT_COPY(map)) { in vm_map_entry_copy_code_signing()
676 vm_map_t map, in vm_map_entry_copy() argument
688 vm_map_entry_copy_code_signing(map, new, old); in vm_map_entry_copy()
689 vm_map_entry_copy_csm_assoc(map, new, old); in vm_map_entry_copy()
707 vm_map_lock_read_to_write(vm_map_t map) in vm_map_lock_read_to_write() argument
709 if (lck_rw_lock_shared_to_exclusive(&(map)->lock)) { in vm_map_lock_read_to_write()
718 vm_map_try_lock(vm_map_t map) in vm_map_try_lock() argument
720 if (lck_rw_try_lock_exclusive(&(map)->lock)) { in vm_map_try_lock()
729 vm_map_try_lock_read(vm_map_t map) in vm_map_try_lock_read() argument
731 if (lck_rw_try_lock_shared(&(map)->lock)) { in vm_map_try_lock_read()
752 kdp_vm_map_is_acquired_exclusive(vm_map_t map) in kdp_vm_map_is_acquired_exclusive() argument
754 return kdp_lck_rw_lock_is_acquired_exclusive(&map->lock); in kdp_vm_map_is_acquired_exclusive()
838 override_nx(vm_map_t map, uint32_t user_tag) /* map unused on arm */ in override_nx() argument
842 if (map->pmap == kernel_pmap) { in override_nx()
850 if (vm_map_is_64bit(map)) { in override_nx()
865 return (allow_data_exec & current_abi) && (map->map_disallow_data_exec == FALSE); in override_nx()
941 vm_map_require(vm_map_t map) in vm_map_require() argument
943 zone_id_require(ZONE_ID_VM_MAP, sizeof(struct _vm_map), map); in vm_map_require()
969 vm_map_t map, in vm_map_set_cache_attr() argument
976 vm_map_lock_read(map); in vm_map_set_cache_attr()
978 if (!vm_map_lookup_entry(map, va, &map_entry) || in vm_map_set_cache_attr()
1001 vm_map_unlock_read(map); in vm_map_set_cache_attr()
1018 vm_map_t map, in vm_map_apple_protected() argument
1039 if (__improbable(vm_map_range_overflows(map, start, end - start))) { in vm_map_apple_protected()
1044 start_aligned = vm_map_trunc_page(start_aligned, VM_MAP_PAGE_MASK(map)); in vm_map_apple_protected()
1045 end_aligned = vm_map_round_page(end_aligned, VM_MAP_PAGE_MASK(map)); in vm_map_apple_protected()
1067 vm_map_lock(map); in vm_map_apple_protected()
1071 if (!vm_map_lookup_entry(map, in vm_map_apple_protected()
1100 vm_map_clip_start(map, map_entry, start_aligned); in vm_map_apple_protected()
1101 vm_map_clip_end(map, map_entry, end_aligned); in vm_map_apple_protected()
1105 vm_map_unlock(map); in vm_map_apple_protected()
1142 if (vm_map_is_alien(map)) { in vm_map_apple_protected()
1182 kr = mach_vm_map_kernel(map, in vm_map_apple_protected()
1208 map, in vm_map_apple_protected()
1239 vm_map_unlock(map); in vm_map_apple_protected()
1462 vm_map_disable_hole_optimization(vm_map_t map) in vm_map_disable_hole_optimization() argument
1466 if (map->holelistenabled) { in vm_map_disable_hole_optimization()
1467 head_entry = hole_entry = CAST_TO_VM_MAP_ENTRY(map->holes_list); in vm_map_disable_hole_optimization()
1483 map->holes_list = NULL; in vm_map_disable_hole_optimization()
1484 map->holelistenabled = FALSE; in vm_map_disable_hole_optimization()
1486 map->first_free = vm_map_first_entry(map); in vm_map_disable_hole_optimization()
1487 SAVE_HINT_HOLE_WRITE(map, NULL); in vm_map_disable_hole_optimization()
1492 vm_kernel_map_is_kernel(vm_map_t map) in vm_kernel_map_is_kernel() argument
1494 return map->pmap == kernel_pmap; in vm_kernel_map_is_kernel()
1706 vm_map_t map, in vm_map_adjust_offsets() argument
1710 assert(map->min_offset == 0); in vm_map_adjust_offsets()
1711 assert(map->max_offset == max_off - min_off); in vm_map_adjust_offsets()
1712 assert(map->hdr.nentries == 0); in vm_map_adjust_offsets()
1713 assert(os_ref_get_count_raw(&map->map_refcnt) == 2); in vm_map_adjust_offsets()
1715 map->min_offset = min_off; in vm_map_adjust_offsets()
1716 map->max_offset = max_off; in vm_map_adjust_offsets()
1718 if (map->holelistenabled) { in vm_map_adjust_offsets()
1719 struct vm_map_links *hole = map->holes_list; in vm_map_adjust_offsets()
1732 vm_map_adjusted_size(vm_map_t map) in vm_map_adjusted_size() argument
1738 if (map == NULL || (map->size == 0)) { in vm_map_adjusted_size()
1742 map_size = map->size; in vm_map_adjusted_size()
1744 if (map->reserved_regions == FALSE || !vm_map_is_exotic(map) || map->terminated) { in vm_map_adjusted_size()
1753 num_regions = ml_get_vm_reserved_regions(vm_map_is_64bit(map), ®ions); in vm_map_adjusted_size()
1784 #define vm_map_entry_create(map) _vm_map_entry_create(&(map)->hdr) argument
1911 vm_map_t map) in first_free_is_valid() argument
1917 return first_free_is_valid_store( map ); in first_free_is_valid()
1935 vm_map_t map) in vm_map_destroy() argument
1942 vm_map_lock(map); in vm_map_destroy()
1944 map->terminated = true; in vm_map_destroy()
1946 (void)vm_map_delete(map, map->min_offset, map->max_offset, flags, in vm_map_destroy()
1949 (void)vm_map_delete(map, 0x0, 0xFFFFFFFFFFFFF000ULL, flags, in vm_map_destroy()
1952 vm_map_disable_hole_optimization(map); in vm_map_destroy()
1953 vm_map_corpse_footprint_destroy(map); in vm_map_destroy()
1955 vm_map_unlock(map); in vm_map_destroy()
1959 assert(map->hdr.nentries == 0); in vm_map_destroy()
1961 if (map->pmap) { in vm_map_destroy()
1962 pmap_destroy(map->pmap); in vm_map_destroy()
1965 lck_rw_destroy(&map->lock, &vm_map_lck_grp); in vm_map_destroy()
1968 kfree_data(map->extra_ranges, in vm_map_destroy()
1969 map->extra_ranges_count * sizeof(struct vm_map_user_range)); in vm_map_destroy()
1972 zfree_id(ZONE_ID_VM_MAP, map); in vm_map_destroy()
1993 vm_map_t task_map = task->map; in find_largest_process_vm_map_entries()
2021 vm_map_t map, in vm_map_lookup_entry() argument
2034 if (map->pmap == kernel_pmap) { in vm_map_lookup_entry()
2039 result = vm_map_store_lookup_entry( map, address, entry ); in vm_map_lookup_entry()
2046 vm_map_t map, in vm_map_lookup_entry_or_next() argument
2050 if (vm_map_lookup_entry(map, address, entry)) { in vm_map_lookup_entry_or_next()
2061 vm_map_t map, in vm_map_lookup_entry_allow_pgz() argument
2071 return vm_map_store_lookup_entry( map, address, entry ); in vm_map_lookup_entry_allow_pgz()
2083 vm_map_t map, in vm_map_range_invalid_panic() argument
2086 panic("invalid range ID (%u) for map %p", range_id, map); in vm_map_range_invalid_panic()
2096 vm_map_t map, in vm_map_get_range() argument
2105 if (map == kernel_map) { in vm_map_get_range()
2123 vm_map_range_invalid_panic(map, range_id); in vm_map_get_range()
2141 } else if (map->uses_user_ranges) { in vm_map_get_range()
2144 effective_range = map->default_range; in vm_map_get_range()
2147 effective_range = map->data_range; in vm_map_get_range()
2150 if (map->large_file_range.min_address != map->large_file_range.max_address) { in vm_map_get_range()
2152 effective_range = map->large_file_range; in vm_map_get_range()
2160 effective_range = map->data_range; in vm_map_get_range()
2171 vm_map_range_invalid_panic(map, range_id); in vm_map_get_range()
2183 effective_range.min_address = MAX(map->min_offset, VM_MAP_PAGE_SIZE(map)); in vm_map_get_range()
2184 effective_range.max_address = map->max_offset; in vm_map_get_range()
2192 vm_map_t map, in vm_map_locate_space_anywhere() argument
2211 if (__improbable(map->wait_for_space)) { in vm_map_locate_space_anywhere()
2216 assert(!map->holelistenabled && in vm_map_locate_space_anywhere()
2221 *start_inout <= map->min_offset); in vm_map_locate_space_anywhere()
2228 guard_offset = VM_MAP_PAGE_SIZE(map); in vm_map_locate_space_anywhere()
2239 effective_range = vm_map_get_range(map, start_inout, &vmk_flags, size, in vm_map_locate_space_anywhere()
2249 assert(map != kernel_map); in vm_map_locate_space_anywhere()
2250 effective_range.max_address = MIN(map->max_offset, 0x00000000FFFFF000ULL); in vm_map_locate_space_anywhere()
2269 if (map->jit_entry_exists && in vm_map_locate_space_anywhere()
2270 !VM_MAP_POLICY_ALLOW_MULTIPLE_JIT(map)) { in vm_map_locate_space_anywhere()
2273 if (VM_MAP_POLICY_ALLOW_JIT_RANDOM_ADDRESS(map)) { in vm_map_locate_space_anywhere()
2281 kr = vm_map_random_address_for_size(map, &hint, size, vmk_flags); in vm_map_locate_space_anywhere()
2287 else if ((hint == 0 || hint == vm_map_min(map)) && in vm_map_locate_space_anywhere()
2288 !map->disable_vmentry_reuse && in vm_map_locate_space_anywhere()
2289 map->vmmap_high_start != 0) { in vm_map_locate_space_anywhere()
2290 hint = map->vmmap_high_start; in vm_map_locate_space_anywhere()
2303 entry = vm_map_store_find_space(map, in vm_map_locate_space_anywhere()
2309 if (map->wait_for_space && in vm_map_locate_space_anywhere()
2312 assert_wait((event_t)map, THREAD_ABORTSAFE); in vm_map_locate_space_anywhere()
2313 vm_map_unlock(map); in vm_map_locate_space_anywhere()
2315 vm_map_lock(map); in vm_map_locate_space_anywhere()
2357 vm_map_t map, in vm_map_locate_space_fixed() argument
2371 effective_min_offset = map->min_offset; in vm_map_locate_space_fixed()
2372 effective_max_offset = map->max_offset; in vm_map_locate_space_fixed()
2379 if (vm_map_is_64bit(map)) { in vm_map_locate_space_fixed()
2384 effective_max_offset = MIN(map->max_offset, 0x00000000FFFFF000ULL); in vm_map_locate_space_fixed()
2388 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT && in vm_map_locate_space_fixed()
2390 map->pmap == kernel_pmap && in vm_map_locate_space_fixed()
2411 if (map->uses_user_ranges) { in vm_map_locate_space_fixed()
2414 vm_map_user_range_resolve(map, start, 1, &r); in vm_map_locate_space_fixed()
2424 (map == kernel_map)) { in vm_map_locate_space_fixed()
2466 remove_kr = vm_map_delete(map, start, end, remove_flags, in vm_map_locate_space_fixed()
2478 if (vm_map_lookup_entry(map, start, &entry)) { in vm_map_locate_space_fixed()
2488 if ((entry->vme_next != vm_map_to_entry(map)) && in vm_map_locate_space_fixed()
2512 vm_map_t map, in vm_map_find_space() argument
2526 new_entry = vm_map_entry_create(map); in vm_map_find_space()
2531 if (VM_MAP_PAGE_SHIFT(map) != PAGE_SHIFT) { in vm_map_find_space()
2538 vm_map_lock(map); in vm_map_find_space()
2540 kr = vm_map_locate_space_anywhere(map, size, mask, vmk_flags, in vm_map_find_space()
2543 vm_map_unlock(map); in vm_map_find_space()
2564 assert(VM_MAP_PAGE_ALIGNED(new_entry->vme_start, VM_MAP_PAGE_MASK(map))); in vm_map_find_space()
2565 assert(VM_MAP_PAGE_ALIGNED(new_entry->vme_end, VM_MAP_PAGE_MASK(map))); in vm_map_find_space()
2571 vm_map_store_entry_link(map, entry, new_entry, in vm_map_find_space()
2573 map->size += size; in vm_map_find_space()
2578 SAVE_HINT_MAP_WRITE(map, new_entry); in vm_map_find_space()
2603 vm_map_t map, in vm_map_pmap_enter() argument
2615 if (map->pmap == 0) { in vm_map_pmap_enter()
2619 assert(VM_MAP_PAGE_SHIFT(map) == PAGE_SHIFT); in vm_map_pmap_enter()
2649 map, (unsigned long long)addr, object, (unsigned long long)offset); in vm_map_pmap_enter()
2652 kr = vm_fault_enter(m, map->pmap, in vm_map_pmap_enter()
2674 vm_map_t map, in vm_map_random_address_for_size() argument
2691 effective_range = vm_map_get_range(map, address, &vmk_flags, size, in vm_map_random_address_for_size()
2700 assert(VM_MAP_PAGE_ALIGNED(size, VM_MAP_PAGE_MASK(map))); in vm_map_random_address_for_size()
2708 random_addr <<= VM_MAP_PAGE_SHIFT(map); in vm_map_random_address_for_size()
2711 VM_MAP_PAGE_MASK(map)); in vm_map_random_address_for_size()
2714 if (map->pmap == kernel_pmap && pgz_owned(random_addr)) { in vm_map_random_address_for_size()
2719 if (vm_map_lookup_entry(map, random_addr, &prev_entry) == FALSE) { in vm_map_random_address_for_size()
2720 if (prev_entry == vm_map_to_entry(map)) { in vm_map_random_address_for_size()
2721 next_entry = vm_map_first_entry(map); in vm_map_random_address_for_size()
2725 if (next_entry == vm_map_to_entry(map)) { in vm_map_random_address_for_size()
2726 hole_end = vm_map_max(map); in vm_map_random_address_for_size()
2780 vm_map_t map, in vm_map_enter() argument
2874 map->pmap != kernel_pmap && in vm_map_enter()
2878 : (vm_map_cs_enforcement(map) in vm_map_enter()
2880 || !VM_MAP_IS_EXOTIC(map) in vm_map_enter()
2885 (csm_address_space_exempt(map->pmap) != KERN_SUCCESS) && in vm_map_enter()
2887 (VM_MAP_POLICY_WX_FAIL(map) || in vm_map_enter()
2888 VM_MAP_POLICY_WX_STRIP_X(map)) && in vm_map_enter()
2890 boolean_t vm_protect_wx_fail = VM_MAP_POLICY_WX_FAIL(map); in vm_map_enter()
2959 if (map->map_disallow_new_exec == TRUE) { in vm_map_enter()
3023 if (VM_MAP_PAGE_SHIFT(submap) != VM_MAP_PAGE_SHIFT(map)) { in vm_map_enter()
3042 (offset & MIN(VM_MAP_PAGE_MASK(map), PAGE_MASK_64)) != 0) { in vm_map_enter()
3047 if (map->pmap == kernel_pmap) { in vm_map_enter()
3061 if (VM_MAP_PAGE_MASK(map) >= PAGE_MASK) { in vm_map_enter()
3066 if (VM_MAP_PAGE_MASK(map) >= PAGE_MASK && in vm_map_enter()
3067 !VM_MAP_PAGE_ALIGNED(size, VM_MAP_PAGE_MASK(map))) { in vm_map_enter()
3079 VM_MAP_PAGE_MASK(map) >= PAGE_MASK && in vm_map_enter()
3080 !VM_MAP_PAGE_ALIGNED(*address, VM_MAP_PAGE_MASK(map))) { in vm_map_enter()
3108 vm_map_lock(map); in vm_map_enter()
3112 result = vm_map_locate_space_anywhere(map, size, mask, vmk_flags, in vm_map_enter()
3117 result = vm_map_locate_space_fixed(map, start, size, mask, in vm_map_enter()
3123 assert(VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map))); in vm_map_enter()
3145 if (entry == vm_map_to_entry(map) || in vm_map_enter()
3255 if (map->pmap == kernel_pmap) { in vm_map_enter()
3277 } else if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) { in vm_map_enter()
3281 (entry != vm_map_to_entry(map)) && in vm_map_enter()
3333 map->size += (end - entry->vme_end); in vm_map_enter()
3336 VM_MAP_PAGE_MASK(map))); in vm_map_enter()
3338 …DTRACE_VM5(map_entry_extend, vm_map_t, map, vm_map_entry_t, entry, vm_address_t, entry->vme_start,… in vm_map_enter()
3341 if (map->holelistenabled) { in vm_map_enter()
3342 vm_map_store_update_first_free(map, entry, TRUE); in vm_map_enter()
3344 vm_map_store_update_first_free(map, map->first_free, TRUE); in vm_map_enter()
3390 vm_map_t, map, in vm_map_enter()
3396 new_entry = vm_map_entry_insert(map, in vm_map_enter()
3401 (entry_for_jit && !VM_MAP_POLICY_ALLOW_JIT_INHERIT(map) ? in vm_map_enter()
3445 vm_map_t, map, in vm_map_enter()
3450 map, in vm_map_enter()
3471 ledger_t ledger = map->pmap->ledger; in vm_map_enter()
3485 if (VM_MAP_PAGE_SHIFT(map) != VM_MAP_PAGE_SHIFT(submap)) { in vm_map_enter()
3486 …ERROR("map %p (%d) submap %p (%d): incompatible page sizes\n", map, VM_MAP_PAGE_SHIFT(map), submap… in vm_map_enter()
3489 kr = pmap_nest(map->pmap, in vm_map_enter()
3567 if (map->size_limit != RLIM_INFINITY && in vm_map_enter()
3568 map->size > map->size_limit) { in vm_map_enter()
3581 (uint64_t) map->size, in vm_map_enter()
3582 (uint64_t) map->size_limit); in vm_map_enter()
3584 vm_map_size_t, map->size, in vm_map_enter()
3585 uint64_t, map->size_limit); in vm_map_enter()
3587 } else if (map->data_limit != RLIM_INFINITY && in vm_map_enter()
3588 map->size > map->data_limit) { in vm_map_enter()
3601 (uint64_t) map->size, in vm_map_enter()
3602 (uint64_t) map->data_limit); in vm_map_enter()
3604 vm_map_size_t, map->size, in vm_map_enter()
3605 uint64_t, map->data_limit); in vm_map_enter()
3617 assert(pmap_is_empty(map->pmap, in vm_map_enter()
3690 vm_map_unlock(map); in vm_map_enter()
3704 if ((map->wiring_required) || (superpage_size)) { in vm_map_enter()
3707 kr = vm_map_wire_nested(map, start, end, in vm_map_enter()
3726 vm_map_lock(map); in vm_map_enter()
3734 (void) vm_map_delete(map, in vm_map_enter()
3748 vm_map_lock(map); in vm_map_enter()
3756 if (vm_map_lookup_entry(map, start, &entry1) || in vm_map_enter()
3757 vm_map_lookup_entry(map, end, &entry2) || in vm_map_enter()
3776 vm_map_store_entry_link(map, entry1, entry2, in vm_map_enter()
3778 map->size += entry_size; in vm_map_enter()
3781 if (map->wiring_required) { in vm_map_enter()
3797 vm_map_unlock(map); in vm_map_enter()
4126 submap = named_entry->backing.map; in vm_map_enter_mem_object()
5145 vm_map_t map, in vm_map_clip_unnest() argument
5165 if (pmap_adjust_unnest_parameters(map->pmap, &start_unnest, &end_unnest)) { in vm_map_clip_unnest()
5168 log_unnest_badness(map, in vm_map_clip_unnest()
5186 _vm_map_clip_start(&map->hdr, in vm_map_clip_unnest()
5189 if (map->holelistenabled) { in vm_map_clip_unnest()
5190 vm_map_store_update_first_free(map, NULL, FALSE); in vm_map_clip_unnest()
5192 vm_map_store_update_first_free(map, map->first_free, FALSE); in vm_map_clip_unnest()
5196 _vm_map_clip_end(&map->hdr, in vm_map_clip_unnest()
5199 if (map->holelistenabled) { in vm_map_clip_unnest()
5200 vm_map_store_update_first_free(map, NULL, FALSE); in vm_map_clip_unnest()
5202 vm_map_store_update_first_free(map, map->first_free, FALSE); in vm_map_clip_unnest()
5206 pmap_unnest(map->pmap, in vm_map_clip_unnest()
5209 if ((map->mapped_in_other_pmaps) && os_ref_get_count_raw(&map->map_refcnt) != 0) { in vm_map_clip_unnest()
5212 map, entry->vme_start, in vm_map_clip_unnest()
5218 if ((map->pmap != kernel_pmap) && in vm_map_clip_unnest()
5228 vm_map_t map, in __vm_map_clip_atomic_entry_panic() argument
5233 "%p [0x%llx:0x%llx] at 0x%llx", map, entry, in __vm_map_clip_atomic_entry_panic()
5248 vm_map_t map, in vm_map_clip_start() argument
5265 start_unnest = startaddr & ~(pmap_shared_region_size_min(map->pmap) - 1); in vm_map_clip_start()
5266 end_unnest = start_unnest + pmap_shared_region_size_min(map->pmap); in vm_map_clip_start()
5267 vm_map_clip_unnest(map, entry, start_unnest, end_unnest); in vm_map_clip_start()
5274 pmap_remove(map->pmap, in vm_map_clip_start()
5279 __vm_map_clip_atomic_entry_panic(map, entry, startaddr); in vm_map_clip_start()
5284 vm_map_t, map, in vm_map_clip_start()
5290 _vm_map_clip_start(&map->hdr, entry, startaddr); in vm_map_clip_start()
5291 if (map->holelistenabled) { in vm_map_clip_start()
5292 vm_map_store_update_first_free(map, NULL, FALSE); in vm_map_clip_start()
5294 vm_map_store_update_first_free(map, map->first_free, FALSE); in vm_map_clip_start()
5368 vm_map_t map, in vm_map_clip_end() argument
5392 (endaddr + pmap_shared_region_size_min(map->pmap) - 1) & in vm_map_clip_end()
5393 ~(pmap_shared_region_size_min(map->pmap) - 1); in vm_map_clip_end()
5394 vm_map_clip_unnest(map, entry, start_unnest, end_unnest); in vm_map_clip_end()
5401 pmap_remove(map->pmap, in vm_map_clip_end()
5406 __vm_map_clip_atomic_entry_panic(map, entry, endaddr); in vm_map_clip_end()
5410 vm_map_t, map, in vm_map_clip_end()
5416 _vm_map_clip_end(&map->hdr, entry, endaddr); in vm_map_clip_end()
5417 if (map->holelistenabled) { in vm_map_clip_end()
5418 vm_map_store_update_first_free(map, NULL, FALSE); in vm_map_clip_end()
5420 vm_map_store_update_first_free(map, map->first_free, FALSE); in vm_map_clip_end()
5488 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
5490 if (start < vm_map_min(map)) \
5491 start = vm_map_min(map); \
5492 if (end > vm_map_max(map)) \
5493 end = vm_map_max(map); \
5514 vm_map_t map, in vm_map_range_check() argument
5525 if (start < vm_map_min(map) || end > vm_map_max(map) || start > end) { in vm_map_range_check()
5533 if (!vm_map_lookup_entry(map, start, &cur)) { in vm_map_range_check()
5554 while ((cur != vm_map_to_entry(map)) && (prev == cur->vme_start)) { in vm_map_range_check()
5567 vm_map_t map, in vm_map_protect_sanitize() argument
5579 map, VM_PROT_COPY, new_prot); in vm_map_protect_sanitize()
5585 map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, start, end, &size); in vm_map_protect_sanitize()
5603 vm_map_t map, in vm_map_protect() argument
5619 kr = vm_map_protect_sanitize(map, in vm_map_protect()
5637 if (start >= map->max_offset) { in vm_map_protect()
5642 map->pmap != kernel_pmap && in vm_map_protect()
5643 (vm_map_cs_enforcement(map) in vm_map_protect()
5645 || !VM_MAP_IS_EXOTIC(map) in vm_map_protect()
5648 VM_MAP_POLICY_WX_FAIL(map)) { in vm_map_protect()
5685 kflags.vmkf_tpro_enforcement_override = !vm_map_tpro_enforcement(map); in vm_map_protect()
5687 kr = vm_map_remap(map, in vm_map_protect()
5692 map, in vm_map_protect()
5704 vm_map_lock(map); in vm_map_protect()
5711 if (start >= map->max_offset) { in vm_map_protect()
5712 vm_map_unlock(map); in vm_map_protect()
5721 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_protect()
5722 vm_map_unlock(map); in vm_map_protect()
5743 while ((current != vm_map_to_entry(map)) && in vm_map_protect()
5749 vm_map_unlock(map); in vm_map_protect()
5757 if (set_max && (new_prot & VM_PROT_ALLEXEC) && !vm_map_cs_enforcement(map)) { in vm_map_protect()
5761 …if (set_max && (new_prot & VM_PROT_EXECUTE) && (csm_address_space_exempt(map->pmap) == KERN_SUCCES… in vm_map_protect()
5766 vm_map_unlock(map); in vm_map_protect()
5771 pmap_has_prot_policy(map->pmap, current->translated_allow_execute, current->protection)) { in vm_map_protect()
5772 vm_map_unlock(map); in vm_map_protect()
5779 vm_map_unlock(map); in vm_map_protect()
5788 map->pmap != kernel_pmap && in vm_map_protect()
5789 (vm_map_cs_enforcement(map) in vm_map_protect()
5791 || !VM_MAP_IS_EXOTIC(map) in vm_map_protect()
5796 (csm_address_space_exempt(map->pmap) != KERN_SUCCESS) && in vm_map_protect()
5818 if (VM_MAP_POLICY_WX_FAIL(map)) { in vm_map_protect()
5819 vm_map_unlock(map); in vm_map_protect()
5830 if (map->map_disallow_new_exec == TRUE) { in vm_map_protect()
5833 vm_map_unlock(map); in vm_map_protect()
5844 end == vm_map_round_page(prev, VM_MAP_PAGE_MASK(map))) { in vm_map_protect()
5848 if (prev_entry != vm_map_to_entry(map) && in vm_map_protect()
5851 VM_MAP_PAGE_MASK(map)) in vm_map_protect()
5865 vm_map_unlock(map); in vm_map_protect()
5876 if (current != vm_map_to_entry(map)) { in vm_map_protect()
5878 vm_map_clip_start(map, current, start); in vm_map_protect()
5881 while ((current != vm_map_to_entry(map)) && in vm_map_protect()
5896 wait_result = vm_map_entry_wait(map, TH_UNINT); in vm_map_protect()
5908 vm_map_lookup_entry_or_next(map, current_start, ¤t); in vm_map_protect()
5909 if (current != vm_map_to_entry(map)) { in vm_map_protect()
5910 vm_map_clip_start(map, current, current_start); in vm_map_protect()
5917 vm_map_clip_end(map, current, end); in vm_map_protect()
5930 map, current, in vm_map_protect()
5976 kr = vm_map_entry_cs_associate(map, current, vmk_flags); in vm_map_protect()
5983 map, current, in vm_map_protect()
6027 if (override_nx(map, VME_ALIAS(current)) && prot) { in vm_map_protect()
6036 …x%llx,0x%llx) old=0x%x new=0x%x - <rdar://23770418> code-signing bypass?", map, (uint64_t)current-… in vm_map_protect()
6040 if (pmap_has_prot_policy(map->pmap, current->translated_allow_execute, prot)) { in vm_map_protect()
6043 map, (uint64_t)current->vme_start, (uint64_t)current->vme_end, prot, current->wired_count); in vm_map_protect()
6068 pmap_protect_options(map->pmap, in vm_map_protect()
6084 vm_map_lookup_entry_or_next(map, original_start, &entry); in vm_map_protect()
6087 while ((current != vm_map_to_entry(map)) && in vm_map_protect()
6089 vm_map_simplify_entry(map, current); in vm_map_protect()
6093 vm_map_unlock(map); in vm_map_protect()
6100 vm_map_t map, in vm_map_inherit_sanitize() argument
6118 map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, start, end, &size); in vm_map_inherit_sanitize()
6136 vm_map_t map, in vm_map_inherit() argument
6148 kr = vm_map_inherit_sanitize(map, in vm_map_inherit()
6159 vm_map_lock(map); in vm_map_inherit()
6161 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
6163 if (vm_map_lookup_entry(map, start, &temp_entry)) { in vm_map_inherit()
6172 while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { in vm_map_inherit()
6175 vm_map_unlock(map); in vm_map_inherit()
6184 if (entry != vm_map_to_entry(map)) { in vm_map_inherit()
6186 vm_map_clip_start(map, entry, start); in vm_map_inherit()
6189 while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { in vm_map_inherit()
6190 vm_map_clip_end(map, entry, end); in vm_map_inherit()
6201 vm_map_unlock(map); in vm_map_inherit()
6212 vm_map_t map, in add_wire_counts() argument
6239 if (size + map->user_wire_size > MIN(map->user_wire_limit, vm_per_task_user_wire_limit) || in add_wire_counts()
6252 …limit. %llu bytes wired and requested to wire %llu bytes more", (uint64_t) map->user_wire_size, (u… in add_wire_counts()
6269 map->user_wire_size += size; in add_wire_counts()
6302 vm_map_t map, in subtract_wire_counts() argument
6319 map->user_wire_size -= entry->vme_end - entry->vme_start; in subtract_wire_counts()
6340 vm_map_t map, in vm_map_wire_nested() argument
6368 if (map->pmap == kernel_pmap || in vm_map_wire_nested()
6369 !vm_map_cs_enforcement(map)) { in vm_map_wire_nested()
6374 if (csm_address_space_exempt(map->pmap) == KERN_SUCCESS) { in vm_map_wire_nested()
6391 assert(end - start == VM_MAP_PAGE_SIZE(map)); in vm_map_wire_nested()
6396 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire_nested()
6397 assert(VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map))); in vm_map_wire_nested()
6398 assert(VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map))); in vm_map_wire_nested()
6404 vm_map_lock(map); in vm_map_wire_nested()
6408 last_timestamp = map->timestamp; in vm_map_wire_nested()
6416 if (vm_map_lookup_entry(map, s, &first_entry)) { in vm_map_wire_nested()
6428 while ((entry != vm_map_to_entry(map)) && (s < end)) { in vm_map_wire_nested()
6466 vm_map_entry_wakeup(map); in vm_map_wire_nested()
6472 wait_result = vm_map_entry_wait(map, in vm_map_wire_nested()
6489 last_timestamp = map->timestamp; in vm_map_wire_nested()
6495 if (!vm_map_lookup_entry(map, s, &first_entry)) { in vm_map_wire_nested()
6526 vm_map_clip_start(map, entry, s); in vm_map_wire_nested()
6527 vm_map_clip_end(map, entry, end); in vm_map_wire_nested()
6553 pmap = map->pmap; in vm_map_wire_nested()
6558 if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) { in vm_map_wire_nested()
6576 lookup_map = map; in vm_map_wire_nested()
6577 vm_map_lock_write_to_read(map); in vm_map_wire_nested()
6589 vm_map_unwire_nested(map, start, in vm_map_wire_nested()
6598 vm_map_lock(map); in vm_map_wire_nested()
6601 if (!vm_map_lookup_entry(map, in vm_map_wire_nested()
6614 vm_map_clip_start(map, entry, s); in vm_map_wire_nested()
6615 vm_map_clip_end(map, entry, end); in vm_map_wire_nested()
6624 last_timestamp = map->timestamp; in vm_map_wire_nested()
6632 if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) { in vm_map_wire_nested()
6638 vm_map_unlock(map); in vm_map_wire_nested()
6644 vm_map_lock(map); in vm_map_wire_nested()
6650 if (!vm_map_lookup_entry(map, local_start, in vm_map_wire_nested()
6663 last_timestamp = map->timestamp; in vm_map_wire_nested()
6664 while ((entry != vm_map_to_entry(map)) && in vm_map_wire_nested()
6673 subtract_wire_counts(map, entry, user_wire); in vm_map_wire_nested()
6712 vm_map_clip_start(map, entry, s); in vm_map_wire_nested()
6713 vm_map_clip_end(map, entry, end); in vm_map_wire_nested()
6715 if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) { in vm_map_wire_nested()
6789 map->pmap != kernel_pmap && in vm_map_wire_nested()
6790 (vm_map_cs_enforcement(map) in vm_map_wire_nested()
6792 || !VM_MAP_IS_EXOTIC(map) in vm_map_wire_nested()
6798 (csm_address_space_exempt(map->pmap) != KERN_SUCCESS) in vm_map_wire_nested()
6843 vm_map_always_shadow(map)); in vm_map_wire_nested()
6871 vm_map_clip_start(map, entry, s); in vm_map_wire_nested()
6872 vm_map_clip_end(map, entry, end); in vm_map_wire_nested()
6880 VME_OBJECT_SHADOW(entry, size, vm_map_always_shadow(map)); in vm_map_wire_nested()
6899 map->pmap, in vm_map_wire_nested()
6900 VM_MAP_PAGE_SIZE(map), in vm_map_wire_nested()
6939 vm_map_clip_start(map, entry, s); in vm_map_wire_nested()
6940 vm_map_clip_end(map, entry, end); in vm_map_wire_nested()
6956 ((entry->vme_next == vm_map_to_entry(map)) || in vm_map_wire_nested()
6970 if ((rc = add_wire_counts(map, entry, user_wire)) != KERN_SUCCESS) { in vm_map_wire_nested()
6992 vm_map_unlock(map); in vm_map_wire_nested()
7001 rc = vm_fault_wire(map, in vm_map_wire_nested()
7005 rc = vm_fault_wire(map, in vm_map_wire_nested()
7006 &tmp_entry, caller_prot, tag, map->pmap, in vm_map_wire_nested()
7015 vm_map_lock(map); in vm_map_wire_nested()
7017 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire_nested()
7022 if (!vm_map_lookup_entry(map, tmp_entry.vme_start, in vm_map_wire_nested()
7030 last_timestamp = map->timestamp; in vm_map_wire_nested()
7032 while ((entry != vm_map_to_entry(map)) && in vm_map_wire_nested()
7041 subtract_wire_counts(map, entry, user_wire); in vm_map_wire_nested()
7050 if ((entry != vm_map_to_entry(map)) && /* we still have entries in the map */ in vm_map_wire_nested()
7065 vm_map_simplify_range(map, start, end); in vm_map_wire_nested()
7068 vm_map_unlock(map); in vm_map_wire_nested()
7074 vm_map_entry_wakeup(map); in vm_map_wire_nested()
7079 vm_map_unwire_nested(map, start, s, user_wire, in vm_map_wire_nested()
7092 vm_map_t map, in vm_map_wire_sanitize() argument
7104 kr = vm_sanitize_addr_end(start_u, end_u, vm_sanitize_caller, map, in vm_map_wire_sanitize()
7111 kr = vm_sanitize_prot(prot_u, vm_sanitize_caller, map, prot); in vm_map_wire_sanitize()
7124 vm_map_t map, in vm_map_wire_impl() argument
7141 kr = vm_map_wire_sanitize(map, in vm_map_wire_impl()
7157 return vm_map_wire_nested(map, start, end, prot, tag, user_wire, in vm_map_wire_impl()
7163 vm_map_t map, in vm_map_wire_external() argument
7171 return vm_map_wire_kernel(map, start_u, end_u, prot_u, tag, user_wire); in vm_map_wire_external()
7176 vm_map_t map, in vm_map_wire_kernel() argument
7183 return vm_map_wire_impl(map, start_u, end_u, prot_u, tag, in vm_map_wire_kernel()
7191 vm_map_t map, in vm_map_wire_and_extract() argument
7198 vm_map_size_ut size_u = vm_sanitize_wrap_size(VM_MAP_PAGE_SIZE(map)); in vm_map_wire_and_extract()
7201 return vm_map_wire_impl(map, start_u, end_u, prot_u, tag, in vm_map_wire_and_extract()
7209 vm_map_t map, in vm_map_unwire_nested() argument
7222 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire_nested()
7223 assert(VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map))); in vm_map_unwire_nested()
7224 assert(VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map))); in vm_map_unwire_nested()
7231 vm_map_lock(map); in vm_map_unwire_nested()
7235 last_timestamp = map->timestamp; in vm_map_unwire_nested()
7237 if (vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire_nested()
7248 vm_map_unlock(map); in vm_map_unwire_nested()
7254 vm_map_unlock(map); in vm_map_unwire_nested()
7259 while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { in vm_map_unwire_nested()
7309 vm_map_clip_start(map, entry, start); in vm_map_unwire_nested()
7310 vm_map_clip_end(map, entry, end); in vm_map_unwire_nested()
7321 pmap = map->pmap; in vm_map_unwire_nested()
7339 ((entry->vme_next == vm_map_to_entry(map)) || in vm_map_unwire_nested()
7351 subtract_wire_counts(map, entry, user_wire); in vm_map_unwire_nested()
7365 vm_map_unlock(map); in vm_map_unwire_nested()
7368 vm_map_lock(map); in vm_map_unwire_nested()
7370 if (last_timestamp + 1 != map->timestamp) { in vm_map_unwire_nested()
7375 if (!vm_map_lookup_entry(map, in vm_map_unwire_nested()
7386 last_timestamp = map->timestamp; in vm_map_unwire_nested()
7393 while ((entry != vm_map_to_entry(map)) && in vm_map_unwire_nested()
7406 vm_map_unlock(map); in vm_map_unwire_nested()
7410 vm_map_lock(map); in vm_map_unwire_nested()
7412 if (last_timestamp + 1 != map->timestamp) { in vm_map_unwire_nested()
7417 if (!vm_map_lookup_entry(map, in vm_map_unwire_nested()
7428 last_timestamp = map->timestamp; in vm_map_unwire_nested()
7446 vm_map_clip_start(map, entry, start); in vm_map_unwire_nested()
7447 vm_map_clip_end(map, entry, end); in vm_map_unwire_nested()
7455 ((entry->vme_next == vm_map_to_entry(map)) || in vm_map_unwire_nested()
7464 subtract_wire_counts(map, entry, user_wire); in vm_map_unwire_nested()
7482 vm_map_unlock(map); in vm_map_unwire_nested()
7484 vm_fault_unwire(map, &tmp_entry, FALSE, map_pmap, in vm_map_unwire_nested()
7487 vm_fault_unwire(map, &tmp_entry, FALSE, map->pmap, in vm_map_unwire_nested()
7490 vm_map_lock(map); in vm_map_unwire_nested()
7492 if (last_timestamp + 1 != map->timestamp) { in vm_map_unwire_nested()
7497 if (!vm_map_lookup_entry(map, tmp_entry.vme_start, in vm_map_unwire_nested()
7507 last_timestamp = map->timestamp; in vm_map_unwire_nested()
7514 while ((entry != vm_map_to_entry(map)) && in vm_map_unwire_nested()
7534 vm_map_simplify_range(map, start, end); in vm_map_unwire_nested()
7536 vm_map_unlock(map); in vm_map_unwire_nested()
7541 vm_map_entry_wakeup(map); in vm_map_unwire_nested()
7548 vm_map_t map, in vm_map_unwire() argument
7553 return vm_map_unwire_impl(map, start_u, end_u, user_wire, in vm_map_unwire()
7560 vm_map_t map, in vm_map_unwire_sanitize() argument
7568 return vm_sanitize_addr_end(start_u, end_u, vm_sanitize_caller, map, in vm_map_unwire_sanitize()
7575 vm_map_t map, in vm_map_unwire_impl() argument
7589 map, in vm_map_unwire_impl()
7600 return vm_map_unwire_nested(map, start, end, in vm_map_unwire_impl()
7613 vm_map_t map, in vm_map_entry_zap() argument
7623 if (VM_MAP_PAGE_MASK(map) >= PAGE_MASK) { in vm_map_entry_zap()
7628 assert(VM_MAP_PAGE_ALIGNED(s, VM_MAP_PAGE_MASK(map))); in vm_map_entry_zap()
7629 assert(VM_MAP_PAGE_ALIGNED(e, VM_MAP_PAGE_MASK(map))); in vm_map_entry_zap()
7635 vm_map_store_entry_unlink(map, entry, false); in vm_map_entry_zap()
7636 map->size -= e - s; in vm_map_entry_zap()
7643 vm_map_t map, in vm_map_submap_pmap_clean() argument
7677 if (map->mapped_in_other_pmaps && in vm_map_submap_pmap_clean()
7678 os_ref_get_count_raw(&map->map_refcnt) != 0 && in vm_map_submap_pmap_clean()
7692 pmap_remove(map->pmap, in vm_map_submap_pmap_clean()
7715 if (map->mapped_in_other_pmaps && in vm_map_submap_pmap_clean()
7716 os_ref_get_count_raw(&map->map_refcnt) != 0 && in vm_map_submap_pmap_clean()
7728 pmap_remove(map->pmap, in vm_map_submap_pmap_clean()
7898 vm_map_t map, in __vm_map_delete_misaligned_panic() argument
7903 map, (uint64_t)start, (uint64_t)end, VM_MAP_PAGE_SIZE(map)); in __vm_map_delete_misaligned_panic()
7909 vm_map_t map, in __vm_map_delete_failed_panic() argument
7915 map, (uint64_t)start, (uint64_t)end, kr); in __vm_map_delete_failed_panic()
7921 vm_map_t map, in __vm_map_delete_gap_panic() argument
7927 map, (uint64_t)start, (uint64_t)end, (uint64_t)where); in __vm_map_delete_gap_panic()
7933 vm_map_t map, in __vm_map_delete_permanent_panic() argument
7940 map, (uint64_t)start, (uint64_t)end, entry, in __vm_map_delete_permanent_panic()
7964 vm_map_clamp_to_pmap(vm_map_t map, vm_map_address_t *start, vm_map_address_t *end) in vm_map_clamp_to_pmap() argument
7971 if (map->pmap == kernel_pmap) { in vm_map_clamp_to_pmap()
7979 min = map->pmap->min; in vm_map_clamp_to_pmap()
7980 max = map->pmap->max; in vm_map_clamp_to_pmap()
8016 vm_map_t map, in vm_map_delete() argument
8036 if (vm_map_pmap(map) == kernel_pmap) { in vm_map_delete()
8046 if (map->terminated || os_ref_get_count_raw(&map->map_refcnt) == 0) { in vm_map_delete()
8050 if (map->corpse_source && in vm_map_delete()
8052 !map->terminated) { in vm_map_delete()
8066 (start & VM_MAP_PAGE_MASK(map))) { in vm_map_delete()
8067 __vm_map_delete_misaligned_panic(map, start, end); in vm_map_delete()
8086 if (end == 0 || end > vm_map_max(map)) { in vm_map_delete()
8088 gap_start = vm_map_max(map); in vm_map_delete()
8090 __vm_map_delete_gap_panic(map, in vm_map_delete()
8098 __vm_map_delete_gap_panic(map, in vm_map_delete()
8099 vm_map_max(map), start, end); in vm_map_delete()
8105 if (start < vm_map_min(map)) { in vm_map_delete()
8109 __vm_map_delete_gap_panic(map, in vm_map_delete()
8136 while (vm_map_lookup_entry_or_next(map, start, &entry)) { in vm_map_delete()
8140 SAVE_HINT_MAP_WRITE(map, entry->vme_prev); in vm_map_delete()
8182 if (vm_map_lookup_entry_or_next(map, s, &entry)) { in vm_map_delete()
8183 SAVE_HINT_MAP_WRITE(map, entry->vme_prev); in vm_map_delete()
8193 it != vm_map_to_entry(map) && in vm_map_delete()
8213 if (entry == vm_map_to_entry(map) || s < entry->vme_start) { in vm_map_delete()
8221 __vm_map_delete_gap_panic(map, s, start, end); in vm_map_delete()
8227 if (entry == vm_map_to_entry(map) || in vm_map_delete()
8242 __vm_map_delete_permanent_panic(map, start, end, entry); in vm_map_delete()
8260 kmem_entry_validate_guard(map, entry, in vm_map_delete()
8287 } else if (map->terminated) { in vm_map_delete()
8352 !VM_MAP_PAGE_ALIGNED(s, VM_MAP_PAGE_MASK(map))) { in vm_map_delete()
8359 vm_map_clip_start(map, entry, s); in vm_map_delete()
8360 SAVE_HINT_MAP_WRITE(map, entry->vme_prev); in vm_map_delete()
8366 !VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map))) { in vm_map_delete()
8373 vm_map_clip_end(map, entry, end); in vm_map_delete()
8419 vm_map_entry_wakeup(map); in vm_map_delete()
8423 wait_result = vm_map_entry_wait(map, interruptible); in vm_map_delete()
8467 subtract_wire_counts(map, entry, user_wire); in vm_map_delete()
8484 assert(map != kernel_map); in vm_map_delete()
8495 wait_result = vm_map_entry_wait(map, in vm_map_delete()
8531 last_timestamp = map->timestamp; in vm_map_delete()
8534 vm_map_unlock(map); in vm_map_delete()
8551 pmap = map->pmap; in vm_map_delete()
8563 max_end = end - VM_MAP_PAGE_SIZE(map); in vm_map_delete()
8571 map->pmap, in vm_map_delete()
8578 vm_fault_unwire(map, &tmp_entry, in vm_map_delete()
8579 tmp_entry.vme_kernel_object, map->pmap, in vm_map_delete()
8583 vm_map_lock(map); in vm_map_delete()
8589 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
8628 } else if ((flags & VM_MAP_REMOVE_IMMUTABLE) || map->terminated) { in vm_map_delete()
8672 assertf(VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)) >= VM_MAP_PAGE_SHIFT(map), in vm_map_delete()
8674 map, VM_MAP_PAGE_SHIFT(map), entry, in vm_map_delete()
8681 if (map->terminated) { in vm_map_delete()
8703 map->pmap, in vm_map_delete()
8708 if (map->mapped_in_other_pmaps && in vm_map_delete()
8709 os_ref_get_count_raw(&map->map_refcnt) != 0) { in vm_map_delete()
8712 map, entry->vme_start, in vm_map_delete()
8719 map, entry->vme_start, entry->vme_end, in vm_map_delete()
8728 } else if (map->mapped_in_other_pmaps && in vm_map_delete()
8729 os_ref_get_count_raw(&map->map_refcnt) != 0) { in vm_map_delete()
8759 vm_map_clamp_to_pmap(map, &remove_start, &remove_end); in vm_map_delete()
8762 pmap_remove(map->pmap, remove_start, remove_end); in vm_map_delete()
8770 assert(pmap_is_empty(map->pmap, in vm_map_delete()
8778 vm_map_t, map, in vm_map_delete()
8782 vm_map_iokit_unmapped_region(map, in vm_map_delete()
8800 rounded_s = VM_MAP_ROUND_PAGE(s, VM_MAP_PAGE_MASK(map)); in vm_map_delete()
8801 if (next == vm_map_to_entry(map)) { in vm_map_delete()
8821 map, in vm_map_delete()
8842 vm_map_entry_zap(map, entry, zap_list); in vm_map_delete()
8849 unsigned int last_timestamp = map->timestamp++; in vm_map_delete()
8851 if (lck_rw_lock_yield_exclusive(&map->lock, in vm_map_delete()
8853 if (last_timestamp != map->timestamp + 1) { in vm_map_delete()
8858 map->timestamp--; in vm_map_delete()
8863 if (map->wait_for_space) { in vm_map_delete()
8864 thread_wakeup((event_t) map); in vm_map_delete()
8868 vm_map_entry_wakeup(map); in vm_map_delete()
8873 __vm_map_delete_failed_panic(map, start, end, ret.kmr_return); in vm_map_delete()
8897 vm_map_t map, in vm_map_remove_and_unlock() argument
8906 ret = vm_map_delete(map, start, end, flags, guard, &zap); in vm_map_remove_and_unlock()
8907 vm_map_unlock(map); in vm_map_remove_and_unlock()
8922 vm_map_t map, in vm_map_remove_guard() argument
8928 vm_map_lock(map); in vm_map_remove_guard()
8929 return vm_map_remove_and_unlock(map, start, end, flags, guard); in vm_map_remove_guard()
8939 vm_map_t map) in vm_map_terminate() argument
8941 vm_map_lock(map); in vm_map_terminate()
8942 map->terminated = TRUE; in vm_map_terminate()
8943 vm_map_disable_hole_optimization(map); in vm_map_terminate()
8944 (void)vm_map_remove_and_unlock(map, map->min_offset, map->max_offset, in vm_map_terminate()
9985 vm_map_t map, in vm_map_copy_addr_size_sanitize() argument
9998 vm_sanitize_caller, map, in vm_map_copy_addr_size_sanitize()
11152 vm_map_t map, in vm_map_copyout_kernel_buffer() argument
11178 if (map == kernel_map) { in vm_map_copyout_kernel_buffer()
11183 kr = vm_map_enter(map, in vm_map_copyout_kernel_buffer()
11186 VM_MAP_PAGE_MASK(map)), in vm_map_copyout_kernel_buffer()
11199 if (map->pmap == kernel_pmap) { in vm_map_copyout_kernel_buffer()
11208 if (thread->map == map) { in vm_map_copyout_kernel_buffer()
11225 vm_map_reference(map); in vm_map_copyout_kernel_buffer()
11226 oldmap = vm_map_switch(map); in vm_map_copyout_kernel_buffer()
11235 vm_map_deallocate(map); in vm_map_copyout_kernel_buffer()
11244 (void) vm_map_remove(map, in vm_map_copyout_kernel_buffer()
11246 VM_MAP_PAGE_MASK(map)), in vm_map_copyout_kernel_buffer()
11249 VM_MAP_PAGE_MASK(map))), in vm_map_copyout_kernel_buffer()
11250 VM_MAP_PAGE_MASK(map))); in vm_map_copyout_kernel_buffer()
11279 vm_map_t map, in vm_map_copy_insert() argument
11288 vm_map_store_entry_link(map, after_where, entry, in vm_map_copy_insert()
11302 vm_map_t map, in vm_map_copy_remap() argument
11316 new_entry = vm_map_entry_create(map); in vm_map_copy_remap()
11318 vm_map_entry_copy(map, new_entry, copy_entry); in vm_map_copy_remap()
11335 vm_map_store_entry_link(map, where, new_entry, in vm_map_copy_remap()
13528 owner->map == old_map) { in vm_map_fork()
13884 vm_map_t map = *var_map; in vm_map_lookup_and_lock_object() local
13912 *real_map = map; in vm_map_lookup_and_lock_object()
13914 fault_page_mask = MIN(VM_MAP_PAGE_MASK(map), PAGE_MASK); in vm_map_lookup_and_lock_object()
13924 entry = map->hint; in vm_map_lookup_and_lock_object()
13926 if ((entry == vm_map_to_entry(map)) || in vm_map_lookup_and_lock_object()
13934 if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) { in vm_map_lookup_and_lock_object()
13935 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
13938 if ((*real_map != map) in vm_map_lookup_and_lock_object()
13947 if (map == old_map) { in vm_map_lookup_and_lock_object()
13979 assertf(VM_MAP_PAGE_SHIFT(VME_SUBMAP(entry)) >= VM_MAP_PAGE_SHIFT(map), in vm_map_lookup_and_lock_object()
13981 map, VM_MAP_PAGE_SHIFT(map), entry, in vm_map_lookup_and_lock_object()
13992 if ((*real_map != map) && in vm_map_lookup_and_lock_object()
14003 if (vm_map_lock_read_to_write(map)) { in vm_map_lookup_and_lock_object()
14004 vm_map_lock_read(map); in vm_map_lookup_and_lock_object()
14005 *real_map = map; in vm_map_lookup_and_lock_object()
14010 cow_sub_map_parent = map; in vm_map_lookup_and_lock_object()
14021 if ((cow_sub_map_parent != map) && in vm_map_lookup_and_lock_object()
14022 (*real_map != map)) { in vm_map_lookup_and_lock_object()
14023 vm_map_unlock(map); in vm_map_lookup_and_lock_object()
14037 if ((*real_map != map) && (map != cow_sub_map_parent)) { in vm_map_lookup_and_lock_object()
14038 vm_map_unlock_read(map); in vm_map_lookup_and_lock_object()
14043 map = *var_map; in vm_map_lookup_and_lock_object()
14052 if (!vm_map_lookup_entry(map, local_vaddr, &submap_entry)) { in vm_map_lookup_and_lock_object()
14053 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14056 if ((*real_map != map) in vm_map_lookup_and_lock_object()
14060 *real_map = map; in vm_map_lookup_and_lock_object()
14105 if (vm_map_lock_read_to_write(map)) { in vm_map_lookup_and_lock_object()
14106 vm_map_lock_read(map); in vm_map_lookup_and_lock_object()
14129 vm_map_clip_start(map, submap_entry, local_start); in vm_map_lookup_and_lock_object()
14130 vm_map_clip_end(map, submap_entry, local_end); in vm_map_lookup_and_lock_object()
14148 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14151 if ((*real_map != map) in vm_map_lookup_and_lock_object()
14155 *real_map = map; in vm_map_lookup_and_lock_object()
14157 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14160 vm_map_t, map, in vm_map_lookup_and_lock_object()
14170 assertf(VM_MAP_PAGE_ALIGNED(VME_OFFSET(submap_entry), VM_MAP_PAGE_MASK(map)), in vm_map_lookup_and_lock_object()
14177 vm_map_t, map, in vm_map_lookup_and_lock_object()
14183 version.main_timestamp = map->timestamp; in vm_map_lookup_and_lock_object()
14184 vm_map_unlock(map); /* Increments timestamp by 1 */ in vm_map_lookup_and_lock_object()
14200 vm_map_lock(map); in vm_map_lookup_and_lock_object()
14204 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14207 if ((*real_map != map) in vm_map_lookup_and_lock_object()
14211 *real_map = map; in vm_map_lookup_and_lock_object()
14215 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14226 (version.main_timestamp + 1) == map->timestamp) { in vm_map_lookup_and_lock_object()
14234 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14251 vm_map_t, map, in vm_map_lookup_and_lock_object()
14268 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14273 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14276 if ((*real_map != map) in vm_map_lookup_and_lock_object()
14280 *real_map = map; in vm_map_lookup_and_lock_object()
14284 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14314 if (pmap_has_prot_policy(map->pmap, submap_entry->translated_allow_execute, prot)) { in vm_map_lookup_and_lock_object()
14317 map, map->pmap, submap_entry, in vm_map_lookup_and_lock_object()
14323 if (pmap_has_prot_policy(map->pmap, submap_entry->translated_allow_execute, prot)) { in vm_map_lookup_and_lock_object()
14326 map, map->pmap, submap_entry, in vm_map_lookup_and_lock_object()
14344 || map->mapped_in_other_pmaps) ? in vm_map_lookup_and_lock_object()
14345 PMAP_NULL : map->pmap, in vm_map_lookup_and_lock_object()
14346 VM_MAP_PAGE_SIZE(map), in vm_map_lookup_and_lock_object()
14378 vm_map_unlock(map); in vm_map_lookup_and_lock_object()
14383 map = cow_sub_map_parent; in vm_map_lookup_and_lock_object()
14388 if (!vm_map_lookup_entry(map, in vm_map_lookup_and_lock_object()
14390 if ((cow_sub_map_parent) && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14393 if ((*real_map != map) in vm_map_lookup_and_lock_object()
14397 *real_map = map; in vm_map_lookup_and_lock_object()
14401 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14419 local_start = vaddr & ~(pmap_shared_region_size_min(map->pmap) - 1); in vm_map_lookup_and_lock_object()
14420 local_end = local_start + pmap_shared_region_size_min(map->pmap); in vm_map_lookup_and_lock_object()
14437 vm_map_clip_start(map, entry, local_start); in vm_map_lookup_and_lock_object()
14438 vm_map_clip_end(map, entry, local_end); in vm_map_lookup_and_lock_object()
14474 if (vm_map_tpro(map)) { in vm_map_lookup_and_lock_object()
14485 map->pmap != kernel_pmap && in vm_map_lookup_and_lock_object()
14486 (vm_map_cs_enforcement(map) in vm_map_lookup_and_lock_object()
14488 || !VM_MAP_IS_EXOTIC(map) in vm_map_lookup_and_lock_object()
14493 (csm_address_space_exempt(map->pmap) != KERN_SUCCESS) && in vm_map_lookup_and_lock_object()
14496 VM_MAP_POLICY_WX_STRIP_X(map)) { in vm_map_lookup_and_lock_object()
14528 if (map != old_map) { in vm_map_lookup_and_lock_object()
14536 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14540 && (cow_sub_map_parent != map)) { in vm_map_lookup_and_lock_object()
14588 if (*real_map != map) { in vm_map_lookup_and_lock_object()
14591 *real_map = map; in vm_map_lookup_and_lock_object()
14638 if (vm_map_lock_read_to_write(map)) { in vm_map_lookup_and_lock_object()
14639 vm_map_lock_read(map); in vm_map_lookup_and_lock_object()
14651 vm_map_always_shadow(map)); in vm_map_lookup_and_lock_object()
14654 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14683 if (vm_map_lock_read_to_write(map)) { in vm_map_lookup_and_lock_object()
14684 vm_map_lock_read(map); in vm_map_lookup_and_lock_object()
14694 vm_map_lock_write_to_read(map); in vm_map_lookup_and_lock_object()
14727 (csm_address_space_exempt(map->pmap) == KERN_SUCCESS) || in vm_map_lookup_and_lock_object()
14776 out_version->main_timestamp = map->timestamp; in vm_map_lookup_and_lock_object()
14792 vm_map_t map, in vm_map_verify() argument
14797 vm_map_lock_assert_held(map); in vm_map_verify()
14798 result = (map->timestamp == version->main_timestamp); in vm_map_verify()
14814 vm_map_t map, in vm_map_region_recurse_64() argument
14872 if (map == VM_MAP_NULL) { in vm_map_region_recurse_64()
14877 user_address = vm_sanitize_addr(map, *address_u); in vm_map_region_recurse_64()
14879 effective_page_shift = vm_self_region_page_shift(map); in vm_map_region_recurse_64()
14915 vm_map_lock_read(map); in vm_map_region_recurse_64()
14920 curr_map = map; in vm_map_region_recurse_64()
15099 user_address <= vm_map_last_entry(map)->vme_end) { in vm_map_region_recurse_64()
15109 task_ledgers_footprint(map->pmap->ledger, in vm_map_region_recurse_64()
15136 submap_info->object_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile); in vm_map_region_recurse_64()
15148 short_info->object_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile); in vm_map_region_recurse_64()
15158 *address_u = vm_sanitize_wrap_addr(vm_map_last_entry(map)->vme_end); in vm_map_region_recurse_64()
15330 vm_map_t map, in vm_map_region() argument
15342 if (map == VM_MAP_NULL) { in vm_map_region()
15346 start = vm_sanitize_addr(map, *address_u); in vm_map_region()
15361 vm_map_lock_read(map); in vm_map_region()
15363 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { in vm_map_region()
15364 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { in vm_map_region()
15365 vm_map_unlock_read(map); in vm_map_region()
15394 vm_map_unlock_read(map); in vm_map_region()
15409 vm_map_lock_read(map); in vm_map_region()
15411 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { in vm_map_region()
15412 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { in vm_map_region()
15413 vm_map_unlock_read(map); in vm_map_region()
15442 vm_map_unlock_read(map); in vm_map_region()
15462 effective_page_shift = vm_self_region_page_shift(map); in vm_map_region()
15465 vm_map_lock_read(map); in vm_map_region()
15467 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { in vm_map_region()
15468 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { in vm_map_region()
15469 vm_map_unlock_read(map); in vm_map_region()
15494 …vm_map_region_walk(map, start, entry, VME_OFFSET(entry), entry->vme_end - start, extended, TRUE, *… in vm_map_region()
15503 vm_map_unlock_read(map); in vm_map_region()
15517 vm_map_lock_read(map); in vm_map_region()
15519 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { in vm_map_region()
15520 if ((entry = tmp_entry->vme_next) == vm_map_to_entry(map)) { in vm_map_region()
15521 vm_map_unlock_read(map); in vm_map_region()
15541 vm_map_unlock_read(map); in vm_map_region()
15645 vm_map_t map, in vm_map_region_walk() argument
15687 effective_page_shift = vm_self_region_page_shift(map); in vm_map_region_walk()
15710 if (map->has_corpse_footprint) { in vm_map_region_walk()
15716 map, in vm_map_region_walk()
15724 map, in vm_map_region_walk()
15744 vm_map_region_look_for_page(map, va, obj, in vm_map_region_walk()
15823 last = vm_map_to_entry(map); in vm_map_region_walk()
15830 for (cur = vm_map_first_entry(map); cur != last; cur = cur->vme_next) { in vm_map_region_walk()
15850 __unused vm_map_t map, in vm_map_region_look_for_page() argument
16017 vm_map_t map, in vm_map_simplify_entry() argument
16024 if ((this_entry != vm_map_to_entry(map)) && in vm_map_simplify_entry()
16025 (prev_entry != vm_map_to_entry(map)) && in vm_map_simplify_entry()
16077 vm_map_store_entry_unlink(map, prev_entry, true); in vm_map_simplify_entry()
16081 VM_MAP_PAGE_MASK(map))); in vm_map_simplify_entry()
16086 if (map->holelistenabled) { in vm_map_simplify_entry()
16087 vm_map_store_update_first_free(map, this_entry, TRUE); in vm_map_simplify_entry()
16096 SAVE_HINT_MAP_WRITE(map, this_entry); in vm_map_simplify_entry()
16102 vm_map_t map, in vm_map_simplify() argument
16107 vm_map_lock(map); in vm_map_simplify()
16108 if (vm_map_lookup_entry(map, start, &this_entry)) { in vm_map_simplify()
16109 vm_map_simplify_entry(map, this_entry); in vm_map_simplify()
16110 vm_map_simplify_entry(map, this_entry->vme_next); in vm_map_simplify()
16112 vm_map_unlock(map); in vm_map_simplify()
16117 vm_map_t map, in vm_map_simplify_range() argument
16133 VM_MAP_PAGE_MASK(map)); in vm_map_simplify_range()
16135 VM_MAP_PAGE_MASK(map)); in vm_map_simplify_range()
16137 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_simplify_range()
16139 if (entry == vm_map_to_entry(map)) { in vm_map_simplify_range()
16141 entry = vm_map_first_entry(map); in vm_map_simplify_range()
16148 while (entry != vm_map_to_entry(map) && in vm_map_simplify_range()
16151 vm_map_simplify_entry(map, entry); in vm_map_simplify_range()
16159 vm_map_t map, in vm_map_machine_attribute_sanitize() argument
16167 VM_SANITIZE_CALLER_VM_MAP_MACHINE_ATTRIBUTE, map, in vm_map_machine_attribute_sanitize()
16189 vm_map_t map, in vm_map_machine_attribute() argument
16200 ret = vm_map_machine_attribute_sanitize(map, in vm_map_machine_attribute()
16210 if (start < vm_map_min(map) || end > vm_map_max(map)) { in vm_map_machine_attribute()
16214 vm_map_lock(map); in vm_map_machine_attribute()
16219 ret = pmap_attribute(map->pmap, start, end - start, in vm_map_machine_attribute()
16221 vm_map_unlock(map); in vm_map_machine_attribute()
16228 if (vm_map_lookup_entry(map, start, &entry)) { in vm_map_machine_attribute()
16305 vm_map_unlock(map); in vm_map_machine_attribute()
16310 vm_map_unlock(map); in vm_map_machine_attribute()
16325 vm_map_t map, in vm_map_behavior_set() argument
16334 start < vm_map_min(map) || in vm_map_behavior_set()
16335 end > vm_map_max(map)) { in vm_map_behavior_set()
16338 if (__improbable(vm_map_range_overflows(map, start, end - start))) { in vm_map_behavior_set()
16354 vm_map_lock(map); in vm_map_behavior_set()
16363 if (vm_map_range_check(map, start, end, &temp_entry)) { in vm_map_behavior_set()
16365 vm_map_clip_start(map, entry, start); in vm_map_behavior_set()
16367 vm_map_unlock(map); in vm_map_behavior_set()
16371 while ((entry != vm_map_to_entry(map)) && (entry->vme_start < end)) { in vm_map_behavior_set()
16372 vm_map_clip_end(map, entry, end); in vm_map_behavior_set()
16385 vm_map_unlock(map); in vm_map_behavior_set()
16395 return vm_map_willneed(map, start, end); in vm_map_behavior_set()
16398 return vm_map_msync(map, start, end - start, VM_SYNC_DEACTIVATE | VM_SYNC_CONTIGUOUS); in vm_map_behavior_set()
16401 return vm_map_msync(map, start, end - start, VM_SYNC_KILLPAGES | VM_SYNC_CONTIGUOUS); in vm_map_behavior_set()
16404 return vm_map_reusable_pages(map, start, end); in vm_map_behavior_set()
16407 return vm_map_reuse_pages(map, start, end); in vm_map_behavior_set()
16410 return vm_map_can_reuse(map, start, end); in vm_map_behavior_set()
16414 return vm_map_pageout(map, start, end); in vm_map_behavior_set()
16418 return vm_map_zero(map, start, end); in vm_map_behavior_set()
16439 vm_map_t map, in vm_map_willneed() argument
16463 vm_map_lock_read(map); in vm_map_willneed()
16471 if (!vm_map_range_check(map, start, end, &entry)) { in vm_map_willneed()
16472 vm_map_unlock_read(map); in vm_map_willneed()
16481 for (; entry != vm_map_to_entry(map) && start < end;) { in vm_map_willneed()
16543 vm_map_unlock_read(map); in vm_map_willneed()
16573 vm_map_unlock_read(map); in vm_map_willneed()
16623 vm_map_lock_read(map); in vm_map_willneed()
16624 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_willneed()
16628 vm_map_unlock_read(map); in vm_map_willneed()
16635 vm_map_unlock_read(map); in vm_map_willneed()
16723 vm_map_t map, in vm_map_reuse_pages() argument
16736 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) { in vm_map_reuse_pages()
16745 vm_map_lock_read(map); in vm_map_reuse_pages()
16746 assert(map->pmap != kernel_pmap); /* protect alias access */ in vm_map_reuse_pages()
16754 if (!vm_map_range_check(map, start, end, &entry)) { in vm_map_reuse_pages()
16755 vm_map_unlock_read(map); in vm_map_reuse_pages()
16763 for (; entry != vm_map_to_entry(map) && entry->vme_start < end; in vm_map_reuse_pages()
16769 vm_map_unlock_read(map); in vm_map_reuse_pages()
16809 vm_map_unlock_read(map); in vm_map_reuse_pages()
16817 vm_map_t map, in vm_map_reusable_pages() argument
16826 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) { in vm_map_reusable_pages()
16840 vm_map_lock_read(map); in vm_map_reusable_pages()
16841 assert(map->pmap != kernel_pmap); /* protect alias access */ in vm_map_reusable_pages()
16849 if (!vm_map_range_check(map, start, end, &entry)) { in vm_map_reusable_pages()
16850 vm_map_unlock_read(map); in vm_map_reusable_pages()
16858 for (; entry != vm_map_to_entry(map) && entry->vme_start < end; in vm_map_reusable_pages()
16867 vm_map_unlock_read(map); in vm_map_reusable_pages()
16878 vm_map_unlock_read(map); in vm_map_reusable_pages()
16950 map->pmap, in vm_map_reusable_pages()
16956 vm_map_t, map, in vm_map_reusable_pages()
16976 vm_map_unlock_read(map); in vm_map_reusable_pages()
16984 vm_map_t map, in vm_map_can_reuse() argument
16995 vm_map_lock_read(map); in vm_map_can_reuse()
16996 assert(map->pmap != kernel_pmap); /* protect alias access */ in vm_map_can_reuse()
17004 if (!vm_map_range_check(map, start, end, &entry)) { in vm_map_can_reuse()
17005 vm_map_unlock_read(map); in vm_map_can_reuse()
17013 for (; entry != vm_map_to_entry(map) && entry->vme_start < end; in vm_map_can_reuse()
17019 vm_map_unlock_read(map); in vm_map_can_reuse()
17025 vm_map_unlock_read(map); in vm_map_can_reuse()
17034 vm_map_t map, in vm_map_pageout() argument
17045 vm_map_lock_read(map); in vm_map_pageout()
17053 if (!vm_map_range_check(map, start, end, &entry)) { in vm_map_pageout()
17054 vm_map_unlock_read(map); in vm_map_pageout()
17061 for (; entry != vm_map_to_entry(map) && entry->vme_start < end; in vm_map_pageout()
17086 vm_map_unlock_read(map); in vm_map_pageout()
17117 vm_map_unlock_read(map); in vm_map_pageout()
17180 vm_map_t map, in vm_map_zero() argument
17193 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) { in vm_map_zero()
17201 vm_map_lock_read(map); in vm_map_zero()
17202 assert(map->pmap != kernel_pmap); /* protect alias access */ in vm_map_zero()
17209 if (!vm_map_range_check(map, cur, end, &entry)) { in vm_map_zero()
17210 vm_map_unlock_read(map); in vm_map_zero()
17217 while (entry != vm_map_to_entry(map) && entry->vme_start < end) { in vm_map_zero()
17220 unsigned int last_timestamp = map->timestamp; in vm_map_zero()
17225 vm_map_unlock_read(map); in vm_map_zero()
17241 vm_map_unlock_read(map); in vm_map_zero()
17261 vm_map_lock_read(map); in vm_map_zero()
17262 if (last_timestamp != map->timestamp) { in vm_map_zero()
17266 if (!vm_map_range_check(map, cur, end, &entry)) { in vm_map_zero()
17267 vm_map_unlock_read(map); in vm_map_zero()
17278 vm_map_unlock_read(map); in vm_map_zero()
17290 vm_map_t map, in vm_map_entry_insert() argument
17307 vm_map_lock_assert_exclusive(map); in vm_map_entry_insert()
17312 if (VM_MAP_PAGE_SHIFT(map) != PAGE_SHIFT) { in vm_map_entry_insert()
17316 (!VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map)) || in vm_map_entry_insert()
17317 !VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map)))) { in vm_map_entry_insert()
17321 assert(VM_MAP_PAGE_ALIGNED(start, VM_MAP_PAGE_MASK(map))); in vm_map_entry_insert()
17322 assert(VM_MAP_PAGE_ALIGNED(end, VM_MAP_PAGE_MASK(map))); in vm_map_entry_insert()
17329 new_entry = vm_map_entry_create(map); in vm_map_entry_insert()
17363 if (!(map->jit_entry_exists) || in vm_map_entry_insert()
17364 VM_MAP_POLICY_ALLOW_MULTIPLE_JIT(map)) { in vm_map_entry_insert()
17366 map->jit_entry_exists = TRUE; in vm_map_entry_insert()
17374 vm_map_store_entry_link(map, insp_entry, new_entry, vmk_flags); in vm_map_entry_insert()
17375 map->size += end - start; in vm_map_entry_insert()
17381 SAVE_HINT_MAP_WRITE(map, new_entry); in vm_map_entry_insert()
17392 vm_map_t map, in vm_map_remap_extract() argument
17430 effective_page_mask = MIN(PAGE_MASK, VM_MAP_PAGE_MASK(map)); in vm_map_remap_extract()
17432 assert(map != VM_MAP_NULL); in vm_map_remap_extract()
17454 map_header->page_shift = (uint16_t)VM_MAP_PAGE_SHIFT(map); in vm_map_remap_extract()
17514 vm_map_lock(map); in vm_map_remap_extract()
17516 if (map->pmap == kernel_pmap) { in vm_map_remap_extract()
17520 } else if (map->uses_user_ranges) { in vm_map_remap_extract()
17522 map_copy->orig_range = vm_map_user_range_resolve(map, addr, size, NULL); in vm_map_remap_extract()
17526 if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) { in vm_map_remap_extract()
17533 vm_map_simplify_range(map, src_start, src_end); in vm_map_remap_extract()
17541 if (!vm_map_lookup_entry(map, src_start, &src_entry)) { in vm_map_remap_extract()
17612 vm_map_unlock(map); in vm_map_remap_extract()
17620 vm_map_t, map, in vm_map_remap_extract()
17794 vm_map_t, map, in vm_map_remap_extract()
17811 vm_map_unlock(map); in vm_map_remap_extract()
17840 vm_map_lock(map); in vm_map_remap_extract()
17866 vm_map_lock(map); in vm_map_remap_extract()
17891 vm_map_lock(map); in vm_map_remap_extract()
17925 map, in vm_map_remap_extract()
17939 map, in vm_map_remap_extract()
17969 assert(!map->mapped_in_other_pmaps); in vm_map_remap_extract()
17995 vm_map_always_shadow(map)); in vm_map_remap_extract()
18009 … if (pmap_has_prot_policy(map->pmap, src_entry->translated_allow_execute, src_entry->protection)) { in vm_map_remap_extract()
18012 map, map->pmap, in vm_map_remap_extract()
18021 if (override_nx(map, in vm_map_remap_extract()
18027 if (pmap_has_prot_policy(map->pmap, src_entry->translated_allow_execute, prot)) { in vm_map_remap_extract()
18030 map, map->pmap, in vm_map_remap_extract()
18037 if (map->mapped_in_other_pmaps) { in vm_map_remap_extract()
18047 } else if (__improbable(map->pmap == PMAP_NULL)) { in vm_map_remap_extract()
18060 pmap_protect(vm_map_pmap(map), in vm_map_remap_extract()
18100 vm_map_entry_copy(map, new_entry, src_entry); in vm_map_remap_extract()
18179 } else if (!VM_MAP_POLICY_ALLOW_JIT_SHARING(map)) { in vm_map_remap_extract()
18219 !(debug4k_no_cow_copyin && VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) && in vm_map_remap_extract()
18228 assertf(new_entry->use_pmap, "map %p new_entry %p\n", map, new_entry); in vm_map_remap_extract()
18236 … if (pmap_has_prot_policy(map->pmap, src_entry->translated_allow_execute, src_entry->protection)) { in vm_map_remap_extract()
18239 map, map->pmap, src_entry, in vm_map_remap_extract()
18247 if (override_nx(map, in vm_map_remap_extract()
18253 if (pmap_has_prot_policy(map->pmap, src_entry->translated_allow_execute, prot)) { in vm_map_remap_extract()
18256 map, map->pmap, src_entry, in vm_map_remap_extract()
18266 || map->mapped_in_other_pmaps) ? in vm_map_remap_extract()
18267 PMAP_NULL : map->pmap), in vm_map_remap_extract()
18268 VM_MAP_PAGE_SIZE(map), in vm_map_remap_extract()
18281 assertf(new_entry->use_pmap, "map %p new_entry %p\n", map, new_entry); in vm_map_remap_extract()
18294 version.main_timestamp = map->timestamp; in vm_map_remap_extract()
18295 vm_map_unlock(map); /* Increments timestamp once! */ in vm_map_remap_extract()
18302 VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT)) { in vm_map_remap_extract()
18349 vm_map_lock(map); in vm_map_remap_extract()
18358 vm_map_lock(map); in vm_map_remap_extract()
18359 if (version.main_timestamp + 1 != map->timestamp) { in vm_map_remap_extract()
18400 map, (uint64_t)addr, (uint64_t)size, (uint64_t)mapped_size); in vm_map_remap_extract()
18401 if (src_entry->vme_next != vm_map_to_entry(map) && in vm_map_remap_extract()
18412 vm_map_unlock(map); in vm_map_remap_extract()
18435 vm_map_t map) in vm_map_is_exotic() argument
18437 return VM_MAP_IS_EXOTIC(map); in vm_map_is_exotic()
18442 vm_map_t map) in vm_map_is_alien() argument
18444 return VM_MAP_IS_ALIEN(map); in vm_map_is_alien()
18450 vm_map_t map) in vm_map_mark_alien() argument
18452 vm_map_lock(map); in vm_map_mark_alien()
18453 map->is_alien = true; in vm_map_mark_alien()
18454 vm_map_unlock(map); in vm_map_mark_alien()
18459 vm_map_t map) in vm_map_single_jit() argument
18461 vm_map_lock(map); in vm_map_single_jit()
18462 map->single_jit = true; in vm_map_single_jit()
18463 vm_map_unlock(map); in vm_map_single_jit()
19028 vm_map_t map, in vm_map_range_physical_size() argument
19042 …DEBUG4K_SHARE("map %p start 0x%llx size 0x%llx -> phys_size 0!\n", map, (uint64_t)start, (uint64_t… in vm_map_range_physical_size()
19047 adjusted_start = vm_map_trunc_page(start, VM_MAP_PAGE_MASK(map)); in vm_map_range_physical_size()
19048 adjusted_end = vm_map_round_page(start + size, VM_MAP_PAGE_MASK(map)); in vm_map_range_physical_size()
19052 …%x: wraparound\n", __FUNCTION__, __LINE__, (uint64_t)start, (uint64_t)size, VM_MAP_PAGE_MASK(map)); in vm_map_range_physical_size()
19056 if (__improbable(vm_map_range_overflows(map, start, size))) { in vm_map_range_physical_size()
19063 if (VM_MAP_PAGE_SIZE(map) == PAGE_SIZE) { in vm_map_range_physical_size()
19088 kr = vm_map_copy_extract(map, adjusted_start, adjusted_size, in vm_map_range_physical_size()
19094 …DEBUG4K_ERROR("map %p start 0x%llx 0x%llx size 0x%llx 0x%llx kr 0x%x\n", map, (uint64_t)start, (ui… in vm_map_range_physical_size()
19114 …end 0x%llx trimmed_start 0x%llx phys_size 0x%llx -> 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint6… in vm_map_range_physical_size()
19118 …DEBUG4K_ERROR("map %p start 0x%llx 0x%llx size 0x%llx 0x%llx kr 0x%x\n", map, (uint64_t)start, (ui… in vm_map_range_physical_size()
19565 vm_map_t map) in vm_map_switch() argument
19568 vm_map_t oldmap = thread->map; in vm_map_switch()
19575 PMAP_SWITCH_USER(thread, map, cpu_number()); in vm_map_switch()
19583 vm_map_t map, in vm_map_rw_user_sanitize() argument
19596 vm_sanitize_caller, map, in vm_map_rw_user_sanitize()
19615 vm_map_t map, in vm_map_write_user() argument
19630 kr = vm_map_rw_user_sanitize(map, in vm_map_write_user()
19641 if (current_map() == map) { in vm_map_write_user()
19651 vm_map_reference(map); in vm_map_write_user()
19652 oldmap = vm_map_switch(map); in vm_map_write_user()
19657 vm_map_deallocate(map); in vm_map_write_user()
19676 vm_map_t map, in vm_map_read_user() argument
19691 kr = vm_map_rw_user_sanitize(map, in vm_map_read_user()
19702 if (current_map() == map) { in vm_map_read_user()
19712 vm_map_reference(map); in vm_map_read_user()
19713 oldmap = vm_map_switch(map); in vm_map_read_user()
19718 vm_map_deallocate(map); in vm_map_read_user()
19727 vm_map_t map, in vm_map_check_protection_sanitize() argument
19739 kr = vm_sanitize_addr_end(start_u, end_u, vm_sanitize_caller, map, in vm_map_check_protection_sanitize()
19764 vm_map_t map, in vm_map_check_protection() argument
19777 kr = vm_map_check_protection_sanitize(map, in vm_map_check_protection()
19793 vm_map_lock(map); in vm_map_check_protection()
19795 if (start < vm_map_min(map) || end > vm_map_max(map)) { in vm_map_check_protection()
19796 vm_map_unlock(map); in vm_map_check_protection()
19800 if (!vm_map_lookup_entry(map, start, &tmp_entry)) { in vm_map_check_protection()
19801 vm_map_unlock(map); in vm_map_check_protection()
19808 if (entry == vm_map_to_entry(map)) { in vm_map_check_protection()
19809 vm_map_unlock(map); in vm_map_check_protection()
19818 vm_map_unlock(map); in vm_map_check_protection()
19827 vm_map_unlock(map); in vm_map_check_protection()
19836 vm_map_unlock(map); in vm_map_check_protection()
19842 vm_map_t map, in vm_map_purgable_control() argument
19857 if (map == VM_MAP_NULL) { in vm_map_purgable_control()
19880 address = vm_sanitize_addr(map, address_u); in vm_map_purgable_control()
19882 vm_map_lock_read(map); in vm_map_purgable_control()
19884 if (!vm_map_lookup_entry(map, address, &entry) || entry->is_sub_map) { in vm_map_purgable_control()
19888 vm_map_unlock_read(map); in vm_map_purgable_control()
19897 vm_map_unlock_read(map); in vm_map_purgable_control()
19907 vm_map_unlock_read(map); in vm_map_purgable_control()
19920 vm_map_unlock_read(map); in vm_map_purgable_control()
19929 vm_map_unlock_read(map); in vm_map_purgable_control()
19937 map->pmap == kernel_pmap) { in vm_map_purgable_control()
19950 vm_map_t map, in vm_map_footprint_query_page_info() argument
19960 vm_map_lock_assert_held(map); in vm_map_footprint_query_page_info()
19961 assert(!map->has_corpse_footprint); in vm_map_footprint_query_page_info()
19980 effective_page_size = MIN(PAGE_SIZE, VM_MAP_PAGE_SIZE(map)); in vm_map_footprint_query_page_info()
19987 pmap_query_page_info(map->pmap, curr_s_offset, &pmap_disp); in vm_map_footprint_query_page_info()
20003 VM_OBJECT_OWNER(object)->map == map) { in vm_map_footprint_query_page_info()
20028 VM_OBJECT_OWNER(object)->map == map) { in vm_map_footprint_query_page_info()
20056 VM_OBJECT_OWNER(object)->map == map) { in vm_map_footprint_query_page_info()
20095 if (map->pmap->footprint_was_suspended) { in vm_map_footprint_query_page_info()
20135 vm_map_t map, in vm_map_page_info() argument
20141 return vm_map_page_range_info_internal(map, in vm_map_page_info()
20153 vm_map_t map, in vm_map_page_range_info_sanitize() argument
20169 VM_SANITIZE_CALLER_VM_MAP_PAGE_RANGE_INFO, map, in vm_map_page_range_info_sanitize()
20193 vm_map_t map, in vm_map_page_range_info_internal() argument
20234 effective_page_shift = vm_self_region_page_shift_safely(map); in vm_map_page_range_info_internal()
20243 retval = vm_map_page_range_info_sanitize(map, in vm_map_page_range_info_internal()
20262 vm_map_lock_read(map); in vm_map_page_range_info_internal()
20264 task_ledgers_footprint(map->pmap->ledger, &ledger_resident, &ledger_compressed); in vm_map_page_range_info_internal()
20276 curr_s_offset >= vm_map_last_entry(map)->vme_end) { in vm_map_page_range_info_internal()
20286 if (curr_s_offset - vm_map_last_entry(map)->vme_end <= in vm_map_page_range_info_internal()
20310 basic_info->object_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile); in vm_map_page_range_info_internal()
20325 if (!vm_map_lookup_entry(map, curr_s_offset, &map_entry)) { in vm_map_page_range_info_internal()
20328 if (curr_s_offset < vm_map_min(map)) { in vm_map_page_range_info_internal()
20332 curr_e_offset = MIN(end, vm_map_min(map)); in vm_map_page_range_info_internal()
20333 } else if (curr_s_offset >= vm_map_max(map)) { in vm_map_page_range_info_internal()
20338 } else if (map_entry == vm_map_to_entry(map)) { in vm_map_page_range_info_internal()
20342 if (map_entry->vme_next == vm_map_to_entry(map)) { in vm_map_page_range_info_internal()
20346 curr_e_offset = MIN(map->max_offset, end); in vm_map_page_range_info_internal()
20354 if (map_entry->vme_next == vm_map_to_entry(map)) { in vm_map_page_range_info_internal()
20358 curr_e_offset = MIN(map->max_offset, end); in vm_map_page_range_info_internal()
20398 vm_map_unlock_read(map); in vm_map_page_range_info_internal()
20402 assertf(VM_MAP_PAGE_SHIFT(sub_map) >= VM_MAP_PAGE_SHIFT(map), in vm_map_page_range_info_internal()
20403 …page size (%d) differs from current map (%d)\n", VM_MAP_PAGE_SIZE(sub_map), VM_MAP_PAGE_SIZE(map)); in vm_map_page_range_info_internal()
20415 vm_map_lock_read(map); in vm_map_page_range_info_internal()
20453 if (map->has_corpse_footprint) { in vm_map_page_range_info_internal()
20459 map, in vm_map_page_range_info_internal()
20468 map, in vm_map_page_range_info_internal()
20478 basic_info->object_id = VM_OBJECT_ID_FAKE(map, task_ledgers.purgeable_nonvolatile); in vm_map_page_range_info_internal()
20498 vm_map_unlock_read(map); in vm_map_page_range_info_internal()
20650 vm_map_lock_read(map); in vm_map_page_range_info_internal()
20653 vm_map_unlock_read(map); in vm_map_page_range_info_internal()
20660 vm_map_t map, in vm_map_msync_sanitize() argument
20670 map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, in vm_map_msync_sanitize()
20714 vm_map_t map, in vm_map_msync() argument
20733 if (map == VM_MAP_NULL) { in vm_map_msync()
20737 kr = vm_map_msync_sanitize(map, in vm_map_msync()
20742 if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) { in vm_map_msync()
20743 …DEBUG4K_SHARE("map %p address 0x%llx size 0x%llx flags 0x%x\n", map, (uint64_t)address, (uint64_t)… in vm_map_msync()
20755 vm_map_lock(map); in vm_map_msync()
20756 if (!vm_map_lookup_entry(map, in vm_map_msync()
20774 vm_map_unlock(map); in vm_map_msync()
20781 if (entry == vm_map_to_entry(map) && in vm_map_msync()
20783 vm_map_unlock(map); in vm_map_msync()
20790 if ((map->hdr.nentries == 0) || in vm_map_msync()
20792 vm_map_unlock(map); in vm_map_msync()
20805 vm_map_unlock(map); in vm_map_msync()
20832 vm_map_unlock(map); in vm_map_msync()
20850 vm_map_unlock(map); in vm_map_msync()
20860 if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) { in vm_map_msync()
20871 vm_map_unlock(map); in vm_map_msync()
20903 map->pmap, in vm_map_msync()
20907 vm_map_unlock(map); in vm_map_msync()
20918 vm_map_unlock(map); in vm_map_msync()
20927 vm_map_unlock(map); in vm_map_msync()
20929 if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) { in vm_map_msync()
21053 vm_map_t map = VM_MAP_NULL; in convert_port_entry_to_map() local
21068 map = named_entry->backing.map; in convert_port_entry_to_map()
21069 if (map->pmap != PMAP_NULL) { in convert_port_entry_to_map()
21070 if (map->pmap == kernel_pmap) { in convert_port_entry_to_map()
21072 "to a kernel map %p", map); in convert_port_entry_to_map()
21074 pmap_require(map->pmap); in convert_port_entry_to_map()
21076 vm_map_reference(map); in convert_port_entry_to_map()
21079 return map; in convert_port_entry_to_map()
21100 vm_map_t map) in vm_map_reference() argument
21102 if (__probable(map != VM_MAP_NULL)) { in vm_map_reference()
21103 vm_map_require(map); in vm_map_reference()
21104 os_ref_retain_raw(&map->map_refcnt, &map_refgrp); in vm_map_reference()
21117 vm_map_t map) in vm_map_deallocate() argument
21119 if (__probable(map != VM_MAP_NULL)) { in vm_map_deallocate()
21120 vm_map_require(map); in vm_map_deallocate()
21121 if (os_ref_release_raw(&map->map_refcnt, &map_refgrp) == 0) { in vm_map_deallocate()
21122 vm_map_destroy(map); in vm_map_deallocate()
21129 vm_map_inspect_t map) in vm_map_inspect_deallocate() argument
21131 vm_map_deallocate((vm_map_t)map); in vm_map_inspect_deallocate()
21136 vm_map_read_t map) in vm_map_read_deallocate() argument
21138 vm_map_deallocate((vm_map_t)map); in vm_map_read_deallocate()
21143 vm_map_disable_NX(vm_map_t map) in vm_map_disable_NX() argument
21145 if (map == NULL) { in vm_map_disable_NX()
21148 if (map->pmap == NULL) { in vm_map_disable_NX()
21152 pmap_disable_NX(map->pmap); in vm_map_disable_NX()
21156 vm_map_disallow_data_exec(vm_map_t map) in vm_map_disallow_data_exec() argument
21158 if (map == NULL) { in vm_map_disallow_data_exec()
21162 map->map_disallow_data_exec = TRUE; in vm_map_disallow_data_exec()
21169 vm_map_set_32bit(vm_map_t map) in vm_map_set_32bit() argument
21172 map->max_offset = pmap_max_offset(FALSE, ARM_PMAP_MAX_OFFSET_DEVICE); in vm_map_set_32bit()
21174 map->max_offset = (vm_map_offset_t)VM_MAX_ADDRESS; in vm_map_set_32bit()
21180 vm_map_set_64bit(vm_map_t map) in vm_map_set_64bit() argument
21183 map->max_offset = pmap_max_offset(TRUE, ARM_PMAP_MAX_OFFSET_DEVICE); in vm_map_set_64bit()
21185 map->max_offset = (vm_map_offset_t)MACH_VM_MAX_ADDRESS; in vm_map_set_64bit()
21193 vm_map_set_jumbo(vm_map_t map) in vm_map_set_jumbo() argument
21196 vm_map_set_max_addr(map, ~0, false); in vm_map_set_jumbo()
21198 (void) map; in vm_map_set_jumbo()
21207 vm_map_set_extra_jumbo(vm_map_t map) in vm_map_set_extra_jumbo() argument
21210 vm_map_set_max_addr(map, ~0, true); in vm_map_set_extra_jumbo()
21212 (void) map; in vm_map_set_extra_jumbo()
21221 vm_map_set_jit_entitled(vm_map_t map) in vm_map_set_jit_entitled() argument
21224 pmap_set_jit_entitled(map->pmap); in vm_map_set_jit_entitled()
21226 (void) map; in vm_map_set_jit_entitled()
21234 vm_map_tpro(vm_map_t map) in vm_map_tpro() argument
21237 return pmap_get_tpro(map->pmap); in vm_map_tpro()
21239 (void) map; in vm_map_tpro()
21248 vm_map_set_tpro(vm_map_t map) in vm_map_set_tpro() argument
21251 pmap_set_tpro(map->pmap); in vm_map_set_tpro()
21253 (void) map; in vm_map_set_tpro()
21261 vm_map_tpro_enforcement(vm_map_t map) in vm_map_tpro_enforcement() argument
21263 return map->tpro_enforcement; in vm_map_tpro_enforcement()
21270 vm_map_set_tpro_enforcement(vm_map_t map) in vm_map_set_tpro_enforcement() argument
21272 if (vm_map_tpro(map)) { in vm_map_set_tpro_enforcement()
21273 vm_map_lock(map); in vm_map_set_tpro_enforcement()
21274 map->tpro_enforcement = TRUE; in vm_map_set_tpro_enforcement()
21275 vm_map_unlock(map); in vm_map_set_tpro_enforcement()
21289 __unused vm_map_t map, in vm_map_set_tpro_range() argument
21301 vm_map_t map, in vm_map_set_max_addr() argument
21310 vm_map_lock(map); in vm_map_set_max_addr()
21312 old_max_offset = map->max_offset; in vm_map_set_max_addr()
21318 max_supported_offset = pmap_max_offset(vm_map_is_64bit(map), option); in vm_map_set_max_addr()
21324 vm_map_unlock(map); in vm_map_set_max_addr()
21332 map->max_offset = new_max_offset; in vm_map_set_max_addr()
21344 if (map->holelistenabled) { in vm_map_set_max_addr()
21345 if (map->holes_list->prev->vme_end == old_max_offset) { in vm_map_set_max_addr()
21349 map->holes_list->prev->vme_end = map->max_offset; in vm_map_set_max_addr()
21359 new_hole->end = map->max_offset; in vm_map_set_max_addr()
21360 new_hole->prev = map->holes_list->prev; in vm_map_set_max_addr()
21361 new_hole->next = (struct vm_map_entry *)map->holes_list; in vm_map_set_max_addr()
21362 map->holes_list->prev->vme_next = (struct vm_map_entry *)new_hole; in vm_map_set_max_addr()
21363 map->holes_list->prev = (struct vm_map_entry *)new_hole; in vm_map_set_max_addr()
21368 vm_map_unlock(map); in vm_map_set_max_addr()
21370 (void)map; in vm_map_set_max_addr()
21387 vm_map_t map __unused, in vm_map_get_max_aslr_slide_section()
21401 vm_map_get_max_aslr_slide_pages(vm_map_t map) in vm_map_get_max_aslr_slide_pages() argument
21408 return 1 << (24 - VM_MAP_PAGE_SHIFT(map)); in vm_map_get_max_aslr_slide_pages()
21410 return 1 << (vm_map_is_64bit(map) ? 16 : 8); in vm_map_get_max_aslr_slide_pages()
21415 vm_map_get_max_loader_aslr_slide_pages(vm_map_t map) in vm_map_get_max_loader_aslr_slide_pages() argument
21421 return 1 << (22 - VM_MAP_PAGE_SHIFT(map)); in vm_map_get_max_loader_aslr_slide_pages()
21423 return 1 << (vm_map_is_64bit(map) ? 16 : 8); in vm_map_get_max_loader_aslr_slide_pages()
21429 vm_map_t map) in vm_map_is_64bit() argument
21431 return map->max_offset > ((vm_map_offset_t)VM_MAX_ADDRESS); in vm_map_is_64bit()
21436 vm_map_t map, in vm_map_has_hard_pagezero() argument
21449 return map->min_offset >= pagezero_size; in vm_map_has_hard_pagezero()
21457 vm_map_t map, in vm_map_raise_max_offset() argument
21462 vm_map_lock(map); in vm_map_raise_max_offset()
21465 if (new_max_offset >= map->max_offset) { in vm_map_raise_max_offset()
21466 if (!vm_map_is_64bit(map)) { in vm_map_raise_max_offset()
21468 map->max_offset = new_max_offset; in vm_map_raise_max_offset()
21473 map->max_offset = new_max_offset; in vm_map_raise_max_offset()
21479 vm_map_unlock(map); in vm_map_raise_max_offset()
21490 vm_map_t map, in vm_map_raise_min_offset() argument
21496 VM_MAP_PAGE_MASK(map)); in vm_map_raise_min_offset()
21498 vm_map_lock(map); in vm_map_raise_min_offset()
21500 if (new_min_offset < map->min_offset) { in vm_map_raise_min_offset()
21506 vm_map_unlock(map); in vm_map_raise_min_offset()
21509 if (new_min_offset >= map->max_offset) { in vm_map_raise_min_offset()
21511 vm_map_unlock(map); in vm_map_raise_min_offset()
21515 first_entry = vm_map_first_entry(map); in vm_map_raise_min_offset()
21516 if (first_entry != vm_map_to_entry(map) && in vm_map_raise_min_offset()
21522 vm_map_unlock(map); in vm_map_raise_min_offset()
21526 map->min_offset = new_min_offset; in vm_map_raise_min_offset()
21528 if (map->holelistenabled) { in vm_map_raise_min_offset()
21529 assert(map->holes_list); in vm_map_raise_min_offset()
21530 map->holes_list->start = new_min_offset; in vm_map_raise_min_offset()
21531 assert(new_min_offset < map->holes_list->end); in vm_map_raise_min_offset()
21534 vm_map_unlock(map); in vm_map_raise_min_offset()
21548 vm_map_set_size_limit(vm_map_t map, uint64_t new_size_limit) in vm_map_set_size_limit() argument
21552 vm_map_lock(map); in vm_map_set_size_limit()
21553 if (new_size_limit < map->size) { in vm_map_set_size_limit()
21556 vm_map_size_t, map->size, in vm_map_set_size_limit()
21559 } else if (new_size_limit == map->size_limit) { in vm_map_set_size_limit()
21565 vm_map_size_t, map->size, in vm_map_set_size_limit()
21570 map->size_limit = new_size_limit; in vm_map_set_size_limit()
21573 vm_map_unlock(map); in vm_map_set_size_limit()
21579 vm_map_set_data_limit(vm_map_t map, uint64_t new_data_limit) in vm_map_set_data_limit() argument
21583 vm_map_lock(map); in vm_map_set_data_limit()
21584 if (new_data_limit < map->size) { in vm_map_set_data_limit()
21587 vm_map_size_t, map->size, in vm_map_set_data_limit()
21590 } else if (new_data_limit == map->data_limit) { in vm_map_set_data_limit()
21596 vm_map_size_t, map->size, in vm_map_set_data_limit()
21601 map->data_limit = new_data_limit; in vm_map_set_data_limit()
21604 vm_map_unlock(map); in vm_map_set_data_limit()
21609 vm_map_set_user_wire_limit(vm_map_t map, in vm_map_set_user_wire_limit() argument
21612 vm_map_lock(map); in vm_map_set_user_wire_limit()
21613 map->user_wire_limit = limit; in vm_map_set_user_wire_limit()
21614 vm_map_unlock(map); in vm_map_set_user_wire_limit()
21619 vm_map_switch_protect(vm_map_t map, in vm_map_switch_protect() argument
21622 vm_map_lock(map); in vm_map_switch_protect()
21623 map->switch_protect = val; in vm_map_switch_protect()
21624 vm_map_unlock(map); in vm_map_switch_protect()
21630 vm_map_t map) in vm_map_cs_enforcement() argument
21635 return map->cs_enforcement; in vm_map_cs_enforcement()
21640 __unused vm_map_t map) in vm_map_cs_wx_enable() argument
21643 kern_return_t ret = csm_allow_invalid_code(vm_map_pmap(map)); in vm_map_cs_wx_enable()
21656 __unused vm_map_t map) in vm_map_csm_allow_jit() argument
21659 return csm_allow_jit_region(vm_map_pmap(map)); in vm_map_csm_allow_jit()
21668 vm_map_t map, in vm_map_cs_debugged_set() argument
21671 vm_map_lock(map); in vm_map_cs_debugged_set()
21672 map->cs_debugged = val; in vm_map_cs_debugged_set()
21673 vm_map_unlock(map); in vm_map_cs_debugged_set()
21678 vm_map_t map, in vm_map_cs_enforcement_set() argument
21681 vm_map_lock(map); in vm_map_cs_enforcement_set()
21682 map->cs_enforcement = val; in vm_map_cs_enforcement_set()
21683 pmap_set_vm_map_cs_enforced(map->pmap, val); in vm_map_cs_enforcement_set()
21684 vm_map_unlock(map); in vm_map_cs_enforcement_set()
21693 vm_map_iokit_mapped_region(vm_map_t map, vm_size_t bytes) in vm_map_iokit_mapped_region() argument
21695 pmap_t pmap = vm_map_pmap(map); in vm_map_iokit_mapped_region()
21702 vm_map_iokit_unmapped_region(vm_map_t map, vm_size_t bytes) in vm_map_iokit_unmapped_region() argument
21704 pmap_t pmap = vm_map_pmap(map); in vm_map_iokit_unmapped_region()
21713 vm_map_sign(vm_map_t map, in vm_map_sign() argument
21725 if (map == VM_MAP_NULL) { in vm_map_sign()
21729 if (__improbable(vm_map_range_overflows(map, start, end - start))) { in vm_map_sign()
21733 vm_map_lock_read(map); in vm_map_sign()
21735 if (!vm_map_lookup_entry(map, start, &entry) || entry->is_sub_map) { in vm_map_sign()
21739 vm_map_unlock_read(map); in vm_map_sign()
21748 vm_map_unlock_read(map); in vm_map_sign()
21757 vm_map_unlock_read(map); in vm_map_sign()
21762 vm_map_unlock_read(map); in vm_map_sign()
21811 vm_map_partial_reap(vm_map_t map, unsigned int *reclaimed_resident, unsigned int *reclaimed_compres… in vm_map_partial_reap() argument
21818 vm_map_lock(map); in vm_map_partial_reap()
21820 for (entry = vm_map_first_entry(map); in vm_map_partial_reap()
21821 entry != vm_map_to_entry(map); in vm_map_partial_reap()
21832 (void)vm_map_delete(map, entry->vme_start, in vm_map_partial_reap()
21838 vm_map_unlock(map); in vm_map_partial_reap()
21850 vm_map_t map, in vm_map_disconnect_page_mappings() argument
21858 vm_map_lock(map); in vm_map_disconnect_page_mappings()
21860 for (entry = vm_map_first_entry(map); in vm_map_disconnect_page_mappings()
21861 entry != vm_map_to_entry(map); in vm_map_disconnect_page_mappings()
21870 vm_map_clip_unnest(map, entry, entry->vme_start, entry->vme_end); in vm_map_disconnect_page_mappings()
21873 vm_map_unlock(map); in vm_map_disconnect_page_mappings()
21876 vm_map_lock_read(map); in vm_map_disconnect_page_mappings()
21878 ledger_get_balance(map->pmap->ledger, task_ledgers.phys_mem, &byte_count); in vm_map_disconnect_page_mappings()
21880 for (entry = vm_map_first_entry(map); in vm_map_disconnect_page_mappings()
21881 entry != vm_map_to_entry(map); in vm_map_disconnect_page_mappings()
21891 pmap_remove_options(map->pmap, entry->vme_start, entry->vme_end, 0); in vm_map_disconnect_page_mappings()
21893 vm_map_unlock_read(map); in vm_map_disconnect_page_mappings()
21895 return (int) (byte_count / VM_MAP_PAGE_SIZE(map)); in vm_map_disconnect_page_mappings()
21899 vm_map_inject_error(vm_map_t map, vm_map_offset_t vaddr) in vm_map_inject_error() argument
21910 vm_map_lock(map); in vm_map_inject_error()
21912 result = vm_map_lookup_and_lock_object(&map, vaddr, VM_PROT_READ, in vm_map_inject_error()
21928 if (real_map != map) { in vm_map_inject_error()
21931 vm_map_unlock(map); in vm_map_inject_error()
21943 vm_map_entries_foreach_locked(vm_map_t map, kern_return_t (^count_handler)(int nentries),
21946 vm_map_lock_assert_held(map);
21947 int nentries = map->hdr.nentries;
21954 vm_map_entry_t entry = vm_map_first_entry(map);
21956 while (entry != vm_map_to_entry(map)) {
21975 vm_map_entries_foreach(vm_map_t map, kern_return_t (^count_handler)(int nentries),
21978 vm_map_lock_read(map);
21979 kern_return_t error = vm_map_entries_foreach_locked(map, count_handler, entry_handler);
21980 vm_map_unlock_read(map);
22083 vm_map_t map = task->map; in vm_map_freeze() local
22085 vm_map_lock(map); in vm_map_freeze()
22124 for (entry2 = vm_map_first_entry(map); in vm_map_freeze()
22125 entry2 != vm_map_to_entry(map); in vm_map_freeze()
22288 vm_map_unlock(map); in vm_map_freeze()
22393 vm_map_t map, in vm_map_range_overflows() argument
22404 pgmask = vm_map_page_mask(map); in vm_map_range_overflows()
22418 vm_map_t, map, in vm_map_range_overflows()
22453 vm_map_t map) in vm_map_page_shift() argument
22455 return VM_MAP_PAGE_SHIFT(map); in vm_map_page_shift()
22460 vm_map_t map) in vm_map_page_size() argument
22462 return VM_MAP_PAGE_SIZE(map); in vm_map_page_size()
22467 vm_map_t map) in vm_map_page_mask() argument
22469 return VM_MAP_PAGE_MASK(map); in vm_map_page_mask()
22474 vm_map_t map, in vm_map_set_page_shift() argument
22477 if (map->hdr.nentries != 0) { in vm_map_set_page_shift()
22482 map->hdr.page_shift = (uint16_t)pageshift; in vm_map_set_page_shift()
22489 vm_map_t map, in vm_map_query_volatile() argument
22513 for (entry = vm_map_first_entry(map); in vm_map_query_volatile()
22514 entry != vm_map_to_entry(map); in vm_map_query_volatile()
22557 pmap_query_resident(map->pmap, in vm_map_query_volatile()
22578 vm_map_sizes(vm_map_t map, in vm_map_sizes() argument
22588 if (!map) { in vm_map_sizes()
22594 vm_map_lock_read(map); in vm_map_sizes()
22596 *psize = map->max_offset - map->min_offset; in vm_map_sizes()
22599 prev = map->min_offset; in vm_map_sizes()
22600 for (entry = vm_map_first_entry(map);; entry = entry->vme_next) { in vm_map_sizes()
22601 end = (entry == vm_map_to_entry(map)); in vm_map_sizes()
22619 vm_map_unlock_read(map); in vm_map_sizes()
22631 vm_map_t map) in vm_map_shadow_max() argument
22637 if (map == NULL) { in vm_map_shadow_max()
22643 vm_map_lock_read(map); in vm_map_shadow_max()
22645 for (entry = vm_map_first_entry(map); in vm_map_shadow_max()
22646 entry != vm_map_to_entry(map); in vm_map_shadow_max()
22669 vm_map_unlock_read(map); in vm_map_shadow_max()
22684 vm_map_t map, in vm_map_set_high_start() argument
22687 map->vmmap_high_start = high_start; in vm_map_set_high_start()
22695 vm_map_t map, in vm_map_entry_cs_associate() argument
22705 if (map->pmap == NULL || in vm_map_entry_cs_associate()
22707 (csm_address_space_exempt(map->pmap) == KERN_SUCCESS) || in vm_map_entry_cs_associate()
22738 vm_map_lock_assert_exclusive(map); in vm_map_entry_cs_associate()
22766 map->pmap, in vm_map_entry_cs_associate()
22789 map, entry, in vm_map_entry_cs_associate()
22800 map->pmap, in vm_map_entry_cs_associate()
22860 cs_ret = cs_associate_blob_with_mapping(map->pmap, in vm_map_entry_cs_associate()
22931 vm_map_is_corpse_source(vm_map_t map) in vm_map_is_corpse_source() argument
22934 if (map) { in vm_map_is_corpse_source()
22935 vm_map_lock_read(map); in vm_map_is_corpse_source()
22936 status = map->corpse_source; in vm_map_is_corpse_source()
22937 vm_map_unlock_read(map); in vm_map_is_corpse_source()
22943 vm_map_set_corpse_source(vm_map_t map) in vm_map_set_corpse_source() argument
22945 if (map) { in vm_map_set_corpse_source()
22946 vm_map_lock(map); in vm_map_set_corpse_source()
22947 map->corpse_source = true; in vm_map_set_corpse_source()
22948 vm_map_unlock(map); in vm_map_set_corpse_source()
22953 vm_map_unset_corpse_source(vm_map_t map) in vm_map_unset_corpse_source() argument
22955 if (map) { in vm_map_unset_corpse_source()
22956 vm_map_lock(map); in vm_map_unset_corpse_source()
22957 map->corpse_source = false; in vm_map_unset_corpse_source()
22958 vm_map_unlock(map); in vm_map_unset_corpse_source()
23517 vm_map_t map, in vm_map_corpse_footprint_query_page_info() argument
23530 if (!map->has_corpse_footprint) { in vm_map_corpse_footprint_query_page_info()
23536 footprint_header = map->vmmap_corpse_footprint; in vm_map_corpse_footprint_query_page_info()
23547 effective_page_size = MIN(PAGE_SIZE, VM_MAP_PAGE_SIZE(map)); in vm_map_corpse_footprint_query_page_info()
23614 vm_map_t, map, in vm_map_corpse_footprint_query_page_info()
23624 vm_map_t map) in vm_map_corpse_footprint_destroy() argument
23626 if (map->has_corpse_footprint && in vm_map_corpse_footprint_destroy()
23627 map->vmmap_corpse_footprint != 0) { in vm_map_corpse_footprint_destroy()
23632 footprint_header = map->vmmap_corpse_footprint; in vm_map_corpse_footprint_destroy()
23635 vm_sanitize_wrap_addr((vm_offset_t) map->vmmap_corpse_footprint), in vm_map_corpse_footprint_destroy()
23638 map->vmmap_corpse_footprint = 0; in vm_map_corpse_footprint_destroy()
23639 map->has_corpse_footprint = FALSE; in vm_map_corpse_footprint_destroy()
23688 assert(new_task->map->has_corpse_footprint); in vm_map_copy_ledger()
23689 if (!new_task->map->has_corpse_footprint) { in vm_map_copy_ledger()
23726 vm_map_get_pmap(vm_map_t map) in vm_map_get_pmap() argument
23728 return vm_map_pmap(map); in vm_map_get_pmap()
23733 vm_map_t map, in vm_map_get_phys_page() argument
23744 vm_map_lock(map); in vm_map_get_phys_page()
23745 while (vm_map_lookup_entry(map, map_offset, &entry)) { in vm_map_get_phys_page()
23749 old_map = map; in vm_map_get_phys_page()
23750 map = VME_SUBMAP(entry); in vm_map_get_phys_page()
23757 vm_map_unlock(map); in vm_map_get_phys_page()
23767 vm_map_unlock(map); in vm_map_get_phys_page()
23768 vm_fault(map, map_offset, VM_PROT_NONE, in vm_map_get_phys_page()
23771 vm_map_lock(map); in vm_map_get_phys_page()
23807 vm_map_unlock(map); in vm_map_get_phys_page()
23945 vm_map_range_configure(vm_map_t map, __unused bool needs_extra_jumbo_va) in vm_map_range_configure() argument
23952 if (!vm_map_is_64bit(map) || vm_map_is_exotic(map) || offmask == 0) { in vm_map_range_configure()
23960 assert(vm_map_pmap(map) != kernel_pmap); in vm_map_range_configure()
24058 vm_map_lock_read(map); in vm_map_range_configure()
24059 default_end = vm_map_max(map); in vm_map_range_configure()
24060 vm_map_unlock_read(map); in vm_map_range_configure()
24085 vm_map_set_jumbo(map); in vm_map_range_configure()
24087 assert3u(7 * GiB(10) / 2, <=, vm_map_max(map) - default_end); in vm_map_range_configure()
24089 default_end + PAGE_SIZE, vm_map_max(map), offmask); in vm_map_range_configure()
24099 kr = vm_map_enter(map, &default_end, in vm_map_range_configure()
24106 if (data_range.max_address != vm_map_max(map)) { in vm_map_range_configure()
24114 vm_map_lock_read(map); in vm_map_range_configure()
24115 vm_map_lookup_entry_or_next(map, data_range.max_address, &entry); in vm_map_range_configure()
24116 if (entry == vm_map_to_entry(map) || entry->vme_start > vm_map_max(map)) { in vm_map_range_configure()
24117 size = vm_map_max(map) - data_range.max_address; in vm_map_range_configure()
24121 vm_map_unlock_read(map); in vm_map_range_configure()
24123 kr = vm_map_enter(map, &data_range.max_address, size, in vm_map_range_configure()
24132 vm_map_set_extra_jumbo(map); in vm_map_range_configure()
24136 vm_map_lock(map); in vm_map_range_configure()
24137 map->default_range.min_address = vm_map_min(map); in vm_map_range_configure()
24138 map->default_range.max_address = default_end; in vm_map_range_configure()
24139 map->data_range = data_range; in vm_map_range_configure()
24143 map->large_file_range = vm_map_range_random_uniform(TiB(1), in vm_map_range_configure()
24147 map->uses_user_ranges = true; in vm_map_range_configure()
24148 vm_map_unlock(map); in vm_map_range_configure()
24193 vm_map_t map, in vm_map_get_user_range() argument
24197 if (map == NULL || !map->uses_user_ranges || range == NULL) { in vm_map_get_user_range()
24203 *range = map->default_range; in vm_map_get_user_range()
24207 *range = map->data_range; in vm_map_get_user_range()
24219 if (map->large_file_range.min_address != map->large_file_range.max_address) { in vm_map_get_user_range()
24221 *range = map->large_file_range; in vm_map_get_user_range()
24234 vm_map_t map, in vm_map_user_range_resolve() argument
24241 vm_map_lock_assert_held(map); in vm_map_user_range_resolve()
24246 if (mach_vm_range_contains(&map->default_range, addr, size)) { in vm_map_user_range_resolve()
24248 *range = map->default_range; in vm_map_user_range_resolve()
24253 if (mach_vm_range_contains(&map->data_range, addr, size)) { in vm_map_user_range_resolve()
24255 *range = map->data_range; in vm_map_user_range_resolve()
24260 if (mach_vm_range_contains(&map->large_file_range, addr, size)) { in vm_map_user_range_resolve()
24262 *range = map->large_file_range; in vm_map_user_range_resolve()
24267 for (size_t i = 0; i < map->extra_ranges_count; i++) { in vm_map_user_range_resolve()
24268 vm_map_user_range_t r = &map->extra_ranges[i]; in vm_map_user_range_resolve()
24291 vm_map_t map, in vm_map_kernel_flags_update_range_id() argument
24294 if (map == kernel_map) { in vm_map_kernel_flags_update_range_id()
24321 vm_map_entry_has_device_pager(vm_map_t map, vm_map_offset_t vaddr) in vm_map_entry_has_device_pager() argument
24327 if (map == NULL) { in vm_map_entry_has_device_pager()
24331 vm_map_lock(map); in vm_map_entry_has_device_pager()
24333 if (!vm_map_lookup_entry(map, vaddr, &entry)) { in vm_map_entry_has_device_pager()
24342 vm_map_unlock(map); in vm_map_entry_has_device_pager()
24343 map = submap; in vm_map_entry_has_device_pager()
24355 vm_map_unlock(map); in vm_map_entry_has_device_pager()
24510 vm_map_t map, in vm_map_pmap_set_process() argument
24514 pmap_set_process(vm_map_pmap(map), pid, procname); in vm_map_pmap_set_process()