Lines Matching refs:map
131 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
134 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
135 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
136 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
138 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
143 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
146 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
168 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
170 if (start < vm_map_min(map)) \
171 start = vm_map_min(map); \
172 if (end > vm_map_max(map)) \
173 end = vm_map_max(map); \
293 vm_map_t map; in vmspace_zinit() local
296 map = &vm->vm_map; in vmspace_zinit()
298 memset(map, 0, sizeof(*map)); in vmspace_zinit()
299 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vmspace_zinit()
301 sx_init(&map->lock, "vm map (user)"); in vmspace_zinit()
528 _vm_map_lock(vm_map_t map, const char *file, int line) in _vm_map_lock() argument
531 if (map->system_map) in _vm_map_lock()
532 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock()
534 sx_xlock_(&map->lock, file, line); in _vm_map_lock()
535 map->timestamp++; in _vm_map_lock()
619 _vm_map_assert_locked(vm_map_t map, const char *file, int line) in _vm_map_assert_locked() argument
622 if (map->system_map) in _vm_map_assert_locked()
623 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_assert_locked()
625 sx_assert_(&map->lock, SA_XLOCKED, file, line); in _vm_map_assert_locked()
628 #define VM_MAP_ASSERT_LOCKED(map) \ argument
629 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
640 static void _vm_map_assert_consistent(vm_map_t map, int check);
642 #define VM_MAP_ASSERT_CONSISTENT(map) \ argument
643 _vm_map_assert_consistent(map, VMMAP_CHECK_ALL)
645 #define VM_MAP_UNLOCK_CONSISTENT(map) do { \ argument
646 if (map->nupdates > map->nentries) { \
647 _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \
648 map->nupdates = 0; \
652 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
655 #define VM_MAP_ASSERT_LOCKED(map) argument
656 #define VM_MAP_ASSERT_CONSISTENT(map) argument
657 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
661 _vm_map_unlock(vm_map_t map, const char *file, int line) in _vm_map_unlock() argument
664 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock()
665 if (map->system_map) { in _vm_map_unlock()
667 if (map == kernel_map && (map->flags & MAP_REPLENISH) != 0) { in _vm_map_unlock()
669 map->flags &= ~MAP_REPLENISH; in _vm_map_unlock()
672 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock()
674 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock()
680 _vm_map_lock_read(vm_map_t map, const char *file, int line) in _vm_map_lock_read() argument
683 if (map->system_map) in _vm_map_lock_read()
684 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock_read()
686 sx_slock_(&map->lock, file, line); in _vm_map_lock_read()
690 _vm_map_unlock_read(vm_map_t map, const char *file, int line) in _vm_map_unlock_read() argument
693 if (map->system_map) { in _vm_map_unlock_read()
694 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_read()
696 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_read()
698 sx_sunlock_(&map->lock, file, line); in _vm_map_unlock_read()
704 _vm_map_trylock(vm_map_t map, const char *file, int line) in _vm_map_trylock() argument
708 error = map->system_map ? in _vm_map_trylock()
709 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock()
710 !sx_try_xlock_(&map->lock, file, line); in _vm_map_trylock()
712 map->timestamp++; in _vm_map_trylock()
717 _vm_map_trylock_read(vm_map_t map, const char *file, int line) in _vm_map_trylock_read() argument
721 error = map->system_map ? in _vm_map_trylock_read()
722 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock_read()
723 !sx_try_slock_(&map->lock, file, line); in _vm_map_trylock_read()
738 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_upgrade() argument
742 if (map->system_map) { in _vm_map_lock_upgrade()
743 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_upgrade()
745 if (!sx_try_upgrade_(&map->lock, file, line)) { in _vm_map_lock_upgrade()
746 last_timestamp = map->timestamp; in _vm_map_lock_upgrade()
747 sx_sunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
753 sx_xlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
754 if (last_timestamp != map->timestamp) { in _vm_map_lock_upgrade()
755 sx_xunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
760 map->timestamp++; in _vm_map_lock_upgrade()
765 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_downgrade() argument
768 if (map->system_map) { in _vm_map_lock_downgrade()
769 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_lock_downgrade()
771 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_downgrade()
773 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_lock_downgrade()
774 sx_downgrade_(&map->lock, file, line); in _vm_map_lock_downgrade()
785 vm_map_locked(vm_map_t map) in vm_map_locked() argument
788 if (map->system_map) in vm_map_locked()
789 return (mtx_owned(&map->system_mtx)); in vm_map_locked()
791 return (sx_xlocked(&map->lock)); in vm_map_locked()
809 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) in _vm_map_unlock_and_wait() argument
812 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock_and_wait()
814 if (map->system_map) { in _vm_map_unlock_and_wait()
815 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_and_wait()
817 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_and_wait()
819 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock_and_wait()
821 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", in _vm_map_unlock_and_wait()
832 vm_map_wakeup(vm_map_t map) in vm_map_wakeup() argument
842 wakeup(&map->root); in vm_map_wakeup()
846 vm_map_busy(vm_map_t map) in vm_map_busy() argument
849 VM_MAP_ASSERT_LOCKED(map); in vm_map_busy()
850 map->busy++; in vm_map_busy()
854 vm_map_unbusy(vm_map_t map) in vm_map_unbusy() argument
857 VM_MAP_ASSERT_LOCKED(map); in vm_map_unbusy()
858 KASSERT(map->busy, ("vm_map_unbusy: not busy")); in vm_map_unbusy()
859 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { in vm_map_unbusy()
860 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); in vm_map_unbusy()
861 wakeup(&map->busy); in vm_map_unbusy()
866 vm_map_wait_busy(vm_map_t map) in vm_map_wait_busy() argument
869 VM_MAP_ASSERT_LOCKED(map); in vm_map_wait_busy()
870 while (map->busy) { in vm_map_wait_busy()
871 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); in vm_map_wait_busy()
872 if (map->system_map) in vm_map_wait_busy()
873 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); in vm_map_wait_busy()
875 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); in vm_map_wait_busy()
877 map->timestamp++; in vm_map_wait_busy()
891 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in _vm_map_init() argument
894 map->header.eflags = MAP_ENTRY_HEADER; in _vm_map_init()
895 map->needs_wakeup = FALSE; in _vm_map_init()
896 map->system_map = 0; in _vm_map_init()
897 map->pmap = pmap; in _vm_map_init()
898 map->header.end = min; in _vm_map_init()
899 map->header.start = max; in _vm_map_init()
900 map->flags = 0; in _vm_map_init()
901 map->header.left = map->header.right = &map->header; in _vm_map_init()
902 map->root = NULL; in _vm_map_init()
903 map->timestamp = 0; in _vm_map_init()
904 map->busy = 0; in _vm_map_init()
905 map->anon_loc = 0; in _vm_map_init()
907 map->nupdates = 0; in _vm_map_init()
912 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init() argument
915 _vm_map_init(map, pmap, min, max); in vm_map_init()
916 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vm_map_init()
918 sx_init(&map->lock, "vm map (user)"); in vm_map_init()
927 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
929 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
939 vm_map_entry_create(vm_map_t map) in vm_map_entry_create() argument
944 if (map == kernel_map) { in vm_map_entry_create()
945 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_create()
963 if (map->system_map) { in vm_map_entry_create()
1130 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, in vm_map_splay_split() argument
1135 left = right = &map->header; in vm_map_splay_split()
1136 root = map->root; in vm_map_splay_split()
1339 vm_map_splay(vm_map_t map, vm_offset_t addr) in vm_map_splay() argument
1344 header = &map->header; in vm_map_splay()
1345 root = vm_map_splay_split(map, addr, 0, &llist, &rlist); in vm_map_splay()
1372 map->root = root; in vm_map_splay()
1373 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_splay()
1387 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1393 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1394 map->nentries, entry); in vm_map_entry_link()
1395 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_link()
1396 map->nentries++; in vm_map_entry_link()
1397 header = &map->header; in vm_map_entry_link()
1398 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1444 map->root = entry; in vm_map_entry_link()
1445 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_link()
1454 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1460 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unlink()
1461 header = &map->header; in vm_map_entry_unlink()
1462 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1490 map->root = root; in vm_map_entry_unlink()
1491 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_unlink()
1492 map->nentries--; in vm_map_entry_unlink()
1493 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1494 map->nentries, entry); in vm_map_entry_unlink()
1506 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) in vm_map_entry_resize() argument
1510 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_resize()
1511 header = &map->header; in vm_map_entry_resize()
1512 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize()
1519 map->root = root; in vm_map_entry_resize()
1520 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_resize()
1522 __func__, map, map->nentries, entry); in vm_map_entry_resize()
1537 vm_map_t map, in vm_map_lookup_entry() argument
1548 header = &map->header; in vm_map_lookup_entry()
1549 cur = map->root; in vm_map_lookup_entry()
1558 if ((locked = vm_map_locked(map)) || in vm_map_lookup_entry()
1559 sx_try_upgrade(&map->lock)) { in vm_map_lookup_entry()
1566 cur = vm_map_splay(map, address); in vm_map_lookup_entry()
1568 VM_MAP_UNLOCK_CONSISTENT(map); in vm_map_lookup_entry()
1569 sx_downgrade(&map->lock); in vm_map_lookup_entry()
1617 vm_map_insert1(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert1() argument
1628 VM_MAP_ASSERT_LOCKED(map); in vm_map_insert1()
1642 if (start == end || !vm_map_range_valid(map, start, end)) in vm_map_insert1()
1645 if ((map->flags & MAP_WXORX) != 0 && (prot & (VM_PROT_WRITE | in vm_map_insert1()
1653 if (vm_map_lookup_entry(map, start, &prev_entry)) in vm_map_insert1()
1722 if (map == kernel_map && end > kernel_vm_end) in vm_map_insert1()
1763 map->size += end - prev_entry->end; in vm_map_insert1()
1764 vm_map_entry_resize(map, prev_entry, in vm_map_insert1()
1766 *res = vm_map_try_merge_entries(map, prev_entry, in vm_map_insert1()
1793 new_entry = vm_map_entry_create(map); in vm_map_insert1()
1817 vm_map_entry_link(map, new_entry); in vm_map_insert1()
1819 map->size += new_entry->end - new_entry->start; in vm_map_insert1()
1827 vm_map_try_merge_entries(map, prev_entry, new_entry); in vm_map_insert1()
1828 *res = vm_map_try_merge_entries(map, new_entry, next_entry); in vm_map_insert1()
1831 vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), in vm_map_insert1()
1850 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert() argument
1855 return (vm_map_insert1(map, object, offset, start, end, prot, max, in vm_map_insert()
1877 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) in vm_map_findspace() argument
1883 VM_MAP_ASSERT_LOCKED(map); in vm_map_findspace()
1889 start = MAX(start, vm_map_min(map)); in vm_map_findspace()
1890 if (start >= vm_map_max(map) || length > vm_map_max(map) - start) in vm_map_findspace()
1891 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1894 if (map->root == NULL) in vm_map_findspace()
1904 header = &map->header; in vm_map_findspace()
1905 root = vm_map_splay_split(map, start, length, &llist, &rlist); in vm_map_findspace()
1925 map->root = root; in vm_map_findspace()
1926 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1932 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1963 map->root = root; in vm_map_findspace()
1964 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1969 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_fixed() argument
1980 vm_map_lock(map); in vm_map_fixed()
1981 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_fixed()
1983 result = vm_map_delete(map, start, end); in vm_map_fixed()
1988 result = vm_map_stack_locked(map, start, length, sgrowsiz, in vm_map_fixed()
1991 result = vm_map_insert(map, object, offset, start, end, in vm_map_fixed()
1995 vm_map_unlock(map); in vm_map_fixed()
2040 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_alignspace() argument
2046 VM_MAP_ASSERT_LOCKED(map); in vm_map_alignspace()
2048 KASSERT(free_addr == vm_map_findspace(map, free_addr, length), in vm_map_alignspace()
2076 *addr = vm_map_findspace(map, aligned_addr, length); in vm_map_alignspace()
2077 if (*addr + length > vm_map_max(map) || in vm_map_alignspace()
2093 vm_map_find_aligned(vm_map_t map, vm_offset_t *addr, vm_size_t length, in vm_map_find_aligned() argument
2097 *addr = vm_map_findspace(map, *addr, length); in vm_map_find_aligned()
2098 if (*addr + length > vm_map_max(map) || in vm_map_find_aligned()
2101 return (vm_map_alignspace(map, NULL, 0, addr, length, max_addr, in vm_map_find_aligned()
2115 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find() argument
2122 vm_map_lock(map); in vm_map_find()
2123 rv = vm_map_find_locked(map, object, offset, addr, length, max_addr, in vm_map_find()
2125 vm_map_unlock(map); in vm_map_find()
2130 vm_map_find_locked(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_locked() argument
2152 en_aslr = (map->flags & MAP_ASLR) != 0; in vm_map_find_locked()
2154 (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && in vm_map_find_locked()
2161 (map->flags & MAP_ASLR_IGNSTART) != 0) in vm_map_find_locked()
2162 curr_min_addr = min_addr = vm_map_min(map); in vm_map_find_locked()
2165 curr_min_addr = map->anon_loc; in vm_map_find_locked()
2200 curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? in vm_map_find_locked()
2201 vm_map_min(map) : min_addr; in vm_map_find_locked()
2213 gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && in vm_map_find_locked()
2216 *addr = vm_map_findspace(map, curr_min_addr, in vm_map_find_locked()
2219 vm_map_max(map)) in vm_map_find_locked()
2226 *addr = vm_map_findspace(map, curr_min_addr, length); in vm_map_find_locked()
2227 if (*addr + length > vm_map_max(map) || in vm_map_find_locked()
2239 (rv = vm_map_alignspace(map, object, offset, addr, length, in vm_map_find_locked()
2251 if (!vm_map_range_valid(map, *addr, *addr + length)) in vm_map_find_locked()
2253 rv = vm_map_delete(map, *addr, *addr + length); in vm_map_find_locked()
2258 rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, in vm_map_find_locked()
2261 rv = vm_map_insert(map, object, offset, *addr, *addr + length, in vm_map_find_locked()
2270 if (update_anon && rv == KERN_SUCCESS && (map->anon_loc == 0 || in vm_map_find_locked()
2271 *addr < map->anon_loc)) in vm_map_find_locked()
2272 map->anon_loc = *addr; in vm_map_find_locked()
2289 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_min() argument
2303 rv = vm_map_find(map, object, offset, addr, length, max_addr, in vm_map_find_min()
2340 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2357 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2371 vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev_entry, in vm_map_try_merge_entries() argument
2375 VM_MAP_ASSERT_LOCKED(map); in vm_map_try_merge_entries()
2378 vm_map_entry_unlink(map, prev_entry, UNLINK_MERGE_NEXT); in vm_map_try_merge_entries()
2379 vm_map_merged_neighbor_dispose(map, prev_entry); in vm_map_try_merge_entries()
2413 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_charge_object() argument
2416 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_charge_object()
2419 if (entry->object.vm_object == NULL && !map->system_map && in vm_map_entry_charge_object()
2441 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_clone() argument
2445 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_clone()
2451 vm_map_entry_charge_object(map, entry); in vm_map_entry_clone()
2454 new_entry = vm_map_entry_create(map); in vm_map_entry_clone()
2479 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) in vm_map_clip_start() argument
2484 if (!map->system_map) in vm_map_clip_start()
2486 "%s: map %p entry %p start 0x%jx", __func__, map, entry, in vm_map_clip_start()
2492 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_start()
2502 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_start()
2509 vm_map_entry_link(map, new_entry); in vm_map_clip_start()
2521 vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, in vm_map_lookup_clip_start() argument
2527 if (!map->system_map) in vm_map_lookup_clip_start()
2529 "%s: map %p start 0x%jx prev %p", __func__, map, in vm_map_lookup_clip_start()
2532 if (vm_map_lookup_entry(map, start, prev_entry)) { in vm_map_lookup_clip_start()
2534 rv = vm_map_clip_start(map, entry, start); in vm_map_lookup_clip_start()
2552 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) in vm_map_clip_end() argument
2557 if (!map->system_map) in vm_map_clip_end()
2559 "%s: map %p entry %p end 0x%jx", __func__, map, entry, in vm_map_clip_end()
2565 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_end()
2575 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_end()
2582 vm_map_entry_link(map, new_entry); in vm_map_clip_end()
2607 vm_map_t map, in vm_map_submap() argument
2621 vm_map_lock(map); in vm_map_submap()
2622 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_submap()
2623 if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && in vm_map_submap()
2626 result = vm_map_clip_start(map, entry, start); in vm_map_submap()
2629 result = vm_map_clip_end(map, entry, end); in vm_map_submap()
2637 vm_map_unlock(map); in vm_map_submap()
2665 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, in vm_map_pmap_enter() argument
2677 pmap_object_init_pt(map->pmap, addr, object, pindex, in vm_map_pmap_enter()
2735 pmap_enter_object(map->pmap, start, addr + in vm_map_pmap_enter()
2741 pmap_enter_object(map->pmap, start, addr + ptoa(psize), in vm_map_pmap_enter()
2775 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_protect() argument
2797 vm_map_lock(map); in vm_map_protect()
2799 if ((map->flags & MAP_WXORX) != 0 && in vm_map_protect()
2802 vm_map_unlock(map); in vm_map_protect()
2812 vm_map_wait_busy(map); in vm_map_protect()
2814 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_protect()
2816 if (!vm_map_lookup_entry(map, start, &first_entry)) in vm_map_protect()
2830 first_entry != vm_map_entry_first(map)) in vm_map_protect()
2846 vm_map_unlock(map); in vm_map_protect()
2857 vm_map_unlock(map); in vm_map_protect()
2873 vm_map_unlock_and_wait(map, 0); in vm_map_protect()
2884 rv = vm_map_clip_start(map, first_entry, start); in vm_map_protect()
2886 vm_map_unlock(map); in vm_map_protect()
2891 rv = vm_map_clip_end(map, entry, end); in vm_map_protect()
2893 vm_map_unlock(map); in vm_map_protect()
2952 vm_map_try_merge_entries(map, prev_entry, entry), in vm_map_protect()
2981 vm_fault_copy_entry(map, map, entry, entry, NULL); in vm_map_protect()
2990 pmap_protect(map->pmap, entry->start, in vm_map_protect()
2996 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_protect()
2997 vm_map_unlock(map); in vm_map_protect()
3011 vm_map_t map, in vm_map_madvise() argument
3037 vm_map_lock(map); in vm_map_madvise()
3045 vm_map_lock_read(map); in vm_map_madvise()
3054 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_madvise()
3063 rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); in vm_map_madvise()
3065 vm_map_unlock(map); in vm_map_madvise()
3074 rv = vm_map_clip_end(map, entry, end); in vm_map_madvise()
3076 vm_map_unlock(map); in vm_map_madvise()
3108 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3110 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3111 vm_map_unlock(map); in vm_map_madvise()
3122 if (!vm_map_lookup_entry(map, start, &entry)) in vm_map_madvise()
3172 pmap_advise(map->pmap, useStart, useEnd, in vm_map_madvise()
3185 vm_map_pmap_enter(map, in vm_map_madvise()
3195 vm_map_unlock_read(map); in vm_map_madvise()
3209 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_inherit() argument
3226 vm_map_lock(map); in vm_map_inherit()
3227 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
3228 rv = vm_map_lookup_clip_start(map, start, &start_entry, &prev_entry); in vm_map_inherit()
3231 if (vm_map_lookup_entry(map, end - 1, &lentry)) { in vm_map_inherit()
3232 rv = vm_map_clip_end(map, lentry, end); in vm_map_inherit()
3253 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3255 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3257 vm_map_unlock(map); in vm_map_inherit()
3270 vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start, in vm_map_entry_in_transition() argument
3277 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_in_transition()
3285 last_timestamp = map->timestamp; in vm_map_entry_in_transition()
3286 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_entry_in_transition()
3291 vm_map_lock(map); in vm_map_entry_in_transition()
3292 if (last_timestamp + 1 == map->timestamp) in vm_map_entry_in_transition()
3300 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_entry_in_transition()
3316 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_unwire() argument
3327 vm_map_lock(map); in vm_map_unwire()
3328 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire()
3329 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3333 vm_map_unlock(map); in vm_map_unwire()
3343 next_entry = vm_map_entry_in_transition(map, start, in vm_map_unwire()
3347 vm_map_unlock(map); in vm_map_unwire()
3357 rv = vm_map_clip_start(map, entry, start); in vm_map_unwire()
3360 rv = vm_map_clip_end(map, entry, end); in vm_map_unwire()
3396 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3425 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3441 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3443 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3444 vm_map_unlock(map); in vm_map_unwire()
3446 vm_map_wakeup(map); in vm_map_unwire()
3480 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3484 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_entry_failure()
3496 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3509 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire() argument
3513 vm_map_lock(map); in vm_map_wire()
3514 rv = vm_map_wire_locked(map, start, end, flags); in vm_map_wire()
3515 vm_map_unlock(map); in vm_map_wire()
3526 vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire_locked() argument
3536 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_locked()
3545 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire_locked()
3546 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3557 next_entry = vm_map_entry_in_transition(map, start, in vm_map_wire_locked()
3569 rv = vm_map_clip_start(map, entry, start); in vm_map_wire_locked()
3572 rv = vm_map_clip_end(map, entry, end); in vm_map_wire_locked()
3598 vm_map_wire_entry_failure(map, entry, in vm_map_wire_locked()
3611 last_timestamp = map->timestamp; in vm_map_wire_locked()
3614 vm_map_busy(map); in vm_map_wire_locked()
3615 vm_map_unlock(map); in vm_map_wire_locked()
3623 rv = vm_fault(map, faddr, VM_PROT_NONE, in vm_map_wire_locked()
3628 vm_map_lock(map); in vm_map_wire_locked()
3629 vm_map_unbusy(map); in vm_map_wire_locked()
3630 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire_locked()
3637 if (!vm_map_lookup_entry(map, saved_start, in vm_map_wire_locked()
3653 vm_map_wire_entry_failure(map, in vm_map_wire_locked()
3658 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire_locked()
3684 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3732 vm_map_entry_unwire(map, entry); in vm_map_wire_locked()
3750 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3752 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3754 vm_map_wakeup(map); in vm_map_wire_locked()
3776 vm_map_t map, in vm_map_sync() argument
3790 vm_map_lock_read(map); in vm_map_sync()
3791 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_sync()
3792 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_sync()
3793 vm_map_unlock_read(map); in vm_map_sync()
3807 vm_map_unlock_read(map); in vm_map_sync()
3814 vm_map_unlock_read(map); in vm_map_sync()
3821 vm_map_unlock_read(map); in vm_map_sync()
3827 pmap_remove(map->pmap, start, end); in vm_map_sync()
3855 last_timestamp = map->timestamp; in vm_map_sync()
3856 vm_map_unlock_read(map); in vm_map_sync()
3861 vm_map_lock_read(map); in vm_map_sync()
3862 if (last_timestamp == map->timestamp || in vm_map_sync()
3863 !vm_map_lookup_entry(map, start, &entry)) in vm_map_sync()
3867 vm_map_unlock_read(map); in vm_map_sync()
3880 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3884 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unwire()
3891 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3912 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3918 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3925 vm_map_entry_deallocate(entry, map->system_map); in vm_map_entry_delete()
3930 map->size -= size; in vm_map_entry_delete()
3976 if (map->system_map) in vm_map_entry_delete()
3991 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_delete() argument
3996 VM_MAP_ASSERT_LOCKED(map); in vm_map_delete()
4005 rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); in vm_map_delete()
4015 (vm_map_pmap(map) != kernel_pmap && in vm_map_delete()
4022 last_timestamp = map->timestamp; in vm_map_delete()
4023 (void) vm_map_unlock_and_wait(map, 0); in vm_map_delete()
4024 vm_map_lock(map); in vm_map_delete()
4025 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
4032 rv = vm_map_lookup_clip_start(map, saved_start, in vm_map_delete()
4042 rv = vm_map_clip_end(map, entry, end); in vm_map_delete()
4052 vm_map_entry_unwire(map, entry); in vm_map_delete()
4061 pmap_map_delete(map->pmap, entry->start, entry->end); in vm_map_delete()
4063 if (entry->end == map->anon_loc) in vm_map_delete()
4064 map->anon_loc = entry->start; in vm_map_delete()
4072 vm_map_entry_delete(map, entry); in vm_map_delete()
4084 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_remove() argument
4088 vm_map_lock(map); in vm_map_remove()
4089 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_remove()
4090 result = vm_map_delete(map, start, end); in vm_map_remove()
4091 vm_map_unlock(map); in vm_map_remove()
4110 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_check_protection() argument
4116 if (!vm_map_lookup_entry(map, start, &tmp_entry)) in vm_map_check_protection()
4554 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack() argument
4561 MPASS((map->flags & MAP_WIREFUTURE) == 0); in vm_map_stack()
4564 vm_map_lock(map); in vm_map_stack()
4567 if (map->size + init_ssize > vmemlim) { in vm_map_stack()
4571 rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, in vm_map_stack()
4574 vm_map_unlock(map); in vm_map_stack()
4584 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack_locked() argument
4603 !vm_map_range_valid(map, addrbos, addrbos + max_ssize)) in vm_map_stack_locked()
4616 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) in vm_map_stack_locked()
4646 rv = vm_map_insert1(map, NULL, 0, bot, top, prot, max, cow, in vm_map_stack_locked()
4660 rv = vm_map_insert1(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, in vm_map_stack_locked()
4683 (void)vm_map_delete(map, bot, top); in vm_map_stack_locked()
4693 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) in vm_map_growstack() argument
4720 if (p != initproc && (map != &p->p_vmspace->vm_map || in vm_map_growstack()
4724 MPASS(!map->system_map); in vm_map_growstack()
4731 if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) in vm_map_growstack()
4799 if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4800 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { in vm_map_growstack()
4808 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { in vm_map_growstack()
4819 if (map->size + grow_amount > vmemlim) { in vm_map_growstack()
4826 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { in vm_map_growstack()
4835 if (vm_map_lock_upgrade(map)) { in vm_map_growstack()
4837 vm_map_lock_read(map); in vm_map_growstack()
4854 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4858 vm_map_entry_resize(map, gap_entry, -grow_amount); in vm_map_growstack()
4861 rv = vm_map_insert(map, NULL, 0, grow_start, in vm_map_growstack()
4865 rv1 = vm_map_insert1(map, NULL, 0, gap_start, in vm_map_growstack()
4873 vm_map_entry_resize(map, gap_entry, in vm_map_growstack()
4890 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4891 vm_map_entry_resize(map, stack_entry, in vm_map_growstack()
4897 map->size += grow_amount; in vm_map_growstack()
4908 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4909 rv = vm_map_wire_locked(map, grow_start, in vm_map_growstack()
4913 vm_map_lock_downgrade(map); in vm_map_growstack()
4919 error = racct_set(p, RACCT_VMEM, map->size); in vm_map_growstack()
4923 ptoa(pmap_wired_count(map->pmap))); in vm_map_growstack()
5034 vm_map_t map = *var_map; in vm_map_lookup() local
5043 vm_map_lock_read(map); in vm_map_lookup()
5049 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { in vm_map_lookup()
5050 vm_map_unlock_read(map); in vm_map_lookup()
5060 vm_map_t old_map = map; in vm_map_lookup()
5062 *var_map = map = entry->object.sub_map; in vm_map_lookup()
5073 if (prot == VM_PROT_NONE && map != kernel_map && in vm_map_lookup()
5077 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
5082 vm_map_unlock_read(map); in vm_map_lookup()
5092 vm_map_unlock_read(map); in vm_map_lookup()
5124 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5136 vm_map_unlock(map); in vm_map_lookup()
5154 vm_map_lock_downgrade(map); in vm_map_lookup()
5167 if (entry->object.vm_object == NULL && !map->system_map) { in vm_map_lookup()
5168 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5174 vm_map_lock_downgrade(map); in vm_map_lookup()
5205 vm_map_t map = *var_map; in vm_map_lookup_locked() local
5212 if (!vm_map_lookup_entry(map, vaddr, out_entry)) in vm_map_lookup_locked()
5255 if (entry->object.vm_object == NULL && !map->system_map) in vm_map_lookup_locked()
5276 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
5281 vm_map_unlock_read(map); in vm_map_lookup_done()
5285 vm_map_max_KBI(const struct vm_map *map) in vm_map_max_KBI() argument
5288 return (vm_map_max(map)); in vm_map_max_KBI()
5292 vm_map_min_KBI(const struct vm_map *map) in vm_map_min_KBI() argument
5295 return (vm_map_min(map)); in vm_map_min_KBI()
5299 vm_map_pmap_KBI(vm_map_t map) in vm_map_pmap_KBI() argument
5302 return (map->pmap); in vm_map_pmap_KBI()
5306 vm_map_range_valid_KBI(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_range_valid_KBI() argument
5309 return (vm_map_range_valid(map, start, end)); in vm_map_range_valid_KBI()
5314 _vm_map_assert_consistent(vm_map_t map, int check) in _vm_map_assert_consistent() argument
5321 ++map->nupdates; in _vm_map_assert_consistent()
5326 header = prev = &map->header; in _vm_map_assert_consistent()
5327 VM_MAP_ENTRY_FOREACH(entry, map) { in _vm_map_assert_consistent()
5329 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5332 ("map %p start = %jx, end = %jx", map, in _vm_map_assert_consistent()
5336 ("map %p left->start = %jx, start = %jx", map, in _vm_map_assert_consistent()
5340 ("map %p start = %jx, right->start = %jx", map, in _vm_map_assert_consistent()
5342 cur = map->root; in _vm_map_assert_consistent()
5350 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5356 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5360 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5367 ("map %p max = %jx, max_left = %jx, max_right = %jx", map, in _vm_map_assert_consistent()
5373 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5385 vm_map_print(vm_map_t map) in vm_map_print() argument
5390 (void *)map, in vm_map_print()
5391 (void *)map->pmap, map->nentries, map->timestamp); in vm_map_print()
5394 prev = &map->header; in vm_map_print()
5395 VM_MAP_ENTRY_FOREACH(entry, map) { in vm_map_print()
5415 if (prev == &map->header || in vm_map_print()
5437 if (prev == &map->header || in vm_map_print()
5452 DB_SHOW_COMMAND(map, map) in DB_SHOW_COMMAND() argument