Lines Matching refs:map
133 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
136 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
137 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
138 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
140 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
145 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
148 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
168 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
170 if (start < vm_map_min(map)) \
171 start = vm_map_min(map); \
172 if (end > vm_map_max(map)) \
173 end = vm_map_max(map); \
293 vm_map_t map; in vmspace_zinit() local
296 map = &vm->vm_map; in vmspace_zinit()
298 memset(map, 0, sizeof(*map)); in vmspace_zinit()
299 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vmspace_zinit()
301 sx_init(&map->lock, "vm map (user)"); in vmspace_zinit()
528 _vm_map_lock(vm_map_t map, const char *file, int line) in _vm_map_lock() argument
531 if (map->system_map) in _vm_map_lock()
532 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock()
534 sx_xlock_(&map->lock, file, line); in _vm_map_lock()
535 map->timestamp++; in _vm_map_lock()
619 _vm_map_assert_locked(vm_map_t map, const char *file, int line) in _vm_map_assert_locked() argument
622 if (map->system_map) in _vm_map_assert_locked()
623 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_assert_locked()
625 sx_assert_(&map->lock, SA_XLOCKED, file, line); in _vm_map_assert_locked()
628 #define VM_MAP_ASSERT_LOCKED(map) \ argument
629 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
640 static void _vm_map_assert_consistent(vm_map_t map, int check);
642 #define VM_MAP_ASSERT_CONSISTENT(map) \ argument
643 _vm_map_assert_consistent(map, VMMAP_CHECK_ALL)
645 #define VM_MAP_UNLOCK_CONSISTENT(map) do { \ argument
646 if (map->nupdates > map->nentries) { \
647 _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \
648 map->nupdates = 0; \
652 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
655 #define VM_MAP_ASSERT_LOCKED(map) argument
656 #define VM_MAP_ASSERT_CONSISTENT(map) argument
657 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
661 _vm_map_unlock(vm_map_t map, const char *file, int line) in _vm_map_unlock() argument
664 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock()
665 if (map->system_map) { in _vm_map_unlock()
667 if (map == kernel_map && (map->flags & MAP_REPLENISH) != 0) { in _vm_map_unlock()
669 map->flags &= ~MAP_REPLENISH; in _vm_map_unlock()
672 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock()
674 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock()
680 _vm_map_lock_read(vm_map_t map, const char *file, int line) in _vm_map_lock_read() argument
683 if (map->system_map) in _vm_map_lock_read()
684 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock_read()
686 sx_slock_(&map->lock, file, line); in _vm_map_lock_read()
690 _vm_map_unlock_read(vm_map_t map, const char *file, int line) in _vm_map_unlock_read() argument
693 if (map->system_map) { in _vm_map_unlock_read()
694 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_read()
696 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_read()
698 sx_sunlock_(&map->lock, file, line); in _vm_map_unlock_read()
704 _vm_map_trylock(vm_map_t map, const char *file, int line) in _vm_map_trylock() argument
708 error = map->system_map ? in _vm_map_trylock()
709 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock()
710 !sx_try_xlock_(&map->lock, file, line); in _vm_map_trylock()
712 map->timestamp++; in _vm_map_trylock()
717 _vm_map_trylock_read(vm_map_t map, const char *file, int line) in _vm_map_trylock_read() argument
721 error = map->system_map ? in _vm_map_trylock_read()
722 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock_read()
723 !sx_try_slock_(&map->lock, file, line); in _vm_map_trylock_read()
738 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_upgrade() argument
742 if (map->system_map) { in _vm_map_lock_upgrade()
743 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_upgrade()
745 if (!sx_try_upgrade_(&map->lock, file, line)) { in _vm_map_lock_upgrade()
746 last_timestamp = map->timestamp; in _vm_map_lock_upgrade()
747 sx_sunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
753 sx_xlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
754 if (last_timestamp != map->timestamp) { in _vm_map_lock_upgrade()
755 sx_xunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
760 map->timestamp++; in _vm_map_lock_upgrade()
765 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_downgrade() argument
768 if (map->system_map) { in _vm_map_lock_downgrade()
769 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_lock_downgrade()
771 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_downgrade()
773 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_lock_downgrade()
774 sx_downgrade_(&map->lock, file, line); in _vm_map_lock_downgrade()
785 vm_map_locked(vm_map_t map) in vm_map_locked() argument
788 if (map->system_map) in vm_map_locked()
789 return (mtx_owned(&map->system_mtx)); in vm_map_locked()
791 return (sx_xlocked(&map->lock)); in vm_map_locked()
809 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) in _vm_map_unlock_and_wait() argument
812 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock_and_wait()
814 if (map->system_map) { in _vm_map_unlock_and_wait()
815 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_and_wait()
817 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_and_wait()
819 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock_and_wait()
821 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", in _vm_map_unlock_and_wait()
832 vm_map_wakeup(vm_map_t map) in vm_map_wakeup() argument
842 wakeup(&map->root); in vm_map_wakeup()
846 vm_map_busy(vm_map_t map) in vm_map_busy() argument
849 VM_MAP_ASSERT_LOCKED(map); in vm_map_busy()
850 map->busy++; in vm_map_busy()
854 vm_map_unbusy(vm_map_t map) in vm_map_unbusy() argument
857 VM_MAP_ASSERT_LOCKED(map); in vm_map_unbusy()
858 KASSERT(map->busy, ("vm_map_unbusy: not busy")); in vm_map_unbusy()
859 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { in vm_map_unbusy()
860 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); in vm_map_unbusy()
861 wakeup(&map->busy); in vm_map_unbusy()
866 vm_map_wait_busy(vm_map_t map) in vm_map_wait_busy() argument
869 VM_MAP_ASSERT_LOCKED(map); in vm_map_wait_busy()
870 while (map->busy) { in vm_map_wait_busy()
871 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); in vm_map_wait_busy()
872 if (map->system_map) in vm_map_wait_busy()
873 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); in vm_map_wait_busy()
875 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); in vm_map_wait_busy()
877 map->timestamp++; in vm_map_wait_busy()
891 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in _vm_map_init() argument
894 map->header.eflags = MAP_ENTRY_HEADER; in _vm_map_init()
895 map->needs_wakeup = FALSE; in _vm_map_init()
896 map->system_map = 0; in _vm_map_init()
897 map->pmap = pmap; in _vm_map_init()
898 map->header.end = min; in _vm_map_init()
899 map->header.start = max; in _vm_map_init()
900 map->flags = 0; in _vm_map_init()
901 map->header.left = map->header.right = &map->header; in _vm_map_init()
902 map->root = NULL; in _vm_map_init()
903 map->timestamp = 0; in _vm_map_init()
904 map->busy = 0; in _vm_map_init()
905 map->anon_loc = 0; in _vm_map_init()
907 map->nupdates = 0; in _vm_map_init()
912 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init() argument
915 _vm_map_init(map, pmap, min, max); in vm_map_init()
916 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vm_map_init()
918 sx_init(&map->lock, "vm map (user)"); in vm_map_init()
927 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
929 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
939 vm_map_entry_create(vm_map_t map) in vm_map_entry_create() argument
944 if (map == kernel_map) { in vm_map_entry_create()
945 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_create()
963 if (map->system_map) { in vm_map_entry_create()
1130 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, in vm_map_splay_split() argument
1135 left = right = &map->header; in vm_map_splay_split()
1136 root = map->root; in vm_map_splay_split()
1339 vm_map_splay(vm_map_t map, vm_offset_t addr) in vm_map_splay() argument
1344 header = &map->header; in vm_map_splay()
1345 root = vm_map_splay_split(map, addr, 0, &llist, &rlist); in vm_map_splay()
1372 map->root = root; in vm_map_splay()
1373 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_splay()
1387 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1393 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1394 map->nentries, entry); in vm_map_entry_link()
1395 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_link()
1396 map->nentries++; in vm_map_entry_link()
1397 header = &map->header; in vm_map_entry_link()
1398 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1440 map->root = entry; in vm_map_entry_link()
1441 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_link()
1450 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1456 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unlink()
1457 header = &map->header; in vm_map_entry_unlink()
1458 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1484 map->root = root; in vm_map_entry_unlink()
1485 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_unlink()
1486 map->nentries--; in vm_map_entry_unlink()
1487 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1488 map->nentries, entry); in vm_map_entry_unlink()
1500 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) in vm_map_entry_resize() argument
1504 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_resize()
1505 header = &map->header; in vm_map_entry_resize()
1506 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize()
1513 map->root = root; in vm_map_entry_resize()
1514 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_resize()
1516 __func__, map, map->nentries, entry); in vm_map_entry_resize()
1531 vm_map_t map, in vm_map_lookup_entry() argument
1542 header = &map->header; in vm_map_lookup_entry()
1543 cur = map->root; in vm_map_lookup_entry()
1552 if ((locked = vm_map_locked(map)) || in vm_map_lookup_entry()
1553 sx_try_upgrade(&map->lock)) { in vm_map_lookup_entry()
1560 cur = vm_map_splay(map, address); in vm_map_lookup_entry()
1562 VM_MAP_UNLOCK_CONSISTENT(map); in vm_map_lookup_entry()
1563 sx_downgrade(&map->lock); in vm_map_lookup_entry()
1616 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert() argument
1626 VM_MAP_ASSERT_LOCKED(map); in vm_map_insert()
1640 if (start == end || !vm_map_range_valid(map, start, end)) in vm_map_insert()
1643 if ((map->flags & MAP_WXORX) != 0 && (prot & (VM_PROT_WRITE | in vm_map_insert()
1651 if (vm_map_lookup_entry(map, start, &prev_entry)) in vm_map_insert()
1720 if (map == kernel_map && end > kernel_vm_end) in vm_map_insert()
1761 map->size += end - prev_entry->end; in vm_map_insert()
1762 vm_map_entry_resize(map, prev_entry, in vm_map_insert()
1764 vm_map_try_merge_entries(map, prev_entry, next_entry); in vm_map_insert()
1790 new_entry = vm_map_entry_create(map); in vm_map_insert()
1814 vm_map_entry_link(map, new_entry); in vm_map_insert()
1816 map->size += new_entry->end - new_entry->start; in vm_map_insert()
1824 vm_map_try_merge_entries(map, prev_entry, new_entry); in vm_map_insert()
1825 vm_map_try_merge_entries(map, new_entry, next_entry); in vm_map_insert()
1828 vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), in vm_map_insert()
1853 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) in vm_map_findspace() argument
1859 VM_MAP_ASSERT_LOCKED(map); in vm_map_findspace()
1865 start = MAX(start, vm_map_min(map)); in vm_map_findspace()
1866 if (start >= vm_map_max(map) || length > vm_map_max(map) - start) in vm_map_findspace()
1867 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1870 if (map->root == NULL) in vm_map_findspace()
1880 header = &map->header; in vm_map_findspace()
1881 root = vm_map_splay_split(map, start, length, &llist, &rlist); in vm_map_findspace()
1901 map->root = root; in vm_map_findspace()
1902 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1908 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1939 map->root = root; in vm_map_findspace()
1940 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1945 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_fixed() argument
1956 vm_map_lock(map); in vm_map_fixed()
1957 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_fixed()
1959 result = vm_map_delete(map, start, end); in vm_map_fixed()
1964 result = vm_map_stack_locked(map, start, length, sgrowsiz, in vm_map_fixed()
1967 result = vm_map_insert(map, object, offset, start, end, in vm_map_fixed()
1971 vm_map_unlock(map); in vm_map_fixed()
2016 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_alignspace() argument
2022 VM_MAP_ASSERT_LOCKED(map); in vm_map_alignspace()
2024 KASSERT(free_addr == vm_map_findspace(map, free_addr, length), in vm_map_alignspace()
2054 *addr = vm_map_findspace(map, aligned_addr, length); in vm_map_alignspace()
2055 if (*addr + length > vm_map_max(map) || in vm_map_alignspace()
2071 vm_map_find_aligned(vm_map_t map, vm_offset_t *addr, vm_size_t length, in vm_map_find_aligned() argument
2075 *addr = vm_map_findspace(map, *addr, length); in vm_map_find_aligned()
2076 if (*addr + length > vm_map_max(map) || in vm_map_find_aligned()
2079 return (vm_map_alignspace(map, NULL, 0, addr, length, max_addr, in vm_map_find_aligned()
2093 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find() argument
2115 en_aslr = (map->flags & MAP_ASLR) != 0; in vm_map_find()
2117 (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && in vm_map_find()
2124 (map->flags & MAP_ASLR_IGNSTART) != 0) in vm_map_find()
2125 curr_min_addr = min_addr = vm_map_min(map); in vm_map_find()
2127 vm_map_lock(map); in vm_map_find()
2129 curr_min_addr = map->anon_loc; in vm_map_find()
2164 curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? in vm_map_find()
2165 vm_map_min(map) : min_addr; in vm_map_find()
2177 gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && in vm_map_find()
2180 *addr = vm_map_findspace(map, curr_min_addr, in vm_map_find()
2183 vm_map_max(map)) in vm_map_find()
2190 *addr = vm_map_findspace(map, curr_min_addr, length); in vm_map_find()
2191 if (*addr + length > vm_map_max(map) || in vm_map_find()
2204 (rv = vm_map_alignspace(map, object, offset, addr, length, in vm_map_find()
2216 if (!vm_map_range_valid(map, *addr, *addr + length)) { in vm_map_find()
2220 rv = vm_map_delete(map, *addr, *addr + length); in vm_map_find()
2225 rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, in vm_map_find()
2228 rv = vm_map_insert(map, object, offset, *addr, *addr + length, in vm_map_find()
2232 map->anon_loc = *addr + length; in vm_map_find()
2234 vm_map_unlock(map); in vm_map_find()
2251 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_min() argument
2261 rv = vm_map_find(map, object, offset, addr, length, max_addr, in vm_map_find_min()
2297 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2314 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2327 vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev_entry, in vm_map_try_merge_entries() argument
2331 VM_MAP_ASSERT_LOCKED(map); in vm_map_try_merge_entries()
2334 vm_map_entry_unlink(map, prev_entry, UNLINK_MERGE_NEXT); in vm_map_try_merge_entries()
2335 vm_map_merged_neighbor_dispose(map, prev_entry); in vm_map_try_merge_entries()
2367 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_charge_object() argument
2370 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_charge_object()
2373 if (entry->object.vm_object == NULL && !map->system_map && in vm_map_entry_charge_object()
2395 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_clone() argument
2399 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_clone()
2405 vm_map_entry_charge_object(map, entry); in vm_map_entry_clone()
2408 new_entry = vm_map_entry_create(map); in vm_map_entry_clone()
2433 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) in vm_map_clip_start() argument
2438 if (!map->system_map) in vm_map_clip_start()
2440 "%s: map %p entry %p start 0x%jx", __func__, map, entry, in vm_map_clip_start()
2446 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_start()
2457 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_start()
2464 vm_map_entry_link(map, new_entry); in vm_map_clip_start()
2476 vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, in vm_map_lookup_clip_start() argument
2482 if (!map->system_map) in vm_map_lookup_clip_start()
2484 "%s: map %p start 0x%jx prev %p", __func__, map, in vm_map_lookup_clip_start()
2487 if (vm_map_lookup_entry(map, start, prev_entry)) { in vm_map_lookup_clip_start()
2489 rv = vm_map_clip_start(map, entry, start); in vm_map_lookup_clip_start()
2507 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) in vm_map_clip_end() argument
2512 if (!map->system_map) in vm_map_clip_end()
2514 "%s: map %p entry %p end 0x%jx", __func__, map, entry, in vm_map_clip_end()
2520 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_end()
2531 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_end()
2538 vm_map_entry_link(map, new_entry); in vm_map_clip_end()
2563 vm_map_t map, in vm_map_submap() argument
2577 vm_map_lock(map); in vm_map_submap()
2578 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_submap()
2579 if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && in vm_map_submap()
2582 result = vm_map_clip_start(map, entry, start); in vm_map_submap()
2585 result = vm_map_clip_end(map, entry, end); in vm_map_submap()
2593 vm_map_unlock(map); in vm_map_submap()
2621 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, in vm_map_pmap_enter() argument
2633 pmap_object_init_pt(map->pmap, addr, object, pindex, in vm_map_pmap_enter()
2691 pmap_enter_object(map->pmap, start, addr + in vm_map_pmap_enter()
2697 pmap_enter_object(map->pmap, start, addr + ptoa(psize), in vm_map_pmap_enter()
2709 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_protect() argument
2728 vm_map_lock(map); in vm_map_protect()
2730 if ((map->flags & MAP_WXORX) != 0 && in vm_map_protect()
2734 vm_map_unlock(map); in vm_map_protect()
2744 vm_map_wait_busy(map); in vm_map_protect()
2746 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_protect()
2748 if (!vm_map_lookup_entry(map, start, &first_entry)) in vm_map_protect()
2759 vm_map_unlock(map); in vm_map_protect()
2768 vm_map_unlock(map); in vm_map_protect()
2784 vm_map_unlock_and_wait(map, 0); in vm_map_protect()
2795 rv = vm_map_clip_start(map, first_entry, start); in vm_map_protect()
2797 vm_map_unlock(map); in vm_map_protect()
2802 rv = vm_map_clip_end(map, entry, end); in vm_map_protect()
2804 vm_map_unlock(map); in vm_map_protect()
2867 vm_map_try_merge_entries(map, prev_entry, entry), in vm_map_protect()
2891 vm_fault_copy_entry(map, map, entry, entry, NULL); in vm_map_protect()
2900 pmap_protect(map->pmap, entry->start, in vm_map_protect()
2906 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_protect()
2907 vm_map_unlock(map); in vm_map_protect()
2921 vm_map_t map, in vm_map_madvise() argument
2947 vm_map_lock(map); in vm_map_madvise()
2955 vm_map_lock_read(map); in vm_map_madvise()
2964 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_madvise()
2973 rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); in vm_map_madvise()
2975 vm_map_unlock(map); in vm_map_madvise()
2984 rv = vm_map_clip_end(map, entry, end); in vm_map_madvise()
2986 vm_map_unlock(map); in vm_map_madvise()
3018 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3020 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3021 vm_map_unlock(map); in vm_map_madvise()
3032 if (!vm_map_lookup_entry(map, start, &entry)) in vm_map_madvise()
3081 pmap_advise(map->pmap, useStart, useEnd, in vm_map_madvise()
3094 vm_map_pmap_enter(map, in vm_map_madvise()
3104 vm_map_unlock_read(map); in vm_map_madvise()
3118 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_inherit() argument
3135 vm_map_lock(map); in vm_map_inherit()
3136 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
3137 rv = vm_map_lookup_clip_start(map, start, &start_entry, &prev_entry); in vm_map_inherit()
3140 if (vm_map_lookup_entry(map, end - 1, &lentry)) { in vm_map_inherit()
3141 rv = vm_map_clip_end(map, lentry, end); in vm_map_inherit()
3162 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3164 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3166 vm_map_unlock(map); in vm_map_inherit()
3179 vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start, in vm_map_entry_in_transition() argument
3186 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_in_transition()
3194 last_timestamp = map->timestamp; in vm_map_entry_in_transition()
3195 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_entry_in_transition()
3200 vm_map_lock(map); in vm_map_entry_in_transition()
3201 if (last_timestamp + 1 == map->timestamp) in vm_map_entry_in_transition()
3209 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_entry_in_transition()
3225 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_unwire() argument
3236 vm_map_lock(map); in vm_map_unwire()
3237 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire()
3238 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3242 vm_map_unlock(map); in vm_map_unwire()
3252 next_entry = vm_map_entry_in_transition(map, start, in vm_map_unwire()
3256 vm_map_unlock(map); in vm_map_unwire()
3266 rv = vm_map_clip_start(map, entry, start); in vm_map_unwire()
3269 rv = vm_map_clip_end(map, entry, end); in vm_map_unwire()
3305 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3334 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3350 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3352 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3353 vm_map_unlock(map); in vm_map_unwire()
3355 vm_map_wakeup(map); in vm_map_unwire()
3389 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3393 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_entry_failure()
3405 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3418 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire() argument
3422 vm_map_lock(map); in vm_map_wire()
3423 rv = vm_map_wire_locked(map, start, end, flags); in vm_map_wire()
3424 vm_map_unlock(map); in vm_map_wire()
3435 vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire_locked() argument
3445 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_locked()
3454 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire_locked()
3455 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3466 next_entry = vm_map_entry_in_transition(map, start, in vm_map_wire_locked()
3478 rv = vm_map_clip_start(map, entry, start); in vm_map_wire_locked()
3481 rv = vm_map_clip_end(map, entry, end); in vm_map_wire_locked()
3507 vm_map_wire_entry_failure(map, entry, in vm_map_wire_locked()
3520 last_timestamp = map->timestamp; in vm_map_wire_locked()
3524 vm_map_busy(map); in vm_map_wire_locked()
3525 vm_map_unlock(map); in vm_map_wire_locked()
3533 rv = vm_fault(map, faddr, VM_PROT_NONE, in vm_map_wire_locked()
3538 vm_map_lock(map); in vm_map_wire_locked()
3539 vm_map_unbusy(map); in vm_map_wire_locked()
3540 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire_locked()
3547 if (!vm_map_lookup_entry(map, saved_start, in vm_map_wire_locked()
3563 vm_map_wire_entry_failure(map, in vm_map_wire_locked()
3568 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire_locked()
3594 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3642 vm_map_entry_unwire(map, entry); in vm_map_wire_locked()
3660 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3662 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3664 vm_map_wakeup(map); in vm_map_wire_locked()
3686 vm_map_t map, in vm_map_sync() argument
3700 vm_map_lock_read(map); in vm_map_sync()
3701 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_sync()
3702 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_sync()
3703 vm_map_unlock_read(map); in vm_map_sync()
3717 vm_map_unlock_read(map); in vm_map_sync()
3726 vm_map_unlock_read(map); in vm_map_sync()
3733 vm_map_unlock_read(map); in vm_map_sync()
3739 pmap_remove(map->pmap, start, end); in vm_map_sync()
3767 last_timestamp = map->timestamp; in vm_map_sync()
3768 vm_map_unlock_read(map); in vm_map_sync()
3773 vm_map_lock_read(map); in vm_map_sync()
3774 if (last_timestamp == map->timestamp || in vm_map_sync()
3775 !vm_map_lookup_entry(map, start, &entry)) in vm_map_sync()
3779 vm_map_unlock_read(map); in vm_map_sync()
3792 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3796 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unwire()
3803 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3824 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3830 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3837 vm_map_entry_deallocate(entry, map->system_map); in vm_map_entry_delete()
3842 map->size -= size; in vm_map_entry_delete()
3888 if (map->system_map) in vm_map_entry_delete()
3903 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_delete() argument
3908 VM_MAP_ASSERT_LOCKED(map); in vm_map_delete()
3917 rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); in vm_map_delete()
3927 (vm_map_pmap(map) != kernel_pmap && in vm_map_delete()
3934 last_timestamp = map->timestamp; in vm_map_delete()
3935 (void) vm_map_unlock_and_wait(map, 0); in vm_map_delete()
3936 vm_map_lock(map); in vm_map_delete()
3937 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
3944 rv = vm_map_lookup_clip_start(map, saved_start, in vm_map_delete()
3954 rv = vm_map_clip_end(map, entry, end); in vm_map_delete()
3964 vm_map_entry_unwire(map, entry); in vm_map_delete()
3973 pmap_remove(map->pmap, entry->start, entry->end); in vm_map_delete()
3975 if (entry->end == map->anon_loc) in vm_map_delete()
3976 map->anon_loc = entry->start; in vm_map_delete()
3984 vm_map_entry_delete(map, entry); in vm_map_delete()
3996 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_remove() argument
4000 vm_map_lock(map); in vm_map_remove()
4001 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_remove()
4002 result = vm_map_delete(map, start, end); in vm_map_remove()
4003 vm_map_unlock(map); in vm_map_remove()
4022 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_check_protection() argument
4028 if (!vm_map_lookup_entry(map, start, &tmp_entry)) in vm_map_check_protection()
4465 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack() argument
4472 MPASS((map->flags & MAP_WIREFUTURE) == 0); in vm_map_stack()
4475 vm_map_lock(map); in vm_map_stack()
4478 if (map->size + init_ssize > vmemlim) { in vm_map_stack()
4482 rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, in vm_map_stack()
4485 vm_map_unlock(map); in vm_map_stack()
4495 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack_locked() argument
4514 !vm_map_range_valid(map, addrbos, addrbos + max_ssize)) in vm_map_stack_locked()
4527 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) in vm_map_stack_locked()
4557 rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow); in vm_map_stack_locked()
4571 rv = vm_map_insert(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, in vm_map_stack_locked()
4586 (void)vm_map_delete(map, bot, top); in vm_map_stack_locked()
4596 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) in vm_map_growstack() argument
4622 if (p != initproc && (map != &p->p_vmspace->vm_map || in vm_map_growstack()
4626 MPASS(!map->system_map); in vm_map_growstack()
4633 if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) in vm_map_growstack()
4701 if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4702 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { in vm_map_growstack()
4710 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { in vm_map_growstack()
4721 if (map->size + grow_amount > vmemlim) { in vm_map_growstack()
4728 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { in vm_map_growstack()
4737 if (vm_map_lock_upgrade(map)) { in vm_map_growstack()
4739 vm_map_lock_read(map); in vm_map_growstack()
4748 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4752 vm_map_entry_resize(map, gap_entry, -grow_amount); in vm_map_growstack()
4755 rv = vm_map_insert(map, NULL, 0, grow_start, in vm_map_growstack()
4761 rv1 = vm_map_insert(map, NULL, 0, gap_start, in vm_map_growstack()
4766 vm_map_entry_resize(map, gap_entry, in vm_map_growstack()
4783 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4784 vm_map_entry_resize(map, stack_entry, in vm_map_growstack()
4790 map->size += grow_amount; in vm_map_growstack()
4801 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4802 rv = vm_map_wire_locked(map, grow_start, in vm_map_growstack()
4806 vm_map_lock_downgrade(map); in vm_map_growstack()
4812 error = racct_set(p, RACCT_VMEM, map->size); in vm_map_growstack()
4816 ptoa(pmap_wired_count(map->pmap))); in vm_map_growstack()
4927 vm_map_t map = *var_map; in vm_map_lookup() local
4936 vm_map_lock_read(map); in vm_map_lookup()
4942 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { in vm_map_lookup()
4943 vm_map_unlock_read(map); in vm_map_lookup()
4953 vm_map_t old_map = map; in vm_map_lookup()
4955 *var_map = map = entry->object.sub_map; in vm_map_lookup()
4966 if (prot == VM_PROT_NONE && map != kernel_map && in vm_map_lookup()
4970 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
4975 vm_map_unlock_read(map); in vm_map_lookup()
4985 vm_map_unlock_read(map); in vm_map_lookup()
5017 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5029 vm_map_unlock(map); in vm_map_lookup()
5047 vm_map_lock_downgrade(map); in vm_map_lookup()
5060 if (entry->object.vm_object == NULL && !map->system_map) { in vm_map_lookup()
5061 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5067 vm_map_lock_downgrade(map); in vm_map_lookup()
5098 vm_map_t map = *var_map; in vm_map_lookup_locked() local
5105 if (!vm_map_lookup_entry(map, vaddr, out_entry)) in vm_map_lookup_locked()
5148 if (entry->object.vm_object == NULL && !map->system_map) in vm_map_lookup_locked()
5169 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
5174 vm_map_unlock_read(map); in vm_map_lookup_done()
5178 vm_map_max_KBI(const struct vm_map *map) in vm_map_max_KBI() argument
5181 return (vm_map_max(map)); in vm_map_max_KBI()
5185 vm_map_min_KBI(const struct vm_map *map) in vm_map_min_KBI() argument
5188 return (vm_map_min(map)); in vm_map_min_KBI()
5192 vm_map_pmap_KBI(vm_map_t map) in vm_map_pmap_KBI() argument
5195 return (map->pmap); in vm_map_pmap_KBI()
5199 vm_map_range_valid_KBI(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_range_valid_KBI() argument
5202 return (vm_map_range_valid(map, start, end)); in vm_map_range_valid_KBI()
5207 _vm_map_assert_consistent(vm_map_t map, int check) in _vm_map_assert_consistent() argument
5214 ++map->nupdates; in _vm_map_assert_consistent()
5219 header = prev = &map->header; in _vm_map_assert_consistent()
5220 VM_MAP_ENTRY_FOREACH(entry, map) { in _vm_map_assert_consistent()
5222 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5225 ("map %p start = %jx, end = %jx", map, in _vm_map_assert_consistent()
5229 ("map %p left->start = %jx, start = %jx", map, in _vm_map_assert_consistent()
5233 ("map %p start = %jx, right->start = %jx", map, in _vm_map_assert_consistent()
5235 cur = map->root; in _vm_map_assert_consistent()
5243 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5249 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5253 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5260 ("map %p max = %jx, max_left = %jx, max_right = %jx", map, in _vm_map_assert_consistent()
5266 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5278 vm_map_print(vm_map_t map) in vm_map_print() argument
5283 (void *)map, in vm_map_print()
5284 (void *)map->pmap, map->nentries, map->timestamp); in vm_map_print()
5287 prev = &map->header; in vm_map_print()
5288 VM_MAP_ENTRY_FOREACH(entry, map) { in vm_map_print()
5308 if (prev == &map->header || in vm_map_print()
5330 if (prev == &map->header || in vm_map_print()
5345 DB_SHOW_COMMAND(map, map) in DB_SHOW_COMMAND() argument