Lines Matching refs:map
133 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
136 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
137 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
138 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
140 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
145 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
148 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
168 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
170 if (start < vm_map_min(map)) \
171 start = vm_map_min(map); \
172 if (end > vm_map_max(map)) \
173 end = vm_map_max(map); \
293 vm_map_t map; in vmspace_zinit() local
296 map = &vm->vm_map; in vmspace_zinit()
298 memset(map, 0, sizeof(*map)); in vmspace_zinit()
299 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vmspace_zinit()
301 sx_init(&map->lock, "vm map (user)"); in vmspace_zinit()
528 _vm_map_lock(vm_map_t map, const char *file, int line) in _vm_map_lock() argument
531 if (map->system_map) in _vm_map_lock()
532 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock()
534 sx_xlock_(&map->lock, file, line); in _vm_map_lock()
535 map->timestamp++; in _vm_map_lock()
650 _vm_map_assert_locked(vm_map_t map, const char *file, int line) in _vm_map_assert_locked() argument
653 if (map->system_map) in _vm_map_assert_locked()
654 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_assert_locked()
656 sx_assert_(&map->lock, SA_XLOCKED, file, line); in _vm_map_assert_locked()
659 #define VM_MAP_ASSERT_LOCKED(map) \ argument
660 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
671 static void _vm_map_assert_consistent(vm_map_t map, int check);
673 #define VM_MAP_ASSERT_CONSISTENT(map) \ argument
674 _vm_map_assert_consistent(map, VMMAP_CHECK_ALL)
676 #define VM_MAP_UNLOCK_CONSISTENT(map) do { \ argument
677 if (map->nupdates > map->nentries) { \
678 _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \
679 map->nupdates = 0; \
683 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
686 #define VM_MAP_ASSERT_LOCKED(map) argument
687 #define VM_MAP_ASSERT_CONSISTENT(map) argument
688 #define VM_MAP_UNLOCK_CONSISTENT(map) argument
692 _vm_map_unlock(vm_map_t map, const char *file, int line) in _vm_map_unlock() argument
695 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock()
696 if (map->system_map) { in _vm_map_unlock()
698 if (map == kernel_map && (map->flags & MAP_REPLENISH) != 0) { in _vm_map_unlock()
700 map->flags &= ~MAP_REPLENISH; in _vm_map_unlock()
703 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock()
705 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock()
711 _vm_map_lock_read(vm_map_t map, const char *file, int line) in _vm_map_lock_read() argument
714 if (map->system_map) in _vm_map_lock_read()
715 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock_read()
717 sx_slock_(&map->lock, file, line); in _vm_map_lock_read()
721 _vm_map_unlock_read(vm_map_t map, const char *file, int line) in _vm_map_unlock_read() argument
724 if (map->system_map) { in _vm_map_unlock_read()
725 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_read()
727 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_read()
729 sx_sunlock_(&map->lock, file, line); in _vm_map_unlock_read()
735 _vm_map_trylock(vm_map_t map, const char *file, int line) in _vm_map_trylock() argument
739 error = map->system_map ? in _vm_map_trylock()
740 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock()
741 !sx_try_xlock_(&map->lock, file, line); in _vm_map_trylock()
743 map->timestamp++; in _vm_map_trylock()
748 _vm_map_trylock_read(vm_map_t map, const char *file, int line) in _vm_map_trylock_read() argument
752 error = map->system_map ? in _vm_map_trylock_read()
753 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock_read()
754 !sx_try_slock_(&map->lock, file, line); in _vm_map_trylock_read()
769 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_upgrade() argument
773 if (map->system_map) { in _vm_map_lock_upgrade()
774 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_upgrade()
776 if (!sx_try_upgrade_(&map->lock, file, line)) { in _vm_map_lock_upgrade()
777 last_timestamp = map->timestamp; in _vm_map_lock_upgrade()
778 sx_sunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
784 sx_xlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
785 if (last_timestamp != map->timestamp) { in _vm_map_lock_upgrade()
786 sx_xunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
791 map->timestamp++; in _vm_map_lock_upgrade()
796 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_downgrade() argument
799 if (map->system_map) { in _vm_map_lock_downgrade()
800 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_lock_downgrade()
802 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_downgrade()
804 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_lock_downgrade()
805 sx_downgrade_(&map->lock, file, line); in _vm_map_lock_downgrade()
816 vm_map_locked(vm_map_t map) in vm_map_locked() argument
819 if (map->system_map) in vm_map_locked()
820 return (mtx_owned(&map->system_mtx)); in vm_map_locked()
822 return (sx_xlocked(&map->lock)); in vm_map_locked()
840 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) in _vm_map_unlock_and_wait() argument
843 VM_MAP_UNLOCK_CONSISTENT(map); in _vm_map_unlock_and_wait()
845 if (map->system_map) { in _vm_map_unlock_and_wait()
846 KASSERT((map->flags & MAP_REPLENISH) == 0, in _vm_map_unlock_and_wait()
848 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_and_wait()
850 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock_and_wait()
852 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", in _vm_map_unlock_and_wait()
863 vm_map_wakeup(vm_map_t map) in vm_map_wakeup() argument
873 wakeup(&map->root); in vm_map_wakeup()
877 vm_map_busy(vm_map_t map) in vm_map_busy() argument
880 VM_MAP_ASSERT_LOCKED(map); in vm_map_busy()
881 map->busy++; in vm_map_busy()
885 vm_map_unbusy(vm_map_t map) in vm_map_unbusy() argument
888 VM_MAP_ASSERT_LOCKED(map); in vm_map_unbusy()
889 KASSERT(map->busy, ("vm_map_unbusy: not busy")); in vm_map_unbusy()
890 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { in vm_map_unbusy()
891 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); in vm_map_unbusy()
892 wakeup(&map->busy); in vm_map_unbusy()
897 vm_map_wait_busy(vm_map_t map) in vm_map_wait_busy() argument
900 VM_MAP_ASSERT_LOCKED(map); in vm_map_wait_busy()
901 while (map->busy) { in vm_map_wait_busy()
902 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); in vm_map_wait_busy()
903 if (map->system_map) in vm_map_wait_busy()
904 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); in vm_map_wait_busy()
906 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); in vm_map_wait_busy()
908 map->timestamp++; in vm_map_wait_busy()
922 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in _vm_map_init() argument
925 map->header.eflags = MAP_ENTRY_HEADER; in _vm_map_init()
926 map->needs_wakeup = FALSE; in _vm_map_init()
927 map->system_map = 0; in _vm_map_init()
928 map->pmap = pmap; in _vm_map_init()
929 map->header.end = min; in _vm_map_init()
930 map->header.start = max; in _vm_map_init()
931 map->flags = 0; in _vm_map_init()
932 map->header.left = map->header.right = &map->header; in _vm_map_init()
933 map->root = NULL; in _vm_map_init()
934 map->timestamp = 0; in _vm_map_init()
935 map->busy = 0; in _vm_map_init()
936 map->anon_loc = 0; in _vm_map_init()
938 map->nupdates = 0; in _vm_map_init()
943 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init() argument
946 _vm_map_init(map, pmap, min, max); in vm_map_init()
947 mtx_init(&map->system_mtx, "vm map (system)", NULL, in vm_map_init()
949 sx_init(&map->lock, "vm map (user)"); in vm_map_init()
958 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
960 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
970 vm_map_entry_create(vm_map_t map) in vm_map_entry_create() argument
975 if (map == kernel_map) { in vm_map_entry_create()
976 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_create()
994 if (map->system_map) { in vm_map_entry_create()
1161 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, in vm_map_splay_split() argument
1166 left = right = &map->header; in vm_map_splay_split()
1167 root = map->root; in vm_map_splay_split()
1370 vm_map_splay(vm_map_t map, vm_offset_t addr) in vm_map_splay() argument
1375 header = &map->header; in vm_map_splay()
1376 root = vm_map_splay_split(map, addr, 0, &llist, &rlist); in vm_map_splay()
1403 map->root = root; in vm_map_splay()
1404 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_splay()
1418 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1424 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1425 map->nentries, entry); in vm_map_entry_link()
1426 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_link()
1427 map->nentries++; in vm_map_entry_link()
1428 header = &map->header; in vm_map_entry_link()
1429 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1471 map->root = entry; in vm_map_entry_link()
1472 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_link()
1481 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1487 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unlink()
1488 header = &map->header; in vm_map_entry_unlink()
1489 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1515 map->root = root; in vm_map_entry_unlink()
1516 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_unlink()
1517 map->nentries--; in vm_map_entry_unlink()
1518 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1519 map->nentries, entry); in vm_map_entry_unlink()
1531 vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry, vm_size_t grow_amount) in vm_map_entry_resize() argument
1535 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_resize()
1536 header = &map->header; in vm_map_entry_resize()
1537 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize()
1544 map->root = root; in vm_map_entry_resize()
1545 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_resize()
1547 __func__, map, map->nentries, entry); in vm_map_entry_resize()
1562 vm_map_t map, in vm_map_lookup_entry() argument
1573 header = &map->header; in vm_map_lookup_entry()
1574 cur = map->root; in vm_map_lookup_entry()
1583 if ((locked = vm_map_locked(map)) || in vm_map_lookup_entry()
1584 sx_try_upgrade(&map->lock)) { in vm_map_lookup_entry()
1591 cur = vm_map_splay(map, address); in vm_map_lookup_entry()
1593 VM_MAP_UNLOCK_CONSISTENT(map); in vm_map_lookup_entry()
1594 sx_downgrade(&map->lock); in vm_map_lookup_entry()
1647 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert() argument
1657 VM_MAP_ASSERT_LOCKED(map); in vm_map_insert()
1671 if (start == end || !vm_map_range_valid(map, start, end)) in vm_map_insert()
1674 if ((map->flags & MAP_WXORX) != 0 && (prot & (VM_PROT_WRITE | in vm_map_insert()
1682 if (vm_map_lookup_entry(map, start, &prev_entry)) in vm_map_insert()
1751 if (map == kernel_map && end > kernel_vm_end) in vm_map_insert()
1792 map->size += end - prev_entry->end; in vm_map_insert()
1793 vm_map_entry_resize(map, prev_entry, in vm_map_insert()
1795 vm_map_try_merge_entries(map, prev_entry, next_entry); in vm_map_insert()
1821 new_entry = vm_map_entry_create(map); in vm_map_insert()
1845 vm_map_entry_link(map, new_entry); in vm_map_insert()
1847 map->size += new_entry->end - new_entry->start; in vm_map_insert()
1855 vm_map_try_merge_entries(map, prev_entry, new_entry); in vm_map_insert()
1856 vm_map_try_merge_entries(map, new_entry, next_entry); in vm_map_insert()
1859 vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), in vm_map_insert()
1884 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) in vm_map_findspace() argument
1890 VM_MAP_ASSERT_LOCKED(map); in vm_map_findspace()
1896 start = MAX(start, vm_map_min(map)); in vm_map_findspace()
1897 if (start >= vm_map_max(map) || length > vm_map_max(map) - start) in vm_map_findspace()
1898 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1901 if (map->root == NULL) in vm_map_findspace()
1911 header = &map->header; in vm_map_findspace()
1912 root = vm_map_splay_split(map, start, length, &llist, &rlist); in vm_map_findspace()
1932 map->root = root; in vm_map_findspace()
1933 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1939 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1970 map->root = root; in vm_map_findspace()
1971 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1976 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_fixed() argument
1987 vm_map_lock(map); in vm_map_fixed()
1988 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_fixed()
1990 result = vm_map_delete(map, start, end); in vm_map_fixed()
1995 result = vm_map_stack_locked(map, start, length, sgrowsiz, in vm_map_fixed()
1998 result = vm_map_insert(map, object, offset, start, end, in vm_map_fixed()
2002 vm_map_unlock(map); in vm_map_fixed()
2047 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_alignspace() argument
2053 VM_MAP_ASSERT_LOCKED(map); in vm_map_alignspace()
2055 KASSERT(free_addr == vm_map_findspace(map, free_addr, length), in vm_map_alignspace()
2085 *addr = vm_map_findspace(map, aligned_addr, length); in vm_map_alignspace()
2086 if (*addr + length > vm_map_max(map) || in vm_map_alignspace()
2102 vm_map_find_aligned(vm_map_t map, vm_offset_t *addr, vm_size_t length, in vm_map_find_aligned() argument
2106 *addr = vm_map_findspace(map, *addr, length); in vm_map_find_aligned()
2107 if (*addr + length > vm_map_max(map) || in vm_map_find_aligned()
2110 return (vm_map_alignspace(map, NULL, 0, addr, length, max_addr, in vm_map_find_aligned()
2124 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find() argument
2146 en_aslr = (map->flags & MAP_ASLR) != 0; in vm_map_find()
2148 (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && in vm_map_find()
2155 (map->flags & MAP_ASLR_IGNSTART) != 0) in vm_map_find()
2156 curr_min_addr = min_addr = vm_map_min(map); in vm_map_find()
2158 vm_map_lock(map); in vm_map_find()
2160 curr_min_addr = map->anon_loc; in vm_map_find()
2195 curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? in vm_map_find()
2196 vm_map_min(map) : min_addr; in vm_map_find()
2208 gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && in vm_map_find()
2211 *addr = vm_map_findspace(map, curr_min_addr, in vm_map_find()
2214 vm_map_max(map)) in vm_map_find()
2221 *addr = vm_map_findspace(map, curr_min_addr, length); in vm_map_find()
2222 if (*addr + length > vm_map_max(map) || in vm_map_find()
2235 (rv = vm_map_alignspace(map, object, offset, addr, length, in vm_map_find()
2247 if (!vm_map_range_valid(map, *addr, *addr + length)) { in vm_map_find()
2251 rv = vm_map_delete(map, *addr, *addr + length); in vm_map_find()
2256 rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, in vm_map_find()
2259 rv = vm_map_insert(map, object, offset, *addr, *addr + length, in vm_map_find()
2263 map->anon_loc = *addr + length; in vm_map_find()
2265 vm_map_unlock(map); in vm_map_find()
2282 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_min() argument
2292 rv = vm_map_find(map, object, offset, addr, length, max_addr, in vm_map_find_min()
2328 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2345 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2358 vm_map_try_merge_entries(vm_map_t map, vm_map_entry_t prev_entry, in vm_map_try_merge_entries() argument
2362 VM_MAP_ASSERT_LOCKED(map); in vm_map_try_merge_entries()
2365 vm_map_entry_unlink(map, prev_entry, UNLINK_MERGE_NEXT); in vm_map_try_merge_entries()
2366 vm_map_merged_neighbor_dispose(map, prev_entry); in vm_map_try_merge_entries()
2398 vm_map_entry_charge_object(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_charge_object() argument
2401 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_charge_object()
2404 if (entry->object.vm_object == NULL && !map->system_map && in vm_map_entry_charge_object()
2426 vm_map_entry_clone(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_clone() argument
2430 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_clone()
2436 vm_map_entry_charge_object(map, entry); in vm_map_entry_clone()
2439 new_entry = vm_map_entry_create(map); in vm_map_entry_clone()
2464 vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t startaddr) in vm_map_clip_start() argument
2469 if (!map->system_map) in vm_map_clip_start()
2471 "%s: map %p entry %p start 0x%jx", __func__, map, entry, in vm_map_clip_start()
2477 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_start()
2488 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_start()
2495 vm_map_entry_link(map, new_entry); in vm_map_clip_start()
2507 vm_map_lookup_clip_start(vm_map_t map, vm_offset_t start, in vm_map_lookup_clip_start() argument
2513 if (!map->system_map) in vm_map_lookup_clip_start()
2515 "%s: map %p start 0x%jx prev %p", __func__, map, in vm_map_lookup_clip_start()
2518 if (vm_map_lookup_entry(map, start, prev_entry)) { in vm_map_lookup_clip_start()
2520 rv = vm_map_clip_start(map, entry, start); in vm_map_lookup_clip_start()
2538 vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t endaddr) in vm_map_clip_end() argument
2543 if (!map->system_map) in vm_map_clip_end()
2545 "%s: map %p entry %p end 0x%jx", __func__, map, entry, in vm_map_clip_end()
2551 VM_MAP_ASSERT_LOCKED(map); in vm_map_clip_end()
2562 new_entry = vm_map_entry_clone(map, entry); in vm_map_clip_end()
2569 vm_map_entry_link(map, new_entry); in vm_map_clip_end()
2594 vm_map_t map, in vm_map_submap() argument
2608 vm_map_lock(map); in vm_map_submap()
2609 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_submap()
2610 if (vm_map_lookup_entry(map, start, &entry) && entry->end >= end && in vm_map_submap()
2613 result = vm_map_clip_start(map, entry, start); in vm_map_submap()
2616 result = vm_map_clip_end(map, entry, end); in vm_map_submap()
2624 vm_map_unlock(map); in vm_map_submap()
2652 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, in vm_map_pmap_enter() argument
2664 pmap_object_init_pt(map->pmap, addr, object, pindex, in vm_map_pmap_enter()
2722 pmap_enter_object(map->pmap, start, addr + in vm_map_pmap_enter()
2728 pmap_enter_object(map->pmap, start, addr + ptoa(psize), in vm_map_pmap_enter()
2740 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_protect() argument
2759 vm_map_lock(map); in vm_map_protect()
2761 if ((map->flags & MAP_WXORX) != 0 && in vm_map_protect()
2765 vm_map_unlock(map); in vm_map_protect()
2775 vm_map_wait_busy(map); in vm_map_protect()
2777 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_protect()
2779 if (!vm_map_lookup_entry(map, start, &first_entry)) in vm_map_protect()
2790 vm_map_unlock(map); in vm_map_protect()
2799 vm_map_unlock(map); in vm_map_protect()
2815 vm_map_unlock_and_wait(map, 0); in vm_map_protect()
2826 rv = vm_map_clip_start(map, first_entry, start); in vm_map_protect()
2828 vm_map_unlock(map); in vm_map_protect()
2833 rv = vm_map_clip_end(map, entry, end); in vm_map_protect()
2835 vm_map_unlock(map); in vm_map_protect()
2896 vm_map_try_merge_entries(map, prev_entry, entry), in vm_map_protect()
2920 vm_fault_copy_entry(map, map, entry, entry, NULL); in vm_map_protect()
2929 pmap_protect(map->pmap, entry->start, in vm_map_protect()
2935 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_protect()
2936 vm_map_unlock(map); in vm_map_protect()
2950 vm_map_t map, in vm_map_madvise() argument
2976 vm_map_lock(map); in vm_map_madvise()
2984 vm_map_lock_read(map); in vm_map_madvise()
2993 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_madvise()
3002 rv = vm_map_lookup_clip_start(map, start, &entry, &prev_entry); in vm_map_madvise()
3004 vm_map_unlock(map); in vm_map_madvise()
3013 rv = vm_map_clip_end(map, entry, end); in vm_map_madvise()
3015 vm_map_unlock(map); in vm_map_madvise()
3047 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3049 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_madvise()
3050 vm_map_unlock(map); in vm_map_madvise()
3061 if (!vm_map_lookup_entry(map, start, &entry)) in vm_map_madvise()
3110 pmap_advise(map->pmap, useStart, useEnd, in vm_map_madvise()
3123 vm_map_pmap_enter(map, in vm_map_madvise()
3133 vm_map_unlock_read(map); in vm_map_madvise()
3147 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_inherit() argument
3164 vm_map_lock(map); in vm_map_inherit()
3165 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
3166 rv = vm_map_lookup_clip_start(map, start, &start_entry, &prev_entry); in vm_map_inherit()
3169 if (vm_map_lookup_entry(map, end - 1, &lentry)) { in vm_map_inherit()
3170 rv = vm_map_clip_end(map, lentry, end); in vm_map_inherit()
3191 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3193 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_inherit()
3195 vm_map_unlock(map); in vm_map_inherit()
3208 vm_map_entry_in_transition(vm_map_t map, vm_offset_t in_start, in vm_map_entry_in_transition() argument
3215 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_in_transition()
3223 last_timestamp = map->timestamp; in vm_map_entry_in_transition()
3224 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_entry_in_transition()
3229 vm_map_lock(map); in vm_map_entry_in_transition()
3230 if (last_timestamp + 1 == map->timestamp) in vm_map_entry_in_transition()
3238 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_entry_in_transition()
3254 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_unwire() argument
3265 vm_map_lock(map); in vm_map_unwire()
3266 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire()
3267 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3271 vm_map_unlock(map); in vm_map_unwire()
3281 next_entry = vm_map_entry_in_transition(map, start, in vm_map_unwire()
3285 vm_map_unlock(map); in vm_map_unwire()
3295 rv = vm_map_clip_start(map, entry, start); in vm_map_unwire()
3298 rv = vm_map_clip_end(map, entry, end); in vm_map_unwire()
3334 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
3363 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3379 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3381 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_unwire()
3382 vm_map_unlock(map); in vm_map_unwire()
3384 vm_map_wakeup(map); in vm_map_unwire()
3418 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3422 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_entry_failure()
3434 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3447 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire() argument
3451 vm_map_lock(map); in vm_map_wire()
3452 rv = vm_map_wire_locked(map, start, end, flags); in vm_map_wire()
3453 vm_map_unlock(map); in vm_map_wire()
3464 vm_map_wire_locked(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) in vm_map_wire_locked() argument
3474 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_locked()
3483 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire_locked()
3484 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3495 next_entry = vm_map_entry_in_transition(map, start, in vm_map_wire_locked()
3507 rv = vm_map_clip_start(map, entry, start); in vm_map_wire_locked()
3510 rv = vm_map_clip_end(map, entry, end); in vm_map_wire_locked()
3536 vm_map_wire_entry_failure(map, entry, in vm_map_wire_locked()
3549 last_timestamp = map->timestamp; in vm_map_wire_locked()
3553 vm_map_busy(map); in vm_map_wire_locked()
3554 vm_map_unlock(map); in vm_map_wire_locked()
3562 rv = vm_fault(map, faddr, VM_PROT_NONE, in vm_map_wire_locked()
3567 vm_map_lock(map); in vm_map_wire_locked()
3568 vm_map_unbusy(map); in vm_map_wire_locked()
3569 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire_locked()
3576 if (!vm_map_lookup_entry(map, saved_start, in vm_map_wire_locked()
3592 vm_map_wire_entry_failure(map, in vm_map_wire_locked()
3597 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire_locked()
3623 !vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire_locked()
3671 vm_map_entry_unwire(map, entry); in vm_map_wire_locked()
3689 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3691 vm_map_try_merge_entries(map, prev_entry, entry); in vm_map_wire_locked()
3693 vm_map_wakeup(map); in vm_map_wire_locked()
3715 vm_map_t map, in vm_map_sync() argument
3729 vm_map_lock_read(map); in vm_map_sync()
3730 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_sync()
3731 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_sync()
3732 vm_map_unlock_read(map); in vm_map_sync()
3746 vm_map_unlock_read(map); in vm_map_sync()
3755 vm_map_unlock_read(map); in vm_map_sync()
3762 vm_map_unlock_read(map); in vm_map_sync()
3768 pmap_remove(map->pmap, start, end); in vm_map_sync()
3796 last_timestamp = map->timestamp; in vm_map_sync()
3797 vm_map_unlock_read(map); in vm_map_sync()
3802 vm_map_lock_read(map); in vm_map_sync()
3803 if (last_timestamp == map->timestamp || in vm_map_sync()
3804 !vm_map_lookup_entry(map, start, &entry)) in vm_map_sync()
3808 vm_map_unlock_read(map); in vm_map_sync()
3821 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3825 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unwire()
3832 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3853 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3859 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3866 vm_map_entry_deallocate(entry, map->system_map); in vm_map_entry_delete()
3871 map->size -= size; in vm_map_entry_delete()
3917 if (map->system_map) in vm_map_entry_delete()
3932 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_delete() argument
3937 VM_MAP_ASSERT_LOCKED(map); in vm_map_delete()
3946 rv = vm_map_lookup_clip_start(map, start, &entry, &scratch_entry); in vm_map_delete()
3956 (vm_map_pmap(map) != kernel_pmap && in vm_map_delete()
3963 last_timestamp = map->timestamp; in vm_map_delete()
3964 (void) vm_map_unlock_and_wait(map, 0); in vm_map_delete()
3965 vm_map_lock(map); in vm_map_delete()
3966 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
3973 rv = vm_map_lookup_clip_start(map, saved_start, in vm_map_delete()
3983 rv = vm_map_clip_end(map, entry, end); in vm_map_delete()
3993 vm_map_entry_unwire(map, entry); in vm_map_delete()
4002 pmap_remove(map->pmap, entry->start, entry->end); in vm_map_delete()
4004 if (entry->end == map->anon_loc) in vm_map_delete()
4005 map->anon_loc = entry->start; in vm_map_delete()
4013 vm_map_entry_delete(map, entry); in vm_map_delete()
4025 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_remove() argument
4029 vm_map_lock(map); in vm_map_remove()
4030 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_remove()
4031 result = vm_map_delete(map, start, end); in vm_map_remove()
4032 vm_map_unlock(map); in vm_map_remove()
4051 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_check_protection() argument
4057 if (!vm_map_lookup_entry(map, start, &tmp_entry)) in vm_map_check_protection()
4493 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack() argument
4500 MPASS((map->flags & MAP_WIREFUTURE) == 0); in vm_map_stack()
4503 vm_map_lock(map); in vm_map_stack()
4506 if (map->size + init_ssize > vmemlim) { in vm_map_stack()
4510 rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, in vm_map_stack()
4513 vm_map_unlock(map); in vm_map_stack()
4523 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack_locked() argument
4542 !vm_map_range_valid(map, addrbos, addrbos + max_ssize)) in vm_map_stack_locked()
4555 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) in vm_map_stack_locked()
4585 rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow); in vm_map_stack_locked()
4599 rv = vm_map_insert(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, in vm_map_stack_locked()
4614 (void)vm_map_delete(map, bot, top); in vm_map_stack_locked()
4624 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) in vm_map_growstack() argument
4650 if (p != initproc && (map != &p->p_vmspace->vm_map || in vm_map_growstack()
4654 MPASS(!map->system_map); in vm_map_growstack()
4661 if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) in vm_map_growstack()
4729 if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4730 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { in vm_map_growstack()
4738 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { in vm_map_growstack()
4749 if (map->size + grow_amount > vmemlim) { in vm_map_growstack()
4756 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { in vm_map_growstack()
4765 if (vm_map_lock_upgrade(map)) { in vm_map_growstack()
4767 vm_map_lock_read(map); in vm_map_growstack()
4776 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4780 vm_map_entry_resize(map, gap_entry, -grow_amount); in vm_map_growstack()
4783 rv = vm_map_insert(map, NULL, 0, grow_start, in vm_map_growstack()
4789 rv1 = vm_map_insert(map, NULL, 0, gap_start, in vm_map_growstack()
4794 vm_map_entry_resize(map, gap_entry, in vm_map_growstack()
4811 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4812 vm_map_entry_resize(map, stack_entry, in vm_map_growstack()
4818 map->size += grow_amount; in vm_map_growstack()
4829 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4830 rv = vm_map_wire_locked(map, grow_start, in vm_map_growstack()
4834 vm_map_lock_downgrade(map); in vm_map_growstack()
4840 error = racct_set(p, RACCT_VMEM, map->size); in vm_map_growstack()
4844 ptoa(pmap_wired_count(map->pmap))); in vm_map_growstack()
4951 vm_map_t map = *var_map; in vm_map_lookup() local
4960 vm_map_lock_read(map); in vm_map_lookup()
4966 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { in vm_map_lookup()
4967 vm_map_unlock_read(map); in vm_map_lookup()
4977 vm_map_t old_map = map; in vm_map_lookup()
4979 *var_map = map = entry->object.sub_map; in vm_map_lookup()
4990 if (prot == VM_PROT_NONE && map != kernel_map && in vm_map_lookup()
4994 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
4999 vm_map_unlock_read(map); in vm_map_lookup()
5009 vm_map_unlock_read(map); in vm_map_lookup()
5041 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5053 vm_map_unlock(map); in vm_map_lookup()
5071 vm_map_lock_downgrade(map); in vm_map_lookup()
5084 if (entry->object.vm_object == NULL && !map->system_map) { in vm_map_lookup()
5085 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
5091 vm_map_lock_downgrade(map); in vm_map_lookup()
5122 vm_map_t map = *var_map; in vm_map_lookup_locked() local
5129 if (!vm_map_lookup_entry(map, vaddr, out_entry)) in vm_map_lookup_locked()
5172 if (entry->object.vm_object == NULL && !map->system_map) in vm_map_lookup_locked()
5193 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
5198 vm_map_unlock_read(map); in vm_map_lookup_done()
5202 vm_map_max_KBI(const struct vm_map *map) in vm_map_max_KBI() argument
5205 return (vm_map_max(map)); in vm_map_max_KBI()
5209 vm_map_min_KBI(const struct vm_map *map) in vm_map_min_KBI() argument
5212 return (vm_map_min(map)); in vm_map_min_KBI()
5216 vm_map_pmap_KBI(vm_map_t map) in vm_map_pmap_KBI() argument
5219 return (map->pmap); in vm_map_pmap_KBI()
5223 vm_map_range_valid_KBI(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_range_valid_KBI() argument
5226 return (vm_map_range_valid(map, start, end)); in vm_map_range_valid_KBI()
5231 _vm_map_assert_consistent(vm_map_t map, int check) in _vm_map_assert_consistent() argument
5238 ++map->nupdates; in _vm_map_assert_consistent()
5243 header = prev = &map->header; in _vm_map_assert_consistent()
5244 VM_MAP_ENTRY_FOREACH(entry, map) { in _vm_map_assert_consistent()
5246 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5249 ("map %p start = %jx, end = %jx", map, in _vm_map_assert_consistent()
5253 ("map %p left->start = %jx, start = %jx", map, in _vm_map_assert_consistent()
5257 ("map %p start = %jx, right->start = %jx", map, in _vm_map_assert_consistent()
5259 cur = map->root; in _vm_map_assert_consistent()
5267 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5273 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5277 map, (uintmax_t)entry->start)); in _vm_map_assert_consistent()
5284 ("map %p max = %jx, max_left = %jx, max_right = %jx", map, in _vm_map_assert_consistent()
5290 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
5302 vm_map_print(vm_map_t map) in vm_map_print() argument
5307 (void *)map, in vm_map_print()
5308 (void *)map->pmap, map->nentries, map->timestamp); in vm_map_print()
5311 prev = &map->header; in vm_map_print()
5312 VM_MAP_ENTRY_FOREACH(entry, map) { in vm_map_print()
5332 if (prev == &map->header || in vm_map_print()
5354 if (prev == &map->header || in vm_map_print()
5369 DB_SHOW_COMMAND(map, map) in DB_SHOW_COMMAND() argument