Lines Matching refs:map
133 static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min,
136 static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry);
137 static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry);
138 static int vm_map_growstack(vm_map_t map, vm_offset_t addr,
140 static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
146 static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos,
149 static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry,
169 #define VM_MAP_RANGE_CHECK(map, start, end) \ argument
171 if (start < vm_map_min(map)) \
172 start = vm_map_min(map); \
173 if (end > vm_map_max(map)) \
174 end = vm_map_max(map); \
237 vm_map_t map; in vm_map_zinit() local
239 map = (vm_map_t)mem; in vm_map_zinit()
240 memset(map, 0, sizeof(*map)); in vm_map_zinit()
241 mtx_init(&map->system_mtx, "vm map (system)", NULL, MTX_DEF | MTX_DUPOK); in vm_map_zinit()
242 sx_init(&map->lock, "vm map (user)"); in vm_map_zinit()
259 vm_map_t map; in vm_map_zdtor() local
261 map = (vm_map_t)mem; in vm_map_zdtor()
262 KASSERT(map->nentries == 0, in vm_map_zdtor()
264 map, map->nentries)); in vm_map_zdtor()
265 KASSERT(map->size == 0, in vm_map_zdtor()
267 map, (unsigned long)map->size)); in vm_map_zdtor()
495 _vm_map_lock(vm_map_t map, const char *file, int line) in _vm_map_lock() argument
498 if (map->system_map) in _vm_map_lock()
499 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock()
501 sx_xlock_(&map->lock, file, line); in _vm_map_lock()
502 map->timestamp++; in _vm_map_lock()
599 _vm_map_unlock(vm_map_t map, const char *file, int line) in _vm_map_unlock() argument
602 if (map->system_map) in _vm_map_unlock()
603 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock()
605 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock()
611 _vm_map_lock_read(vm_map_t map, const char *file, int line) in _vm_map_lock_read() argument
614 if (map->system_map) in _vm_map_lock_read()
615 mtx_lock_flags_(&map->system_mtx, 0, file, line); in _vm_map_lock_read()
617 sx_slock_(&map->lock, file, line); in _vm_map_lock_read()
621 _vm_map_unlock_read(vm_map_t map, const char *file, int line) in _vm_map_unlock_read() argument
624 if (map->system_map) in _vm_map_unlock_read()
625 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_read()
627 sx_sunlock_(&map->lock, file, line); in _vm_map_unlock_read()
633 _vm_map_trylock(vm_map_t map, const char *file, int line) in _vm_map_trylock() argument
637 error = map->system_map ? in _vm_map_trylock()
638 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock()
639 !sx_try_xlock_(&map->lock, file, line); in _vm_map_trylock()
641 map->timestamp++; in _vm_map_trylock()
646 _vm_map_trylock_read(vm_map_t map, const char *file, int line) in _vm_map_trylock_read() argument
650 error = map->system_map ? in _vm_map_trylock_read()
651 !mtx_trylock_flags_(&map->system_mtx, 0, file, line) : in _vm_map_trylock_read()
652 !sx_try_slock_(&map->lock, file, line); in _vm_map_trylock_read()
667 _vm_map_lock_upgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_upgrade() argument
671 if (map->system_map) { in _vm_map_lock_upgrade()
672 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_upgrade()
674 if (!sx_try_upgrade_(&map->lock, file, line)) { in _vm_map_lock_upgrade()
675 last_timestamp = map->timestamp; in _vm_map_lock_upgrade()
676 sx_sunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
682 sx_xlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
683 if (last_timestamp != map->timestamp) { in _vm_map_lock_upgrade()
684 sx_xunlock_(&map->lock, file, line); in _vm_map_lock_upgrade()
689 map->timestamp++; in _vm_map_lock_upgrade()
694 _vm_map_lock_downgrade(vm_map_t map, const char *file, int line) in _vm_map_lock_downgrade() argument
697 if (map->system_map) { in _vm_map_lock_downgrade()
698 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_lock_downgrade()
700 sx_downgrade_(&map->lock, file, line); in _vm_map_lock_downgrade()
710 vm_map_locked(vm_map_t map) in vm_map_locked() argument
713 if (map->system_map) in vm_map_locked()
714 return (mtx_owned(&map->system_mtx)); in vm_map_locked()
716 return (sx_xlocked(&map->lock)); in vm_map_locked()
721 _vm_map_assert_locked(vm_map_t map, const char *file, int line) in _vm_map_assert_locked() argument
724 if (map->system_map) in _vm_map_assert_locked()
725 mtx_assert_(&map->system_mtx, MA_OWNED, file, line); in _vm_map_assert_locked()
727 sx_assert_(&map->lock, SA_XLOCKED, file, line); in _vm_map_assert_locked()
730 #define VM_MAP_ASSERT_LOCKED(map) \ argument
731 _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE)
742 _vm_map_assert_consistent(vm_map_t map) in _vm_map_assert_consistent() argument
751 for (entry = map->header.next; entry != &map->header; in _vm_map_assert_consistent()
754 ("map %p prev->end = %jx, start = %jx", map, in _vm_map_assert_consistent()
757 ("map %p start = %jx, end = %jx", map, in _vm_map_assert_consistent()
760 ("map %p end = %jx, next->start = %jx", map, in _vm_map_assert_consistent()
764 ("map %p left->start = %jx, start = %jx", map, in _vm_map_assert_consistent()
768 ("map %p start = %jx, right->start = %jx", map, in _vm_map_assert_consistent()
777 ("map %p max = %jx, max_left = %jx, max_right = %jx", map, in _vm_map_assert_consistent()
783 #define VM_MAP_ASSERT_CONSISTENT(map) \ argument
784 _vm_map_assert_consistent(map)
786 #define VM_MAP_ASSERT_LOCKED(map) argument
787 #define VM_MAP_ASSERT_CONSISTENT(map) argument
805 _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) in _vm_map_unlock_and_wait() argument
809 if (map->system_map) in _vm_map_unlock_and_wait()
810 mtx_unlock_flags_(&map->system_mtx, 0, file, line); in _vm_map_unlock_and_wait()
812 sx_xunlock_(&map->lock, file, line); in _vm_map_unlock_and_wait()
813 return (msleep(&map->root, &map_sleep_mtx, PDROP | PVM, "vmmaps", in _vm_map_unlock_and_wait()
824 vm_map_wakeup(vm_map_t map) in vm_map_wakeup() argument
834 wakeup(&map->root); in vm_map_wakeup()
838 vm_map_busy(vm_map_t map) in vm_map_busy() argument
841 VM_MAP_ASSERT_LOCKED(map); in vm_map_busy()
842 map->busy++; in vm_map_busy()
846 vm_map_unbusy(vm_map_t map) in vm_map_unbusy() argument
849 VM_MAP_ASSERT_LOCKED(map); in vm_map_unbusy()
850 KASSERT(map->busy, ("vm_map_unbusy: not busy")); in vm_map_unbusy()
851 if (--map->busy == 0 && (map->flags & MAP_BUSY_WAKEUP)) { in vm_map_unbusy()
852 vm_map_modflags(map, 0, MAP_BUSY_WAKEUP); in vm_map_unbusy()
853 wakeup(&map->busy); in vm_map_unbusy()
858 vm_map_wait_busy(vm_map_t map) in vm_map_wait_busy() argument
861 VM_MAP_ASSERT_LOCKED(map); in vm_map_wait_busy()
862 while (map->busy) { in vm_map_wait_busy()
863 vm_map_modflags(map, MAP_BUSY_WAKEUP, 0); in vm_map_wait_busy()
864 if (map->system_map) in vm_map_wait_busy()
865 msleep(&map->busy, &map->system_mtx, 0, "mbusy", 0); in vm_map_wait_busy()
867 sx_sleep(&map->busy, &map->lock, 0, "mbusy", 0); in vm_map_wait_busy()
869 map->timestamp++; in vm_map_wait_busy()
901 _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in _vm_map_init() argument
904 map->header.next = map->header.prev = &map->header; in _vm_map_init()
905 map->header.eflags = MAP_ENTRY_HEADER; in _vm_map_init()
906 map->needs_wakeup = FALSE; in _vm_map_init()
907 map->system_map = 0; in _vm_map_init()
908 map->pmap = pmap; in _vm_map_init()
909 map->header.end = min; in _vm_map_init()
910 map->header.start = max; in _vm_map_init()
911 map->flags = 0; in _vm_map_init()
912 map->root = NULL; in _vm_map_init()
913 map->timestamp = 0; in _vm_map_init()
914 map->busy = 0; in _vm_map_init()
915 map->anon_loc = 0; in _vm_map_init()
919 vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max) in vm_map_init() argument
922 _vm_map_init(map, pmap, min, max); in vm_map_init()
923 mtx_init(&map->system_mtx, "system map", NULL, MTX_DEF | MTX_DUPOK); in vm_map_init()
924 sx_init(&map->lock, "user map"); in vm_map_init()
933 vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_dispose() argument
935 uma_zfree(map->system_map ? kmapentzone : mapentzone, entry); in vm_map_entry_dispose()
945 vm_map_entry_create(vm_map_t map) in vm_map_entry_create() argument
949 if (map->system_map) in vm_map_entry_create()
1071 vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, in vm_map_splay_split() argument
1076 llist = rlist = &map->header; in vm_map_splay_split()
1077 root = map->root; in vm_map_splay_split()
1134 vm_map_splay_merge(vm_map_t map, vm_map_entry_t root, in vm_map_splay_merge() argument
1141 if (llist != &map->header) { in vm_map_splay_merge()
1153 } while (llist != &map->header); in vm_map_splay_merge()
1157 if (rlist != &map->header) { in vm_map_splay_merge()
1169 } while (rlist != &map->header); in vm_map_splay_merge()
1173 map->root = root; in vm_map_splay_merge()
1194 vm_map_splay(vm_map_t map, vm_offset_t addr) in vm_map_splay() argument
1198 root = vm_map_splay_split(map, addr, 0, &llist, &rlist); in vm_map_splay()
1201 } else if (llist != &map->header) { in vm_map_splay()
1209 } else if (rlist != &map->header) { in vm_map_splay()
1221 vm_map_splay_merge(map, root, llist, rlist); in vm_map_splay()
1222 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_splay()
1232 vm_map_entry_link(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_link() argument
1237 "vm_map_entry_link: map %p, nentries %d, entry %p", map, in vm_map_entry_link()
1238 map->nentries, entry); in vm_map_entry_link()
1239 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_link()
1240 map->nentries++; in vm_map_entry_link()
1241 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_link()
1248 vm_map_splay_merge(map, entry, llist, rlist); in vm_map_entry_link()
1249 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_link()
1259 vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry, in vm_map_entry_unlink() argument
1264 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unlink()
1265 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_unlink()
1290 if (llist != &map->header) { in vm_map_entry_unlink()
1294 } else if (rlist != &map->header) { in vm_map_entry_unlink()
1306 vm_map_splay_merge(map, root, llist, rlist); in vm_map_entry_unlink()
1308 map->root = NULL; in vm_map_entry_unlink()
1309 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_unlink()
1310 map->nentries--; in vm_map_entry_unlink()
1311 CTR3(KTR_VM, "vm_map_entry_unlink: map %p, nentries %d, entry %p", map, in vm_map_entry_unlink()
1312 map->nentries, entry); in vm_map_entry_unlink()
1326 vm_map_entry_resize_free(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_resize_free() argument
1330 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_resize_free()
1331 root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); in vm_map_entry_resize_free()
1336 vm_map_splay_merge(map, root, llist, rlist); in vm_map_entry_resize_free()
1337 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_entry_resize_free()
1338 CTR3(KTR_VM, "vm_map_entry_resize_free: map %p, nentries %d, entry %p", map, in vm_map_entry_resize_free()
1339 map->nentries, entry); in vm_map_entry_resize_free()
1354 vm_map_t map, in vm_map_lookup_entry() argument
1365 cur = map->root; in vm_map_lookup_entry()
1367 *entry = &map->header; in vm_map_lookup_entry()
1374 if ((locked = vm_map_locked(map)) || in vm_map_lookup_entry()
1375 sx_try_upgrade(&map->lock)) { in vm_map_lookup_entry()
1382 cur = vm_map_splay(map, address); in vm_map_lookup_entry()
1384 sx_downgrade(&map->lock); in vm_map_lookup_entry()
1392 *entry = &map->header; in vm_map_lookup_entry()
1402 lbound = &map->header; in vm_map_lookup_entry()
1431 vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_insert() argument
1439 VM_MAP_ASSERT_LOCKED(map); in vm_map_insert()
1451 if (start < vm_map_min(map) || end > vm_map_max(map) || in vm_map_insert()
1459 if (vm_map_lookup_entry(map, start, &temp_entry)) in vm_map_insert()
1518 if (map == kernel_map && end > kernel_vm_end) in vm_map_insert()
1557 map->size += end - prev_entry->end; in vm_map_insert()
1559 vm_map_entry_resize_free(map, prev_entry); in vm_map_insert()
1560 vm_map_simplify_entry(map, prev_entry); in vm_map_insert()
1586 new_entry = vm_map_entry_create(map); in vm_map_insert()
1610 vm_map_entry_link(map, new_entry); in vm_map_insert()
1612 map->size += new_entry->end - new_entry->start; in vm_map_insert()
1620 vm_map_simplify_entry(map, new_entry); in vm_map_insert()
1623 vm_map_pmap_enter(map, start, prot, object, OFF_TO_IDX(offset), in vm_map_insert()
1648 vm_map_findspace(vm_map_t map, vm_offset_t start, vm_size_t length) in vm_map_findspace() argument
1658 start = MAX(start, vm_map_min(map)); in vm_map_findspace()
1659 if (start >= vm_map_max(map) || length > vm_map_max(map) - start) in vm_map_findspace()
1660 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1663 if (map->root == NULL) in vm_map_findspace()
1673 root = vm_map_splay_split(map, start, length, &llist, &rlist); in vm_map_findspace()
1679 } else if (rlist != &map->header) { in vm_map_findspace()
1688 vm_map_splay_merge(map, root, llist, rlist); in vm_map_findspace()
1689 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1695 return (vm_map_max(map) - length + 1); in vm_map_findspace()
1700 llist = rlist = &map->header; in vm_map_findspace()
1715 if (rlist != &map->header) { in vm_map_findspace()
1719 vm_map_splay_merge(map, y, &map->header, rlist); in vm_map_findspace()
1722 vm_map_entry_max_free_right(y, &map->header)); in vm_map_findspace()
1725 vm_map_splay_merge(map, root, llist, &map->header); in vm_map_findspace()
1726 VM_MAP_ASSERT_CONSISTENT(map); in vm_map_findspace()
1731 vm_map_fixed(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_fixed() argument
1742 vm_map_lock(map); in vm_map_fixed()
1743 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_fixed()
1745 vm_map_delete(map, start, end); in vm_map_fixed()
1747 result = vm_map_stack_locked(map, start, length, sgrowsiz, in vm_map_fixed()
1750 result = vm_map_insert(map, object, offset, start, end, in vm_map_fixed()
1753 vm_map_unlock(map); in vm_map_fixed()
1800 vm_map_alignspace(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_alignspace() argument
1806 VM_MAP_ASSERT_LOCKED(map); in vm_map_alignspace()
1808 KASSERT(free_addr == vm_map_findspace(map, free_addr, length), in vm_map_alignspace()
1838 *addr = vm_map_findspace(map, aligned_addr, length); in vm_map_alignspace()
1839 if (*addr + length > vm_map_max(map) || in vm_map_alignspace()
1864 vm_map_find(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find() argument
1886 en_aslr = (map->flags & MAP_ASLR) != 0; in vm_map_find()
1888 (map->flags & MAP_IS_SUB_MAP) == 0 && max_addr == 0 && in vm_map_find()
1895 (map->flags & MAP_ASLR_IGNSTART) != 0) in vm_map_find()
1896 curr_min_addr = min_addr = vm_map_min(map); in vm_map_find()
1898 vm_map_lock(map); in vm_map_find()
1900 curr_min_addr = map->anon_loc; in vm_map_find()
1935 curr_min_addr = (map->flags & MAP_ASLR_IGNSTART) != 0 ? in vm_map_find()
1936 vm_map_min(map) : min_addr; in vm_map_find()
1948 gap = vm_map_max(map) > MAP_32BIT_MAX_ADDR && in vm_map_find()
1951 *addr = vm_map_findspace(map, curr_min_addr, in vm_map_find()
1954 vm_map_max(map)) in vm_map_find()
1961 *addr = vm_map_findspace(map, curr_min_addr, length); in vm_map_find()
1962 if (*addr + length > vm_map_max(map) || in vm_map_find()
1975 (rv = vm_map_alignspace(map, object, offset, addr, length, in vm_map_find()
1987 if (*addr < vm_map_min(map) || in vm_map_find()
1988 *addr + length > vm_map_max(map) || in vm_map_find()
1993 vm_map_delete(map, *addr, *addr + length); in vm_map_find()
1996 rv = vm_map_stack_locked(map, *addr, length, sgrowsiz, prot, in vm_map_find()
1999 rv = vm_map_insert(map, object, offset, *addr, *addr + length, in vm_map_find()
2003 map->anon_loc = *addr + length; in vm_map_find()
2005 vm_map_unlock(map); in vm_map_find()
2022 vm_map_find_min(vm_map_t map, vm_object_t object, vm_ooffset_t offset, in vm_map_find_min() argument
2032 rv = vm_map_find(map, object, offset, addr, length, max_addr, in vm_map_find_min()
2068 vm_map_merged_neighbor_dispose(vm_map_t map, vm_map_entry_t entry) in vm_map_merged_neighbor_dispose() argument
2085 vm_map_entry_dispose(map, entry); in vm_map_merged_neighbor_dispose()
2101 vm_map_simplify_entry(vm_map_t map, vm_map_entry_t entry) in vm_map_simplify_entry() argument
2109 vm_map_entry_unlink(map, prev, UNLINK_MERGE_NEXT); in vm_map_simplify_entry()
2110 vm_map_merged_neighbor_dispose(map, prev); in vm_map_simplify_entry()
2114 vm_map_entry_unlink(map, next, UNLINK_MERGE_PREV); in vm_map_simplify_entry()
2115 vm_map_merged_neighbor_dispose(map, next); in vm_map_simplify_entry()
2126 #define vm_map_clip_start(map, entry, startaddr) \ argument
2129 _vm_map_clip_start(map, entry, startaddr); \
2137 _vm_map_clip_start(vm_map_t map, vm_map_entry_t entry, vm_offset_t start) in _vm_map_clip_start() argument
2141 VM_MAP_ASSERT_LOCKED(map); in _vm_map_clip_start()
2150 vm_map_simplify_entry(map, entry); in _vm_map_clip_start()
2159 if (entry->object.vm_object == NULL && !map->system_map && in _vm_map_clip_start()
2183 new_entry = vm_map_entry_create(map); in _vm_map_clip_start()
2192 vm_map_entry_link(map, new_entry); in _vm_map_clip_start()
2214 #define vm_map_clip_end(map, entry, endaddr) \ argument
2217 _vm_map_clip_end((map), (entry), (endaddr)); \
2225 _vm_map_clip_end(vm_map_t map, vm_map_entry_t entry, vm_offset_t end) in _vm_map_clip_end() argument
2229 VM_MAP_ASSERT_LOCKED(map); in _vm_map_clip_end()
2240 if (entry->object.vm_object == NULL && !map->system_map && in _vm_map_clip_end()
2267 new_entry = vm_map_entry_create(map); in _vm_map_clip_end()
2275 vm_map_entry_link(map, new_entry); in _vm_map_clip_end()
2303 vm_map_t map, in vm_map_submap() argument
2317 vm_map_lock(map); in vm_map_submap()
2319 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_submap()
2321 if (vm_map_lookup_entry(map, start, &entry)) { in vm_map_submap()
2322 vm_map_clip_start(map, entry, start); in vm_map_submap()
2326 vm_map_clip_end(map, entry, end); in vm_map_submap()
2335 vm_map_unlock(map); in vm_map_submap()
2363 vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, in vm_map_pmap_enter() argument
2377 pmap_object_init_pt(map->pmap, addr, object, pindex, in vm_map_pmap_enter()
2434 pmap_enter_object(map->pmap, start, addr + in vm_map_pmap_enter()
2440 pmap_enter_object(map->pmap, start, addr + ptoa(psize), in vm_map_pmap_enter()
2454 vm_map_protect(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_protect() argument
2467 vm_map_lock(map); in vm_map_protect()
2475 vm_map_wait_busy(map); in vm_map_protect()
2477 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_protect()
2479 if (vm_map_lookup_entry(map, start, &entry)) { in vm_map_protect()
2480 vm_map_clip_start(map, entry, start); in vm_map_protect()
2492 vm_map_unlock(map); in vm_map_protect()
2496 vm_map_unlock(map); in vm_map_protect()
2513 vm_map_unlock_and_wait(map, 0); in vm_map_protect()
2524 vm_map_clip_end(map, current, end); in vm_map_protect()
2538 vm_map_unlock(map); in vm_map_protect()
2562 vm_map_unlock(map); in vm_map_protect()
2598 vm_fault_copy_entry(map, map, current, current, NULL); in vm_map_protect()
2607 pmap_protect(map->pmap, current->start, in vm_map_protect()
2612 vm_map_simplify_entry(map, current); in vm_map_protect()
2614 vm_map_unlock(map); in vm_map_protect()
2628 vm_map_t map, in vm_map_madvise() argument
2653 vm_map_lock(map); in vm_map_madvise()
2661 vm_map_lock_read(map); in vm_map_madvise()
2670 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_madvise()
2672 if (vm_map_lookup_entry(map, start, &entry)) { in vm_map_madvise()
2674 vm_map_clip_start(map, entry, start); in vm_map_madvise()
2691 vm_map_clip_end(map, current, end); in vm_map_madvise()
2718 vm_map_simplify_entry(map, current); in vm_map_madvise()
2720 vm_map_unlock(map); in vm_map_madvise()
2778 pmap_advise(map->pmap, useStart, useEnd, in vm_map_madvise()
2791 vm_map_pmap_enter(map, in vm_map_madvise()
2801 vm_map_unlock_read(map); in vm_map_madvise()
2816 vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_inherit() argument
2833 vm_map_lock(map); in vm_map_inherit()
2834 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_inherit()
2835 if (vm_map_lookup_entry(map, start, &temp_entry)) { in vm_map_inherit()
2837 vm_map_clip_start(map, entry, start); in vm_map_inherit()
2841 vm_map_clip_end(map, entry, end); in vm_map_inherit()
2845 vm_map_simplify_entry(map, entry); in vm_map_inherit()
2848 vm_map_unlock(map); in vm_map_inherit()
2858 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_unwire() argument
2870 vm_map_lock(map); in vm_map_unwire()
2871 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_unwire()
2872 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_unwire()
2876 vm_map_unlock(map); in vm_map_unwire()
2880 last_timestamp = map->timestamp; in vm_map_unwire()
2890 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_unwire()
2895 vm_map_lock(map); in vm_map_unwire()
2896 if (last_timestamp+1 != map->timestamp) { in vm_map_unwire()
2903 if (!vm_map_lookup_entry(map, saved_start, in vm_map_unwire()
2912 vm_map_unlock(map); in vm_map_unwire()
2926 last_timestamp = map->timestamp; in vm_map_unwire()
2929 vm_map_clip_start(map, entry, start); in vm_map_unwire()
2930 vm_map_clip_end(map, entry, end); in vm_map_unwire()
2965 result = vm_map_lookup_entry(map, start, &first_entry); in vm_map_unwire()
2993 vm_map_entry_unwire(map, entry); in vm_map_unwire()
3007 vm_map_simplify_entry(map, entry); in vm_map_unwire()
3009 vm_map_unlock(map); in vm_map_unwire()
3011 vm_map_wakeup(map); in vm_map_unwire()
3023 vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, in vm_map_wire_entry_failure() argument
3027 VM_MAP_ASSERT_LOCKED(map); in vm_map_wire_entry_failure()
3039 pmap_unwire(map->pmap, entry->start, failed_addr); in vm_map_wire_entry_failure()
3057 vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_wire() argument
3073 vm_map_lock(map); in vm_map_wire()
3074 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_wire()
3075 if (!vm_map_lookup_entry(map, start, &first_entry)) { in vm_map_wire()
3079 vm_map_unlock(map); in vm_map_wire()
3083 last_timestamp = map->timestamp; in vm_map_wire()
3093 if (vm_map_unlock_and_wait(map, 0)) { in vm_map_wire()
3098 vm_map_lock(map); in vm_map_wire()
3099 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire()
3106 if (!vm_map_lookup_entry(map, saved_start, in vm_map_wire()
3115 vm_map_unlock(map); in vm_map_wire()
3129 last_timestamp = map->timestamp; in vm_map_wire()
3132 vm_map_clip_start(map, entry, start); in vm_map_wire()
3133 vm_map_clip_end(map, entry, end); in vm_map_wire()
3162 vm_map_busy(map); in vm_map_wire()
3163 vm_map_unlock(map); in vm_map_wire()
3171 if ((rv = vm_fault(map, faddr, VM_PROT_NONE, in vm_map_wire()
3175 vm_map_lock(map); in vm_map_wire()
3176 vm_map_unbusy(map); in vm_map_wire()
3177 if (last_timestamp + 1 != map->timestamp) { in vm_map_wire()
3184 result = vm_map_lookup_entry(map, saved_start, in vm_map_wire()
3201 vm_map_wire_entry_failure(map, in vm_map_wire()
3206 last_timestamp = map->timestamp; in vm_map_wire()
3208 vm_map_wire_entry_failure(map, entry, faddr); in vm_map_wire()
3233 result = vm_map_lookup_entry(map, start, &first_entry); in vm_map_wire()
3275 vm_map_entry_unwire(map, entry); in vm_map_wire()
3291 vm_map_simplify_entry(map, entry); in vm_map_wire()
3293 vm_map_unlock(map); in vm_map_wire()
3295 vm_map_wakeup(map); in vm_map_wire()
3317 vm_map_t map, in vm_map_sync() argument
3331 vm_map_lock_read(map); in vm_map_sync()
3332 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_sync()
3333 if (!vm_map_lookup_entry(map, start, &entry)) { in vm_map_sync()
3334 vm_map_unlock_read(map); in vm_map_sync()
3345 vm_map_unlock_read(map); in vm_map_sync()
3350 vm_map_unlock_read(map); in vm_map_sync()
3356 pmap_remove(map->pmap, start, end); in vm_map_sync()
3384 last_timestamp = map->timestamp; in vm_map_sync()
3385 vm_map_unlock_read(map); in vm_map_sync()
3390 vm_map_lock_read(map); in vm_map_sync()
3391 if (last_timestamp == map->timestamp || in vm_map_sync()
3392 !vm_map_lookup_entry(map, start, ¤t)) in vm_map_sync()
3396 vm_map_unlock_read(map); in vm_map_sync()
3409 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_unwire() argument
3412 VM_MAP_ASSERT_LOCKED(map); in vm_map_entry_unwire()
3415 pmap_unwire(map->pmap, entry->start, entry->end); in vm_map_entry_unwire()
3436 vm_map_entry_delete(vm_map_t map, vm_map_entry_t entry) in vm_map_entry_delete() argument
3442 vm_map_entry_unlink(map, entry, UNLINK_MERGE_NONE); in vm_map_entry_delete()
3449 vm_map_entry_deallocate(entry, map->system_map); in vm_map_entry_delete()
3454 map->size -= size; in vm_map_entry_delete()
3503 if (map->system_map) in vm_map_entry_delete()
3518 vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_delete() argument
3523 VM_MAP_ASSERT_LOCKED(map); in vm_map_delete()
3530 if (!vm_map_lookup_entry(map, start, &first_entry)) in vm_map_delete()
3534 vm_map_clip_start(map, entry, start); in vm_map_delete()
3549 (vm_map_pmap(map) != kernel_pmap && in vm_map_delete()
3557 last_timestamp = map->timestamp; in vm_map_delete()
3558 (void) vm_map_unlock_and_wait(map, 0); in vm_map_delete()
3559 vm_map_lock(map); in vm_map_delete()
3560 if (last_timestamp + 1 != map->timestamp) { in vm_map_delete()
3567 if (!vm_map_lookup_entry(map, saved_start, in vm_map_delete()
3572 vm_map_clip_start(map, entry, in vm_map_delete()
3578 vm_map_clip_end(map, entry, end); in vm_map_delete()
3587 vm_map_entry_unwire(map, entry); in vm_map_delete()
3596 pmap_remove(map->pmap, entry->start, entry->end); in vm_map_delete()
3598 if (entry->end == map->anon_loc) in vm_map_delete()
3599 map->anon_loc = entry->start; in vm_map_delete()
3607 vm_map_entry_delete(map, entry); in vm_map_delete()
3620 vm_map_remove(vm_map_t map, vm_offset_t start, vm_offset_t end) in vm_map_remove() argument
3624 vm_map_lock(map); in vm_map_remove()
3625 VM_MAP_RANGE_CHECK(map, start, end); in vm_map_remove()
3626 result = vm_map_delete(map, start, end); in vm_map_remove()
3627 vm_map_unlock(map); in vm_map_remove()
3646 vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, in vm_map_check_protection() argument
3652 if (!vm_map_lookup_entry(map, start, &tmp_entry)) in vm_map_check_protection()
4065 vm_map_stack(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack() argument
4072 MPASS((map->flags & MAP_WIREFUTURE) == 0); in vm_map_stack()
4075 vm_map_lock(map); in vm_map_stack()
4078 if (map->size + init_ssize > vmemlim) { in vm_map_stack()
4082 rv = vm_map_stack_locked(map, addrbos, max_ssize, growsize, prot, in vm_map_stack()
4085 vm_map_unlock(map); in vm_map_stack()
4095 vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, in vm_map_stack_locked() argument
4113 if (addrbos < vm_map_min(map) || in vm_map_stack_locked()
4114 addrbos + max_ssize > vm_map_max(map) || in vm_map_stack_locked()
4127 if (vm_map_lookup_entry(map, addrbos, &prev_entry)) in vm_map_stack_locked()
4157 rv = vm_map_insert(map, NULL, 0, bot, top, prot, max, cow); in vm_map_stack_locked()
4171 rv = vm_map_insert(map, NULL, 0, gap_bot, gap_top, VM_PROT_NONE, in vm_map_stack_locked()
4186 (void)vm_map_delete(map, bot, top); in vm_map_stack_locked()
4196 vm_map_growstack(vm_map_t map, vm_offset_t addr, vm_map_entry_t gap_entry) in vm_map_growstack() argument
4222 if (p != initproc && (map != &p->p_vmspace->vm_map || in vm_map_growstack()
4226 MPASS(!map->system_map); in vm_map_growstack()
4233 if (gap_entry == NULL && !vm_map_lookup_entry(map, addr, &gap_entry)) in vm_map_growstack()
4300 if (!old_mlock && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4301 if (ptoa(pmap_wired_count(map->pmap)) + grow_amount > lmemlim) { in vm_map_growstack()
4309 ptoa(pmap_wired_count(map->pmap)) + grow_amount)) { in vm_map_growstack()
4320 if (map->size + grow_amount > vmemlim) { in vm_map_growstack()
4327 if (racct_set(p, RACCT_VMEM, map->size + grow_amount)) { in vm_map_growstack()
4336 if (vm_map_lock_upgrade(map)) { in vm_map_growstack()
4338 vm_map_lock_read(map); in vm_map_growstack()
4347 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4352 vm_map_entry_resize_free(map, gap_entry); in vm_map_growstack()
4355 rv = vm_map_insert(map, NULL, 0, grow_start, in vm_map_growstack()
4361 rv1 = vm_map_insert(map, NULL, 0, gap_start, in vm_map_growstack()
4367 vm_map_entry_resize_free(map, gap_entry); in vm_map_growstack()
4384 vm_map_entry_delete(map, gap_entry); in vm_map_growstack()
4388 map->size += grow_amount; in vm_map_growstack()
4389 vm_map_entry_resize_free(map, stack_entry); in vm_map_growstack()
4400 if (rv == KERN_SUCCESS && (map->flags & MAP_WIREFUTURE) != 0) { in vm_map_growstack()
4401 vm_map_unlock(map); in vm_map_growstack()
4402 vm_map_wire(map, grow_start, grow_start + grow_amount, in vm_map_growstack()
4404 vm_map_lock_read(map); in vm_map_growstack()
4406 vm_map_lock_downgrade(map); in vm_map_growstack()
4412 error = racct_set(p, RACCT_VMEM, map->size); in vm_map_growstack()
4416 ptoa(pmap_wired_count(map->pmap))); in vm_map_growstack()
4523 vm_map_t map = *var_map; in vm_map_lookup() local
4532 vm_map_lock_read(map); in vm_map_lookup()
4538 if (!vm_map_lookup_entry(map, vaddr, out_entry)) { in vm_map_lookup()
4539 vm_map_unlock_read(map); in vm_map_lookup()
4549 vm_map_t old_map = map; in vm_map_lookup()
4551 *var_map = map = entry->object.sub_map; in vm_map_lookup()
4562 if (prot == VM_PROT_NONE && map != kernel_map && in vm_map_lookup()
4566 vm_map_growstack(map, vaddr, entry) == KERN_SUCCESS) in vm_map_lookup()
4571 vm_map_unlock_read(map); in vm_map_lookup()
4581 vm_map_unlock_read(map); in vm_map_lookup()
4612 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
4624 vm_map_unlock(map); in vm_map_lookup()
4648 vm_map_lock_downgrade(map); in vm_map_lookup()
4662 !map->system_map) { in vm_map_lookup()
4663 if (vm_map_lock_upgrade(map)) in vm_map_lookup()
4675 vm_map_lock_downgrade(map); in vm_map_lookup()
4706 vm_map_t map = *var_map; in vm_map_lookup_locked() local
4713 if (!vm_map_lookup_entry(map, vaddr, out_entry)) in vm_map_lookup_locked()
4756 if (entry->object.vm_object == NULL && !map->system_map) in vm_map_lookup_locked()
4777 vm_map_lookup_done(vm_map_t map, vm_map_entry_t entry) in vm_map_lookup_done() argument
4782 vm_map_unlock_read(map); in vm_map_lookup_done()
4786 vm_map_max_KBI(const struct vm_map *map) in vm_map_max_KBI() argument
4789 return (vm_map_max(map)); in vm_map_max_KBI()
4793 vm_map_min_KBI(const struct vm_map *map) in vm_map_min_KBI() argument
4796 return (vm_map_min(map)); in vm_map_min_KBI()
4800 vm_map_pmap_KBI(vm_map_t map) in vm_map_pmap_KBI() argument
4803 return (map->pmap); in vm_map_pmap_KBI()
4813 vm_map_print(vm_map_t map) in vm_map_print() argument
4818 (void *)map, in vm_map_print()
4819 (void *)map->pmap, map->nentries, map->timestamp); in vm_map_print()
4822 for (entry = map->header.next; entry != &map->header; in vm_map_print()
4842 if ((entry->prev == &map->header) || in vm_map_print()
4864 if ((entry->prev == &map->header) || in vm_map_print()
4878 DB_SHOW_COMMAND(map, map) in DB_SHOW_COMMAND() argument