115 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
120 static void	vm_object_backing_remove(vm_object_t object);
153 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
181 	vm_object_t object;  in vm_object_zdtor()  local
183 	object = (vm_object_t)mem;  in vm_object_zdtor()
184 	KASSERT(object->ref_count == 0,  in vm_object_zdtor()
185 	    ("object %p ref_count = %d", object, object->ref_count));  in vm_object_zdtor()
186 	KASSERT(TAILQ_EMPTY(&object->memq),  in vm_object_zdtor()
187 	    ("object %p has resident pages in its memq", object));  in vm_object_zdtor()
188 	KASSERT(vm_radix_is_empty(&object->rtree),  in vm_object_zdtor()
189 	    ("object %p has resident pages in its trie", object));  in vm_object_zdtor()
191 	KASSERT(LIST_EMPTY(&object->rvq),  in vm_object_zdtor()
193 	    object));  in vm_object_zdtor()
195 	KASSERT(!vm_object_busied(object),  in vm_object_zdtor()
196 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));  in vm_object_zdtor()
197 	KASSERT(object->resident_page_count == 0,  in vm_object_zdtor()
199 	    object, object->resident_page_count));  in vm_object_zdtor()
200 	KASSERT(atomic_load_int(&object->shadow_count) == 0,  in vm_object_zdtor()
202 	    object, atomic_load_int(&object->shadow_count)));  in vm_object_zdtor()
203 	KASSERT(object->type == OBJT_DEAD,  in vm_object_zdtor()
205 	    object, object->type));  in vm_object_zdtor()
206 	KASSERT(object->charge == 0 && object->cred == NULL,  in vm_object_zdtor()
208 	    object, (uintmax_t)object->charge, object->cred));  in vm_object_zdtor()
215 	vm_object_t object;  in vm_object_zinit()  local
217 	object = (vm_object_t)mem;  in vm_object_zinit()
218 	rw_init_flags(&object->lock, "vmobject", RW_DUPOK | RW_NEW);  in vm_object_zinit()
221 	object->type = OBJT_DEAD;  in vm_object_zinit()
222 	vm_radix_init(&object->rtree);  in vm_object_zinit()
223 	refcount_init(&object->ref_count, 0);  in vm_object_zinit()
224 	blockcount_init(&object->paging_in_progress);  in vm_object_zinit()
225 	blockcount_init(&object->busy);  in vm_object_zinit()
226 	object->resident_page_count = 0;  in vm_object_zinit()
227 	atomic_store_int(&object->shadow_count, 0);  in vm_object_zinit()
228 	object->flags = OBJ_DEAD;  in vm_object_zinit()
231 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);  in vm_object_zinit()
238     vm_object_t object, void *handle)  in _vm_object_allocate()  argument
241 	TAILQ_INIT(&object->memq);  in _vm_object_allocate()
242 	LIST_INIT(&object->shadow_head);  in _vm_object_allocate()
244 	object->type = type;  in _vm_object_allocate()
245 	object->flags = flags;  in _vm_object_allocate()
247 		pctrie_init(&object->un_pager.swp.swp_blks);  in _vm_object_allocate()
248 		object->un_pager.swp.writemappings = 0;  in _vm_object_allocate()
258 	object->pg_color = 0;  in _vm_object_allocate()
259 	object->size = size;  in _vm_object_allocate()
260 	object->domain.dr_policy = NULL;  in _vm_object_allocate()
261 	object->generation = 1;  in _vm_object_allocate()
262 	object->cleangeneration = 1;  in _vm_object_allocate()
263 	refcount_init(&object->ref_count, 1);  in _vm_object_allocate()
264 	object->memattr = VM_MEMATTR_DEFAULT;  in _vm_object_allocate()
265 	object->cred = NULL;  in _vm_object_allocate()
266 	object->charge = 0;  in _vm_object_allocate()
267 	object->handle = handle;  in _vm_object_allocate()
268 	object->backing_object = NULL;  in _vm_object_allocate()
269 	object->backing_object_offset = (vm_ooffset_t) 0;  in _vm_object_allocate()
271 	LIST_INIT(&object->rvq);  in _vm_object_allocate()
273 	umtx_shm_object_init(object);  in _vm_object_allocate()
317 vm_object_clear_flag(vm_object_t object, u_short bits)  in vm_object_clear_flag()  argument
320 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_clear_flag()
321 	object->flags &= ~bits;  in vm_object_clear_flag()
334 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)  in vm_object_set_memattr()  argument
337 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_set_memattr()
339 	if (object->type == OBJT_DEAD)  in vm_object_set_memattr()
341 	if (!TAILQ_EMPTY(&object->memq))  in vm_object_set_memattr()
344 	object->memattr = memattr;  in vm_object_set_memattr()
349 vm_object_pip_add(vm_object_t object, short i)  in vm_object_pip_add()  argument
353 		blockcount_acquire(&object->paging_in_progress, i);  in vm_object_pip_add()
357 vm_object_pip_wakeup(vm_object_t object)  in vm_object_pip_wakeup()  argument
360 	vm_object_pip_wakeupn(object, 1);  in vm_object_pip_wakeup()
364 vm_object_pip_wakeupn(vm_object_t object, short i)  in vm_object_pip_wakeupn()  argument
368 		blockcount_release(&object->paging_in_progress, i);  in vm_object_pip_wakeupn()
377 vm_object_pip_sleep(vm_object_t object, const char *waitid)  in vm_object_pip_sleep()  argument
380 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,  in vm_object_pip_sleep()
385 vm_object_pip_wait(vm_object_t object, const char *waitid)  in vm_object_pip_wait()  argument
388 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_pip_wait()
390 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,  in vm_object_pip_wait()
395 vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)  in vm_object_pip_wait_unlocked()  argument
398 	VM_OBJECT_ASSERT_UNLOCKED(object);  in vm_object_pip_wait_unlocked()
400 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);  in vm_object_pip_wait_unlocked()
411 	vm_object_t object;  in vm_object_allocate()  local
437 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate()
438 	_vm_object_allocate(type, size, flags, object, NULL);  in vm_object_allocate()
440 	return (object);  in vm_object_allocate()
446 	vm_object_t object;  in vm_object_allocate_dyn()  local
449 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_dyn()
450 	_vm_object_allocate(dyntype, size, flags, object, NULL);  in vm_object_allocate_dyn()
452 	return (object);  in vm_object_allocate_dyn()
466 	vm_object_t handle, object;  in vm_object_allocate_anon()  local
474 	object = uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_anon()
476 	    OBJ_ANON | OBJ_ONEMAPPING | OBJ_SWAP, object, handle);  in vm_object_allocate_anon()
477 	object->cred = cred;  in vm_object_allocate_anon()
478 	object->charge = cred != NULL ? charge : 0;  in vm_object_allocate_anon()
479 	return (object);  in vm_object_allocate_anon()
483 vm_object_reference_vnode(vm_object_t object)  in vm_object_reference_vnode()  argument
491 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {  in vm_object_reference_vnode()
492 		VM_OBJECT_RLOCK(object);  in vm_object_reference_vnode()
493 		old = refcount_acquire(&object->ref_count);  in vm_object_reference_vnode()
494 		if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_vnode()
495 			vref(object->handle);  in vm_object_reference_vnode()
496 		VM_OBJECT_RUNLOCK(object);  in vm_object_reference_vnode()
506 vm_object_reference(vm_object_t object)  in vm_object_reference()  argument
509 	if (object == NULL)  in vm_object_reference()
512 	if (object->type == OBJT_VNODE)  in vm_object_reference()
513 		vm_object_reference_vnode(object);  in vm_object_reference()
515 		refcount_acquire(&object->ref_count);  in vm_object_reference()
516 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference()
528 vm_object_reference_locked(vm_object_t object)  in vm_object_reference_locked()  argument
532 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_reference_locked()
533 	old = refcount_acquire(&object->ref_count);  in vm_object_reference_locked()
534 	if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_locked()
535 		vref(object->handle);  in vm_object_reference_locked()
536 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference_locked()
544 vm_object_deallocate_vnode(vm_object_t object)  in vm_object_deallocate_vnode()  argument
546 	struct vnode *vp = (struct vnode *) object->handle;  in vm_object_deallocate_vnode()
549 	KASSERT(object->type == OBJT_VNODE,  in vm_object_deallocate_vnode()
554 	last = refcount_release(&object->ref_count);  in vm_object_deallocate_vnode()
555 	VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate_vnode()
561 		umtx_shm_object_terminated(object);  in vm_object_deallocate_vnode()
575 	vm_object_t object;  in vm_object_deallocate_anon()  local
578 	object = LIST_FIRST(&backing_object->shadow_head);  in vm_object_deallocate_anon()
579 	KASSERT(object != NULL &&  in vm_object_deallocate_anon()
584 	KASSERT((object->flags & OBJ_ANON) != 0,  in vm_object_deallocate_anon()
585 	    ("invalid shadow object %p", object));  in vm_object_deallocate_anon()
587 	if (!VM_OBJECT_TRYWLOCK(object)) {  in vm_object_deallocate_anon()
592 		vm_object_pip_add(object, 1);  in vm_object_deallocate_anon()
594 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate_anon()
595 		vm_object_pip_wakeup(object);  in vm_object_deallocate_anon()
602 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||  in vm_object_deallocate_anon()
603 	    !refcount_acquire_if_not_zero(&object->ref_count)) {  in vm_object_deallocate_anon()
604 		VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
607 	backing_object = object->backing_object;  in vm_object_deallocate_anon()
609 		vm_object_collapse(object);  in vm_object_deallocate_anon()
610 	VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
612 	return (object);  in vm_object_deallocate_anon()
627 vm_object_deallocate(vm_object_t object)  in vm_object_deallocate()  argument
632 	while (object != NULL) {  in vm_object_deallocate()
640 		if ((object->flags & OBJ_ANON) == 0)  in vm_object_deallocate()
641 			released = refcount_release_if_gt(&object->ref_count, 1);  in vm_object_deallocate()
643 			released = refcount_release_if_gt(&object->ref_count, 2);  in vm_object_deallocate()
647 		if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
648 			VM_OBJECT_RLOCK(object);  in vm_object_deallocate()
649 			if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
650 				vm_object_deallocate_vnode(object);  in vm_object_deallocate()
653 			VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate()
656 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate()
657 		KASSERT(object->ref_count > 0,  in vm_object_deallocate()
659 		    object->type));  in vm_object_deallocate()
665 		if (!refcount_release(&object->ref_count)) {  in vm_object_deallocate()
666 			if (object->ref_count > 1 ||  in vm_object_deallocate()
667 			    atomic_load_int(&object->shadow_count) == 0) {  in vm_object_deallocate()
668 				if ((object->flags & OBJ_ANON) != 0 &&  in vm_object_deallocate()
669 				    object->ref_count == 1)  in vm_object_deallocate()
670 					vm_object_set_flag(object,  in vm_object_deallocate()
672 				VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate()
677 			object = vm_object_deallocate_anon(object);  in vm_object_deallocate()
685 		umtx_shm_object_terminated(object);  in vm_object_deallocate()
686 		temp = object->backing_object;  in vm_object_deallocate()
688 			KASSERT(object->type == OBJT_SWAP,  in vm_object_deallocate()
689 			    ("shadowed tmpfs v_object 2 %p", object));  in vm_object_deallocate()
690 			vm_object_backing_remove(object);  in vm_object_deallocate()
693 		KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_deallocate()
695 		vm_object_set_flag(object, OBJ_DEAD);  in vm_object_deallocate()
696 		vm_object_terminate(object);  in vm_object_deallocate()
697 		object = temp;  in vm_object_deallocate()
702 vm_object_destroy(vm_object_t object)  in vm_object_destroy()  argument
704 	uma_zfree(obj_zone, object);  in vm_object_destroy()
708 vm_object_sub_shadow(vm_object_t object)  in vm_object_sub_shadow()  argument
710 	KASSERT(object->shadow_count >= 1,  in vm_object_sub_shadow()
711 	    ("object %p sub_shadow count zero", object));  in vm_object_sub_shadow()
712 	atomic_subtract_int(&object->shadow_count, 1);  in vm_object_sub_shadow()
716 vm_object_backing_remove_locked(vm_object_t object)  in vm_object_backing_remove_locked()  argument
720 	backing_object = object->backing_object;  in vm_object_backing_remove_locked()
721 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove_locked()
724 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_backing_remove_locked()
728 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove_locked()
729 		LIST_REMOVE(object, shadow_list);  in vm_object_backing_remove_locked()
730 		vm_object_clear_flag(object, OBJ_SHADOWLIST);  in vm_object_backing_remove_locked()
732 	object->backing_object = NULL;  in vm_object_backing_remove_locked()
736 vm_object_backing_remove(vm_object_t object)  in vm_object_backing_remove()  argument
740 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove()
742 	backing_object = object->backing_object;  in vm_object_backing_remove()
743 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove()
745 		vm_object_backing_remove_locked(object);  in vm_object_backing_remove()
748 		object->backing_object = NULL;  in vm_object_backing_remove()
754 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_locked()  argument
757 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_locked()
762 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,  in vm_object_backing_insert_locked()
764 		vm_object_set_flag(object, OBJ_SHADOWLIST);  in vm_object_backing_insert_locked()
766 	object->backing_object = backing_object;  in vm_object_backing_insert_locked()
770 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert()  argument
773 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert()
777 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert()
780 		object->backing_object = backing_object;  in vm_object_backing_insert()
790 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_ref()  argument
793 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_ref()
800 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert_ref()
806 		object->backing_object = backing_object;  in vm_object_backing_insert_ref()
814 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_transfer()  argument
822 	vm_object_backing_remove_locked(object);  in vm_object_backing_transfer()
829 		vm_object_backing_insert_locked(object, new_backing_object);  in vm_object_backing_transfer()
837 		object->backing_object = new_backing_object;  in vm_object_backing_transfer()
846 vm_object_collapse_wait(vm_object_t object)  in vm_object_collapse_wait()  argument
849 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_wait()
851 	while ((object->flags & OBJ_COLLAPSING) != 0) {  in vm_object_collapse_wait()
852 		vm_object_pip_wait(object, "vmcolwait");  in vm_object_collapse_wait()
862 vm_object_backing_collapse_wait(vm_object_t object)  in vm_object_backing_collapse_wait()  argument
866 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_collapse_wait()
869 		backing_object = object->backing_object;  in vm_object_backing_collapse_wait()
876 		VM_OBJECT_WUNLOCK(object);  in vm_object_backing_collapse_wait()
879 		VM_OBJECT_WLOCK(object);  in vm_object_backing_collapse_wait()
889 vm_object_terminate_pages(vm_object_t object)  in vm_object_terminate_pages()  argument
893 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate_pages()
901 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {  in vm_object_terminate_pages()
903 		KASSERT(p->object == object &&  in vm_object_terminate_pages()
907 		p->object = NULL;  in vm_object_terminate_pages()
919 	if (object->resident_page_count != 0) {  in vm_object_terminate_pages()
920 		vm_radix_reclaim_allnodes(&object->rtree);  in vm_object_terminate_pages()
921 		TAILQ_INIT(&object->memq);  in vm_object_terminate_pages()
922 		object->resident_page_count = 0;  in vm_object_terminate_pages()
923 		if (object->type == OBJT_VNODE)  in vm_object_terminate_pages()
924 			vdrop(object->handle);  in vm_object_terminate_pages()
936 vm_object_terminate(vm_object_t object)  in vm_object_terminate()  argument
939 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate()
940 	KASSERT((object->flags & OBJ_DEAD) != 0,  in vm_object_terminate()
941 	    ("terminating non-dead obj %p", object));  in vm_object_terminate()
942 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_terminate()
943 	    ("terminating collapsing obj %p", object));  in vm_object_terminate()
944 	KASSERT(object->backing_object == NULL,  in vm_object_terminate()
945 	    ("terminating shadow obj %p", object));  in vm_object_terminate()
954 	vm_object_pip_wait(object, "objtrm");  in vm_object_terminate()
956 	KASSERT(object->ref_count == 0,  in vm_object_terminate()
958 	    object->ref_count));  in vm_object_terminate()
960 	if ((object->flags & OBJ_PG_DTOR) == 0)  in vm_object_terminate()
961 		vm_object_terminate_pages(object);  in vm_object_terminate()
964 	if (__predict_false(!LIST_EMPTY(&object->rvq)))  in vm_object_terminate()
965 		vm_reserv_break_all(object);  in vm_object_terminate()
968 	KASSERT(object->cred == NULL || (object->flags & OBJ_SWAP) != 0,  in vm_object_terminate()
969 	    ("%s: non-swap obj %p has cred", __func__, object));  in vm_object_terminate()
974 	vm_pager_deallocate(object);  in vm_object_terminate()
975 	VM_OBJECT_WUNLOCK(object);  in vm_object_terminate()
977 	vm_object_destroy(object);  in vm_object_terminate()
1029 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,  in vm_object_page_clean()  argument
1037 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_clean()
1039 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)  in vm_object_page_clean()
1047 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);  in vm_object_page_clean()
1048 	allclean = tstart == 0 && tend >= object->size;  in vm_object_page_clean()
1052 	curgeneration = object->generation;  in vm_object_page_clean()
1054 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {  in vm_object_page_clean()
1062 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1065 			np = vm_page_find_least(object, pi);  in vm_object_page_clean()
1072 		if (object->type == OBJT_VNODE) {  in vm_object_page_clean()
1073 			n = vm_object_page_collect_flush(object, p, pagerflags,  in vm_object_page_clean()
1079 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1103 		np = vm_page_find_least(object, pi + n);  in vm_object_page_clean()
1114 	if (allclean && object->type == OBJT_VNODE)  in vm_object_page_clean()
1115 		object->cleangeneration = curgeneration;  in vm_object_page_clean()
1120 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,  in vm_object_page_collect_flush()  argument
1128 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_collect_flush()
1177 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,  in vm_object_sync()  argument
1186 	if (object == NULL)  in vm_object_sync()
1190 	VM_OBJECT_WLOCK(object);  in vm_object_sync()
1191 	while ((backing_object = object->backing_object) != NULL) {  in vm_object_sync()
1193 		offset += object->backing_object_offset;  in vm_object_sync()
1194 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1195 		object = backing_object;  in vm_object_sync()
1196 		if (object->size < OFF_TO_IDX(offset + size))  in vm_object_sync()
1197 			size = IDX_TO_OFF(object->size) - offset;  in vm_object_sync()
1211 	if (object->type == OBJT_VNODE &&  in vm_object_sync()
1212 	    vm_object_mightbedirty(object) != 0 &&  in vm_object_sync()
1213 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {  in vm_object_sync()
1214 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1218 		    atop(size) == object->size) {  in vm_object_sync()
1232 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1233 		res = vm_object_page_clean(object, offset, offset + size,  in vm_object_sync()
1235 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1257 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1259 	if ((object->type == OBJT_VNODE ||  in vm_object_sync()
1260 	     object->type == OBJT_DEVICE) && invalidate) {  in vm_object_sync()
1261 		if (object->type == OBJT_DEVICE)  in vm_object_sync()
1272 		vm_object_page_remove(object, OFF_TO_IDX(offset),  in vm_object_sync()
1275 	VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1286 vm_object_advice_applies(vm_object_t object, int advice)  in vm_object_advice_applies()  argument
1289 	if ((object->flags & OBJ_UNMANAGED) != 0)  in vm_object_advice_applies()
1293 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==  in vm_object_advice_applies()
1298 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,  in vm_object_madvise_freespace()  argument
1303 		vm_pager_freespace(object, pindex, size);  in vm_object_madvise_freespace()
1327 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,  in vm_object_madvise()  argument
1334 	if (object == NULL)  in vm_object_madvise()
1338 	VM_OBJECT_WLOCK(object);  in vm_object_madvise()
1339 	if (!vm_object_advice_applies(object, advice)) {  in vm_object_madvise()
1340 		VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1343 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {  in vm_object_madvise()
1344 		tobject = object;  in vm_object_madvise()
1358 			if (object->backing_object == NULL) {  in vm_object_madvise()
1361 				vm_object_madvise_freespace(object, advice,  in vm_object_madvise()
1382 				if (tobject != object)  in vm_object_madvise()
1406 			if (object != tobject)  in vm_object_madvise()
1407 				VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1424 		if (tobject != object)  in vm_object_madvise()
1427 	VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1441 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,  in vm_object_shadow()  argument
1447 	source = *object;  in vm_object_shadow()
1511 	*object = result;  in vm_object_shadow()
1529 	orig_object = entry->object.vm_object;  in vm_object_split()
1653 	entry->object.vm_object = new_object;  in vm_object_split()
1660 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)  in vm_object_collapse_scan_wait()  argument
1664 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan_wait()
1665 	backing_object = object->backing_object;  in vm_object_collapse_scan_wait()
1668 	KASSERT(p == NULL || p->object == object || p->object == backing_object,  in vm_object_collapse_scan_wait()
1669 	    ("invalid ownership %p %p %p", p, object, backing_object));  in vm_object_collapse_scan_wait()
1672 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1675 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1676 	} else if (p->object == object) {  in vm_object_collapse_scan_wait()
1679 			VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1681 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1684 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1691 vm_object_scan_all_shadowed(vm_object_t object)  in vm_object_scan_all_shadowed()  argument
1697 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_scan_all_shadowed()
1698 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in vm_object_scan_all_shadowed()
1700 	backing_object = object->backing_object;  in vm_object_scan_all_shadowed()
1705 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in vm_object_scan_all_shadowed()
1726 		if (new_pindex >= object->size)  in vm_object_scan_all_shadowed()
1756 		pp = vm_page_lookup(object, new_pindex);  in vm_object_scan_all_shadowed()
1764 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))  in vm_object_scan_all_shadowed()
1778 vm_object_collapse_scan(vm_object_t object)  in vm_object_collapse_scan()  argument
1784 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan()
1785 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in vm_object_collapse_scan()
1787 	backing_object = object->backing_object;  in vm_object_collapse_scan()
1788 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in vm_object_collapse_scan()
1801 			next = vm_object_collapse_scan_wait(object, p);  in vm_object_collapse_scan()
1805 		KASSERT(object->backing_object == backing_object,  in vm_object_collapse_scan()
1807 		    object->backing_object, backing_object));  in vm_object_collapse_scan()
1808 		KASSERT(p->object == backing_object,  in vm_object_collapse_scan()
1810 		    p->object, backing_object));  in vm_object_collapse_scan()
1813 		    new_pindex >= object->size) {  in vm_object_collapse_scan()
1831 		pp = vm_page_lookup(object, new_pindex);  in vm_object_collapse_scan()
1840 			next = vm_object_collapse_scan_wait(object, pp);  in vm_object_collapse_scan()
1855 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,  in vm_object_collapse_scan()
1880 		if (vm_page_rename(p, object, new_pindex)) {  in vm_object_collapse_scan()
1882 			next = vm_object_collapse_scan_wait(object, NULL);  in vm_object_collapse_scan()
1894 		vm_reserv_rename(p, object, backing_object,  in vm_object_collapse_scan()
1910 vm_object_collapse(vm_object_t object)  in vm_object_collapse()  argument
1914 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse()
1917 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,  in vm_object_collapse()
1925 		backing_object = vm_object_backing_collapse_wait(object);  in vm_object_collapse()
1929 		KASSERT(object->ref_count > 0 &&  in vm_object_collapse()
1930 		    object->ref_count > atomic_load_int(&object->shadow_count),  in vm_object_collapse()
1932 		    object->ref_count, atomic_load_int(&object->shadow_count)));  in vm_object_collapse()
1936 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,  in vm_object_collapse()
1950 			vm_object_pip_add(object, 1);  in vm_object_collapse()
1951 			vm_object_set_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
1959 			vm_object_collapse_scan(object);  in vm_object_collapse()
1976 			swap_pager_copy(backing_object, object,  in vm_object_collapse()
1977 			    OFF_TO_IDX(object->backing_object_offset), TRUE);  in vm_object_collapse()
1982 			vm_object_clear_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
1983 			vm_object_backing_transfer(object, backing_object);  in vm_object_collapse()
1984 			object->backing_object_offset +=  in vm_object_collapse()
1986 			VM_OBJECT_WUNLOCK(object);  in vm_object_collapse()
1987 			vm_object_pip_wakeup(object);  in vm_object_collapse()
2003 			VM_OBJECT_WLOCK(object);  in vm_object_collapse()
2012 			if (!vm_object_scan_all_shadowed(object)) {  in vm_object_collapse()
2022 			vm_object_backing_remove_locked(object);  in vm_object_collapse()
2025 				vm_object_backing_insert_ref(object,  in vm_object_collapse()
2027 				object->backing_object_offset +=  in vm_object_collapse()
2076 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,  in vm_object_page_remove()  argument
2081 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_remove()
2082 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||  in vm_object_page_remove()
2084 	    ("vm_object_page_remove: illegal options for object %p", object));  in vm_object_page_remove()
2085 	if (object->resident_page_count == 0)  in vm_object_page_remove()
2087 	vm_object_pip_add(object, 1);  in vm_object_page_remove()
2089 	p = vm_page_find_least(object, start);  in vm_object_page_remove()
2123 				VM_OBJECT_WLOCK(object);  in vm_object_page_remove()
2133 			    object->ref_count != 0)  in vm_object_page_remove()
2147 			    object->ref_count != 0 &&  in vm_object_page_remove()
2156 		    object->ref_count != 0 && !vm_page_try_remove_all(p))  in vm_object_page_remove()
2160 	vm_object_pip_wakeup(object);  in vm_object_page_remove()
2162 	vm_pager_freespace(object, start, (end == 0 ? object->size : end) -  in vm_object_page_remove()
2183 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_page_noreuse()  argument
2187 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_page_noreuse()
2188 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,  in vm_object_page_noreuse()
2189 	    ("vm_object_page_noreuse: illegal object %p", object));  in vm_object_page_noreuse()
2190 	if (object->resident_page_count == 0)  in vm_object_page_noreuse()
2192 	p = vm_page_find_least(object, start);  in vm_object_page_noreuse()
2215 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_populate()  argument
2221 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_populate()
2223 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);  in vm_object_populate()
2233 		m = vm_page_lookup(object, start);  in vm_object_populate()
2353 vm_object_set_writeable_dirty_(vm_object_t object)  in vm_object_set_writeable_dirty_()  argument
2355 	atomic_add_int(&object->generation, 1);  in vm_object_set_writeable_dirty_()
2359 vm_object_mightbedirty_(vm_object_t object)  in vm_object_mightbedirty_()  argument
2361 	return (object->generation != object->cleangeneration);  in vm_object_mightbedirty_()
2373 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,  in vm_object_unwire()  argument
2386 	if ((object->flags & OBJ_FICTITIOUS) != 0)  in vm_object_unwire()
2392 	VM_OBJECT_RLOCK(object);  in vm_object_unwire()
2393 	m = vm_page_find_least(object, pindex);  in vm_object_unwire()
2401 			tobject = object;  in vm_object_unwire()
2424 			for (tobject = object; locked_depth >= 1;  in vm_object_unwire()
2427 				if (tm->object != tobject)  in vm_object_unwire()
2431 			tobject = tm->object;  in vm_object_unwire()
2443 	for (tobject = object; locked_depth >= 1; locked_depth--) {  in vm_object_unwire()
2456 vm_object_vnode(vm_object_t object)  in vm_object_vnode()  argument
2460 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_vnode()
2461 	vm_pager_getvp(object, &vp, NULL);  in vm_object_vnode()
2704 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)  in _vm_object_in_map()  argument
2715 			if (_vm_object_in_map(map, object, tmpe)) {  in _vm_object_in_map()
2720 		tmpm = entry->object.sub_map;  in _vm_object_in_map()
2722 			if (_vm_object_in_map(tmpm, object, tmpe)) {  in _vm_object_in_map()
2726 	} else if ((obj = entry->object.vm_object) != NULL) {  in _vm_object_in_map()
2728 			if (obj == object) {  in _vm_object_in_map()
2736 vm_object_in_map(vm_object_t object)  in vm_object_in_map()  argument
2744 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {  in vm_object_in_map()
2750 	if (_vm_object_in_map(kernel_map, object, 0))  in vm_object_in_map()
2757 	vm_object_t object;  in DB_SHOW_COMMAND_FLAGS()  local
2763 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND_FLAGS()
2764 		if ((object->flags & OBJ_ANON) != 0) {  in DB_SHOW_COMMAND_FLAGS()
2765 			if (object->ref_count == 0) {  in DB_SHOW_COMMAND_FLAGS()
2767 					(long)object->size);  in DB_SHOW_COMMAND_FLAGS()
2769 			if (!vm_object_in_map(object)) {  in DB_SHOW_COMMAND_FLAGS()
2773 				    object->ref_count, (u_long)object->size,   in DB_SHOW_COMMAND_FLAGS()
2774 				    (u_long)object->size,  in DB_SHOW_COMMAND_FLAGS()
2775 				    (void *)object->backing_object);  in DB_SHOW_COMMAND_FLAGS()
2786 DB_SHOW_COMMAND(object, vm_object_print_static)  in DB_SHOW_COMMAND()  argument
2789 	vm_object_t object = (vm_object_t)addr;  in DB_SHOW_COMMAND()  local
2799 	if (object == NULL)  in DB_SHOW_COMMAND()
2804 	    object, (int)object->type, (uintmax_t)object->size,  in DB_SHOW_COMMAND()
2805 	    object->resident_page_count, object->ref_count, object->flags,  in DB_SHOW_COMMAND()
2806 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);  in DB_SHOW_COMMAND()
2808 	    atomic_load_int(&object->shadow_count),  in DB_SHOW_COMMAND()
2809 	    object->backing_object ? object->backing_object->ref_count : 0,  in DB_SHOW_COMMAND()
2810 	    object->backing_object, (uintmax_t)object->backing_object_offset);  in DB_SHOW_COMMAND()
2817 	TAILQ_FOREACH(p, &object->memq, listq) {  in DB_SHOW_COMMAND()
2855 	vm_object_t object;  in DB_SHOW_COMMAND_FLAGS()  local
2861 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND_FLAGS()
2862 		db_printf("new object: %p\n", (void *)object);  in DB_SHOW_COMMAND_FLAGS()
2869 		TAILQ_FOREACH(m, &object->memq, listq) {  in DB_SHOW_COMMAND_FLAGS()