116 static int	vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
121 static void	vm_object_backing_remove(vm_object_t object);
154 static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
182 	vm_object_t object;  in vm_object_zdtor()  local
184 	object = (vm_object_t)mem;  in vm_object_zdtor()
185 	KASSERT(object->ref_count == 0,  in vm_object_zdtor()
186 	    ("object %p ref_count = %d", object, object->ref_count));  in vm_object_zdtor()
187 	KASSERT(TAILQ_EMPTY(&object->memq),  in vm_object_zdtor()
188 	    ("object %p has resident pages in its memq", object));  in vm_object_zdtor()
189 	KASSERT(vm_radix_is_empty(&object->rtree),  in vm_object_zdtor()
190 	    ("object %p has resident pages in its trie", object));  in vm_object_zdtor()
192 	KASSERT(LIST_EMPTY(&object->rvq),  in vm_object_zdtor()
194 	    object));  in vm_object_zdtor()
196 	KASSERT(!vm_object_busied(object),  in vm_object_zdtor()
197 	    ("object %p busy = %d", object, blockcount_read(&object->busy)));  in vm_object_zdtor()
198 	KASSERT(object->resident_page_count == 0,  in vm_object_zdtor()
200 	    object, object->resident_page_count));  in vm_object_zdtor()
201 	KASSERT(atomic_load_int(&object->shadow_count) == 0,  in vm_object_zdtor()
203 	    object, atomic_load_int(&object->shadow_count)));  in vm_object_zdtor()
204 	KASSERT(object->type == OBJT_DEAD,  in vm_object_zdtor()
206 	    object, object->type));  in vm_object_zdtor()
213 	vm_object_t object;  in vm_object_zinit()  local
215 	object = (vm_object_t)mem;  in vm_object_zinit()
216 	rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);  in vm_object_zinit()
219 	object->type = OBJT_DEAD;  in vm_object_zinit()
220 	vm_radix_init(&object->rtree);  in vm_object_zinit()
221 	refcount_init(&object->ref_count, 0);  in vm_object_zinit()
222 	blockcount_init(&object->paging_in_progress);  in vm_object_zinit()
223 	blockcount_init(&object->busy);  in vm_object_zinit()
224 	object->resident_page_count = 0;  in vm_object_zinit()
225 	atomic_store_int(&object->shadow_count, 0);  in vm_object_zinit()
226 	object->flags = OBJ_DEAD;  in vm_object_zinit()
229 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);  in vm_object_zinit()
236     vm_object_t object, void *handle)  in _vm_object_allocate()  argument
239 	TAILQ_INIT(&object->memq);  in _vm_object_allocate()
240 	LIST_INIT(&object->shadow_head);  in _vm_object_allocate()
242 	object->type = type;  in _vm_object_allocate()
243 	object->flags = flags;  in _vm_object_allocate()
245 		pctrie_init(&object->un_pager.swp.swp_blks);  in _vm_object_allocate()
254 	object->pg_color = 0;  in _vm_object_allocate()
255 	object->size = size;  in _vm_object_allocate()
256 	object->domain.dr_policy = NULL;  in _vm_object_allocate()
257 	object->generation = 1;  in _vm_object_allocate()
258 	object->cleangeneration = 1;  in _vm_object_allocate()
259 	refcount_init(&object->ref_count, 1);  in _vm_object_allocate()
260 	object->memattr = VM_MEMATTR_DEFAULT;  in _vm_object_allocate()
261 	object->cred = NULL;  in _vm_object_allocate()
262 	object->charge = 0;  in _vm_object_allocate()
263 	object->handle = handle;  in _vm_object_allocate()
264 	object->backing_object = NULL;  in _vm_object_allocate()
265 	object->backing_object_offset = (vm_ooffset_t) 0;  in _vm_object_allocate()
267 	LIST_INIT(&object->rvq);  in _vm_object_allocate()
269 	umtx_shm_object_init(object);  in _vm_object_allocate()
312 vm_object_clear_flag(vm_object_t object, u_short bits)  in vm_object_clear_flag()  argument
315 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_clear_flag()
316 	object->flags &= ~bits;  in vm_object_clear_flag()
329 vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)  in vm_object_set_memattr()  argument
332 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_set_memattr()
334 	if (object->type == OBJT_DEAD)  in vm_object_set_memattr()
336 	if (!TAILQ_EMPTY(&object->memq))  in vm_object_set_memattr()
339 	object->memattr = memattr;  in vm_object_set_memattr()
344 vm_object_pip_add(vm_object_t object, short i)  in vm_object_pip_add()  argument
348 		blockcount_acquire(&object->paging_in_progress, i);  in vm_object_pip_add()
352 vm_object_pip_wakeup(vm_object_t object)  in vm_object_pip_wakeup()  argument
355 	vm_object_pip_wakeupn(object, 1);  in vm_object_pip_wakeup()
359 vm_object_pip_wakeupn(vm_object_t object, short i)  in vm_object_pip_wakeupn()  argument
363 		blockcount_release(&object->paging_in_progress, i);  in vm_object_pip_wakeupn()
372 vm_object_pip_sleep(vm_object_t object, const char *waitid)  in vm_object_pip_sleep()  argument
375 	(void)blockcount_sleep(&object->paging_in_progress, &object->lock,  in vm_object_pip_sleep()
380 vm_object_pip_wait(vm_object_t object, const char *waitid)  in vm_object_pip_wait()  argument
383 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_pip_wait()
385 	blockcount_wait(&object->paging_in_progress, &object->lock, waitid,  in vm_object_pip_wait()
390 vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)  in vm_object_pip_wait_unlocked()  argument
393 	VM_OBJECT_ASSERT_UNLOCKED(object);  in vm_object_pip_wait_unlocked()
395 	blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);  in vm_object_pip_wait_unlocked()
406 	vm_object_t object;  in vm_object_allocate()  local
435 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate()
436 	_vm_object_allocate(type, size, flags, object, NULL);  in vm_object_allocate()
438 	return (object);  in vm_object_allocate()
444 	vm_object_t object;  in vm_object_allocate_dyn()  local
447 	object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_dyn()
448 	_vm_object_allocate(dyntype, size, flags, object, NULL);  in vm_object_allocate_dyn()
450 	return (object);  in vm_object_allocate_dyn()
464 	vm_object_t handle, object;  in vm_object_allocate_anon()  local
472 	object = uma_zalloc(obj_zone, M_WAITOK);  in vm_object_allocate_anon()
474 	    object, handle);  in vm_object_allocate_anon()
475 	object->cred = cred;  in vm_object_allocate_anon()
476 	object->charge = cred != NULL ? charge : 0;  in vm_object_allocate_anon()
477 	return (object);  in vm_object_allocate_anon()
481 vm_object_reference_vnode(vm_object_t object)  in vm_object_reference_vnode()  argument
489 	if (!refcount_acquire_if_gt(&object->ref_count, 0)) {  in vm_object_reference_vnode()
490 		VM_OBJECT_RLOCK(object);  in vm_object_reference_vnode()
491 		old = refcount_acquire(&object->ref_count);  in vm_object_reference_vnode()
492 		if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_vnode()
493 			vref(object->handle);  in vm_object_reference_vnode()
494 		VM_OBJECT_RUNLOCK(object);  in vm_object_reference_vnode()
504 vm_object_reference(vm_object_t object)  in vm_object_reference()  argument
507 	if (object == NULL)  in vm_object_reference()
510 	if (object->type == OBJT_VNODE)  in vm_object_reference()
511 		vm_object_reference_vnode(object);  in vm_object_reference()
513 		refcount_acquire(&object->ref_count);  in vm_object_reference()
514 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference()
526 vm_object_reference_locked(vm_object_t object)  in vm_object_reference_locked()  argument
530 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_reference_locked()
531 	old = refcount_acquire(&object->ref_count);  in vm_object_reference_locked()
532 	if (object->type == OBJT_VNODE && old == 0)  in vm_object_reference_locked()
533 		vref(object->handle);  in vm_object_reference_locked()
534 	KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_reference_locked()
542 vm_object_deallocate_vnode(vm_object_t object)  in vm_object_deallocate_vnode()  argument
544 	struct vnode *vp = (struct vnode *) object->handle;  in vm_object_deallocate_vnode()
547 	KASSERT(object->type == OBJT_VNODE,  in vm_object_deallocate_vnode()
552 	last = refcount_release(&object->ref_count);  in vm_object_deallocate_vnode()
553 	VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate_vnode()
559 		umtx_shm_object_terminated(object);  in vm_object_deallocate_vnode()
573 	vm_object_t object;  in vm_object_deallocate_anon()  local
576 	object = LIST_FIRST(&backing_object->shadow_head);  in vm_object_deallocate_anon()
577 	KASSERT(object != NULL &&  in vm_object_deallocate_anon()
582 	KASSERT((object->flags & OBJ_ANON) != 0,  in vm_object_deallocate_anon()
583 	    ("invalid shadow object %p", object));  in vm_object_deallocate_anon()
585 	if (!VM_OBJECT_TRYWLOCK(object)) {  in vm_object_deallocate_anon()
590 		vm_object_pip_add(object, 1);  in vm_object_deallocate_anon()
592 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate_anon()
593 		vm_object_pip_wakeup(object);  in vm_object_deallocate_anon()
600 	if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||  in vm_object_deallocate_anon()
601 	    !refcount_acquire_if_not_zero(&object->ref_count)) {  in vm_object_deallocate_anon()
602 		VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
605 	backing_object = object->backing_object;  in vm_object_deallocate_anon()
607 		vm_object_collapse(object);  in vm_object_deallocate_anon()
608 	VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate_anon()
610 	return (object);  in vm_object_deallocate_anon()
625 vm_object_deallocate(vm_object_t object)  in vm_object_deallocate()  argument
630 	while (object != NULL) {  in vm_object_deallocate()
638 		if ((object->flags & OBJ_ANON) == 0)  in vm_object_deallocate()
639 			released = refcount_release_if_gt(&object->ref_count, 1);  in vm_object_deallocate()
641 			released = refcount_release_if_gt(&object->ref_count, 2);  in vm_object_deallocate()
645 		if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
646 			VM_OBJECT_RLOCK(object);  in vm_object_deallocate()
647 			if (object->type == OBJT_VNODE) {  in vm_object_deallocate()
648 				vm_object_deallocate_vnode(object);  in vm_object_deallocate()
651 			VM_OBJECT_RUNLOCK(object);  in vm_object_deallocate()
654 		VM_OBJECT_WLOCK(object);  in vm_object_deallocate()
655 		KASSERT(object->ref_count > 0,  in vm_object_deallocate()
657 		    object->type));  in vm_object_deallocate()
663 		if (!refcount_release(&object->ref_count)) {  in vm_object_deallocate()
664 			if (object->ref_count > 1 ||  in vm_object_deallocate()
665 			    atomic_load_int(&object->shadow_count) == 0) {  in vm_object_deallocate()
666 				if ((object->flags & OBJ_ANON) != 0 &&  in vm_object_deallocate()
667 				    object->ref_count == 1)  in vm_object_deallocate()
668 					vm_object_set_flag(object,  in vm_object_deallocate()
670 				VM_OBJECT_WUNLOCK(object);  in vm_object_deallocate()
675 			object = vm_object_deallocate_anon(object);  in vm_object_deallocate()
683 		umtx_shm_object_terminated(object);  in vm_object_deallocate()
684 		temp = object->backing_object;  in vm_object_deallocate()
686 			KASSERT(object->type == OBJT_DEFAULT ||  in vm_object_deallocate()
687 			    object->type == OBJT_SWAP,  in vm_object_deallocate()
688 			    ("shadowed tmpfs v_object 2 %p", object));  in vm_object_deallocate()
689 			vm_object_backing_remove(object);  in vm_object_deallocate()
692 		KASSERT((object->flags & OBJ_DEAD) == 0,  in vm_object_deallocate()
694 		vm_object_set_flag(object, OBJ_DEAD);  in vm_object_deallocate()
695 		vm_object_terminate(object);  in vm_object_deallocate()
696 		object = temp;  in vm_object_deallocate()
705 vm_object_destroy(vm_object_t object)  in vm_object_destroy()  argument
711 	if (object->cred != NULL) {  in vm_object_destroy()
712 		swap_release_by_cred(object->charge, object->cred);  in vm_object_destroy()
713 		object->charge = 0;  in vm_object_destroy()
714 		crfree(object->cred);  in vm_object_destroy()
715 		object->cred = NULL;  in vm_object_destroy()
721 	uma_zfree(obj_zone, object);  in vm_object_destroy()
725 vm_object_sub_shadow(vm_object_t object)  in vm_object_sub_shadow()  argument
727 	KASSERT(object->shadow_count >= 1,  in vm_object_sub_shadow()
728 	    ("object %p sub_shadow count zero", object));  in vm_object_sub_shadow()
729 	atomic_subtract_int(&object->shadow_count, 1);  in vm_object_sub_shadow()
733 vm_object_backing_remove_locked(vm_object_t object)  in vm_object_backing_remove_locked()  argument
737 	backing_object = object->backing_object;  in vm_object_backing_remove_locked()
738 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove_locked()
741 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_backing_remove_locked()
745 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove_locked()
746 		LIST_REMOVE(object, shadow_list);  in vm_object_backing_remove_locked()
747 		object->flags &= ~OBJ_SHADOWLIST;  in vm_object_backing_remove_locked()
749 	object->backing_object = NULL;  in vm_object_backing_remove_locked()
753 vm_object_backing_remove(vm_object_t object)  in vm_object_backing_remove()  argument
757 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_remove()
759 	backing_object = object->backing_object;  in vm_object_backing_remove()
760 	if ((object->flags & OBJ_SHADOWLIST) != 0) {  in vm_object_backing_remove()
762 		vm_object_backing_remove_locked(object);  in vm_object_backing_remove()
765 		object->backing_object = NULL;  in vm_object_backing_remove()
771 vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_locked()  argument
774 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_locked()
779 		LIST_INSERT_HEAD(&backing_object->shadow_head, object,  in vm_object_backing_insert_locked()
781 		object->flags |= OBJ_SHADOWLIST;  in vm_object_backing_insert_locked()
783 	object->backing_object = backing_object;  in vm_object_backing_insert_locked()
787 vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert()  argument
790 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert()
794 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert()
797 		object->backing_object = backing_object;  in vm_object_backing_insert()
807 vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_insert_ref()  argument
810 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_insert_ref()
817 		vm_object_backing_insert_locked(object, backing_object);  in vm_object_backing_insert_ref()
823 		object->backing_object = backing_object;  in vm_object_backing_insert_ref()
831 vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)  in vm_object_backing_transfer()  argument
839 	vm_object_backing_remove_locked(object);  in vm_object_backing_transfer()
846 		vm_object_backing_insert_locked(object, new_backing_object);  in vm_object_backing_transfer()
854 		object->backing_object = new_backing_object;  in vm_object_backing_transfer()
863 vm_object_collapse_wait(vm_object_t object)  in vm_object_collapse_wait()  argument
866 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_wait()
868 	while ((object->flags & OBJ_COLLAPSING) != 0) {  in vm_object_collapse_wait()
869 		vm_object_pip_wait(object, "vmcolwait");  in vm_object_collapse_wait()
879 vm_object_backing_collapse_wait(vm_object_t object)  in vm_object_backing_collapse_wait()  argument
883 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_backing_collapse_wait()
886 		backing_object = object->backing_object;  in vm_object_backing_collapse_wait()
893 		VM_OBJECT_WUNLOCK(object);  in vm_object_backing_collapse_wait()
896 		VM_OBJECT_WLOCK(object);  in vm_object_backing_collapse_wait()
906 vm_object_terminate_pages(vm_object_t object)  in vm_object_terminate_pages()  argument
910 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate_pages()
918 	TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {  in vm_object_terminate_pages()
920 		KASSERT(p->object == object &&  in vm_object_terminate_pages()
924 		p->object = NULL;  in vm_object_terminate_pages()
936 	if (object->resident_page_count != 0) {  in vm_object_terminate_pages()
937 		vm_radix_reclaim_allnodes(&object->rtree);  in vm_object_terminate_pages()
938 		TAILQ_INIT(&object->memq);  in vm_object_terminate_pages()
939 		object->resident_page_count = 0;  in vm_object_terminate_pages()
940 		if (object->type == OBJT_VNODE)  in vm_object_terminate_pages()
941 			vdrop(object->handle);  in vm_object_terminate_pages()
953 vm_object_terminate(vm_object_t object)  in vm_object_terminate()  argument
956 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_terminate()
957 	KASSERT((object->flags & OBJ_DEAD) != 0,  in vm_object_terminate()
958 	    ("terminating non-dead obj %p", object));  in vm_object_terminate()
959 	KASSERT((object->flags & OBJ_COLLAPSING) == 0,  in vm_object_terminate()
960 	    ("terminating collapsing obj %p", object));  in vm_object_terminate()
961 	KASSERT(object->backing_object == NULL,  in vm_object_terminate()
962 	    ("terminating shadow obj %p", object));  in vm_object_terminate()
971 	vm_object_pip_wait(object, "objtrm");  in vm_object_terminate()
973 	KASSERT(object->ref_count == 0,  in vm_object_terminate()
975 	    object->ref_count));  in vm_object_terminate()
977 	if ((object->flags & OBJ_PG_DTOR) == 0)  in vm_object_terminate()
978 		vm_object_terminate_pages(object);  in vm_object_terminate()
981 	if (__predict_false(!LIST_EMPTY(&object->rvq)))  in vm_object_terminate()
982 		vm_reserv_break_all(object);  in vm_object_terminate()
985 	KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||  in vm_object_terminate()
986 	    (object->flags & OBJ_SWAP) != 0,  in vm_object_terminate()
987 	    ("%s: non-swap obj %p has cred", __func__, object));  in vm_object_terminate()
992 	vm_pager_deallocate(object);  in vm_object_terminate()
993 	VM_OBJECT_WUNLOCK(object);  in vm_object_terminate()
995 	vm_object_destroy(object);  in vm_object_terminate()
1047 vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,  in vm_object_page_clean()  argument
1055 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_clean()
1057 	if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)  in vm_object_page_clean()
1065 	tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);  in vm_object_page_clean()
1066 	allclean = tstart == 0 && tend >= object->size;  in vm_object_page_clean()
1070 	curgeneration = object->generation;  in vm_object_page_clean()
1072 	for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {  in vm_object_page_clean()
1080 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1083 			np = vm_page_find_least(object, pi);  in vm_object_page_clean()
1090 		if (object->type == OBJT_VNODE) {  in vm_object_page_clean()
1091 			n = vm_object_page_collect_flush(object, p, pagerflags,  in vm_object_page_clean()
1097 			if (object->generation != curgeneration &&  in vm_object_page_clean()
1121 		np = vm_page_find_least(object, pi + n);  in vm_object_page_clean()
1132 	if (allclean && object->type == OBJT_VNODE)  in vm_object_page_clean()
1133 		object->cleangeneration = curgeneration;  in vm_object_page_clean()
1138 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,  in vm_object_page_collect_flush()  argument
1146 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_collect_flush()
1195 vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,  in vm_object_sync()  argument
1204 	if (object == NULL)  in vm_object_sync()
1208 	VM_OBJECT_WLOCK(object);  in vm_object_sync()
1209 	while ((backing_object = object->backing_object) != NULL) {  in vm_object_sync()
1211 		offset += object->backing_object_offset;  in vm_object_sync()
1212 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1213 		object = backing_object;  in vm_object_sync()
1214 		if (object->size < OFF_TO_IDX(offset + size))  in vm_object_sync()
1215 			size = IDX_TO_OFF(object->size) - offset;  in vm_object_sync()
1229 	if (object->type == OBJT_VNODE &&  in vm_object_sync()
1230 	    vm_object_mightbedirty(object) != 0 &&  in vm_object_sync()
1231 	    ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {  in vm_object_sync()
1232 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1236 		    atop(size) == object->size) {  in vm_object_sync()
1250 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1251 		res = vm_object_page_clean(object, offset, offset + size,  in vm_object_sync()
1253 		VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1260 		VM_OBJECT_WLOCK(object);  in vm_object_sync()
1262 	if ((object->type == OBJT_VNODE ||  in vm_object_sync()
1263 	     object->type == OBJT_DEVICE) && invalidate) {  in vm_object_sync()
1264 		if (object->type == OBJT_DEVICE)  in vm_object_sync()
1275 		vm_object_page_remove(object, OFF_TO_IDX(offset),  in vm_object_sync()
1278 	VM_OBJECT_WUNLOCK(object);  in vm_object_sync()
1289 vm_object_advice_applies(vm_object_t object, int advice)  in vm_object_advice_applies()  argument
1292 	if ((object->flags & OBJ_UNMANAGED) != 0)  in vm_object_advice_applies()
1296 	return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==  in vm_object_advice_applies()
1301 vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,  in vm_object_madvise_freespace()  argument
1306 		vm_pager_freespace(object, pindex, size);  in vm_object_madvise_freespace()
1331 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,  in vm_object_madvise()  argument
1338 	if (object == NULL)  in vm_object_madvise()
1342 	VM_OBJECT_WLOCK(object);  in vm_object_madvise()
1343 	if (!vm_object_advice_applies(object, advice)) {  in vm_object_madvise()
1344 		VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1347 	for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {  in vm_object_madvise()
1348 		tobject = object;  in vm_object_madvise()
1362 			if (object->backing_object == NULL) {  in vm_object_madvise()
1365 				vm_object_madvise_freespace(object, advice,  in vm_object_madvise()
1386 				if (tobject != object)  in vm_object_madvise()
1410 			if (object != tobject)  in vm_object_madvise()
1411 				VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1428 		if (tobject != object)  in vm_object_madvise()
1431 	VM_OBJECT_WUNLOCK(object);  in vm_object_madvise()
1445 vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,  in vm_object_shadow()  argument
1451 	source = *object;  in vm_object_shadow()
1514 	*object = result;  in vm_object_shadow()
1532 	orig_object = entry->object.vm_object;  in vm_object_split()
1670 	entry->object.vm_object = new_object;  in vm_object_split()
1677 vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)  in vm_object_collapse_scan_wait()  argument
1681 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan_wait()
1682 	backing_object = object->backing_object;  in vm_object_collapse_scan_wait()
1685 	KASSERT(p == NULL || p->object == object || p->object == backing_object,  in vm_object_collapse_scan_wait()
1686 	    ("invalid ownership %p %p %p", p, object, backing_object));  in vm_object_collapse_scan_wait()
1689 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1692 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1693 	} else if (p->object == object) {  in vm_object_collapse_scan_wait()
1696 			VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1698 		VM_OBJECT_WUNLOCK(object);  in vm_object_collapse_scan_wait()
1701 		VM_OBJECT_WLOCK(object);  in vm_object_collapse_scan_wait()
1708 vm_object_scan_all_shadowed(vm_object_t object)  in vm_object_scan_all_shadowed()  argument
1714 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_scan_all_shadowed()
1715 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in vm_object_scan_all_shadowed()
1717 	backing_object = object->backing_object;  in vm_object_scan_all_shadowed()
1722 	pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in vm_object_scan_all_shadowed()
1743 		if (new_pindex >= object->size)  in vm_object_scan_all_shadowed()
1773 		pp = vm_page_lookup(object, new_pindex);  in vm_object_scan_all_shadowed()
1781 		    !vm_pager_has_page(object, new_pindex, NULL, NULL))  in vm_object_scan_all_shadowed()
1795 vm_object_collapse_scan(vm_object_t object)  in vm_object_collapse_scan()  argument
1801 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse_scan()
1802 	VM_OBJECT_ASSERT_WLOCKED(object->backing_object);  in vm_object_collapse_scan()
1804 	backing_object = object->backing_object;  in vm_object_collapse_scan()
1805 	backing_offset_index = OFF_TO_IDX(object->backing_object_offset);  in vm_object_collapse_scan()
1818 			next = vm_object_collapse_scan_wait(object, p);  in vm_object_collapse_scan()
1822 		KASSERT(object->backing_object == backing_object,  in vm_object_collapse_scan()
1824 		    object->backing_object, backing_object));  in vm_object_collapse_scan()
1825 		KASSERT(p->object == backing_object,  in vm_object_collapse_scan()
1827 		    p->object, backing_object));  in vm_object_collapse_scan()
1830 		    new_pindex >= object->size) {  in vm_object_collapse_scan()
1848 		pp = vm_page_lookup(object, new_pindex);  in vm_object_collapse_scan()
1857 			next = vm_object_collapse_scan_wait(object, pp);  in vm_object_collapse_scan()
1872 		if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,  in vm_object_collapse_scan()
1897 		if (vm_page_rename(p, object, new_pindex)) {  in vm_object_collapse_scan()
1899 			next = vm_object_collapse_scan_wait(object, NULL);  in vm_object_collapse_scan()
1911 		vm_reserv_rename(p, object, backing_object,  in vm_object_collapse_scan()
1927 vm_object_collapse(vm_object_t object)  in vm_object_collapse()  argument
1931 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_collapse()
1934 		KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,  in vm_object_collapse()
1942 		backing_object = vm_object_backing_collapse_wait(object);  in vm_object_collapse()
1946 		KASSERT(object->ref_count > 0 &&  in vm_object_collapse()
1947 		    object->ref_count > atomic_load_int(&object->shadow_count),  in vm_object_collapse()
1949 		    object->ref_count, atomic_load_int(&object->shadow_count)));  in vm_object_collapse()
1953 		KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,  in vm_object_collapse()
1967 			vm_object_pip_add(object, 1);  in vm_object_collapse()
1968 			vm_object_set_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
1976 			vm_object_collapse_scan(object);  in vm_object_collapse()
2000 				    object,  in vm_object_collapse()
2001 				    OFF_TO_IDX(object->backing_object_offset), TRUE);  in vm_object_collapse()
2007 			vm_object_clear_flag(object, OBJ_COLLAPSING);  in vm_object_collapse()
2008 			vm_object_backing_transfer(object, backing_object);  in vm_object_collapse()
2009 			object->backing_object_offset +=  in vm_object_collapse()
2011 			VM_OBJECT_WUNLOCK(object);  in vm_object_collapse()
2012 			vm_object_pip_wakeup(object);  in vm_object_collapse()
2028 			VM_OBJECT_WLOCK(object);  in vm_object_collapse()
2037 			if (!vm_object_scan_all_shadowed(object)) {  in vm_object_collapse()
2047 			vm_object_backing_remove_locked(object);  in vm_object_collapse()
2050 				vm_object_backing_insert_ref(object,  in vm_object_collapse()
2052 				object->backing_object_offset +=  in vm_object_collapse()
2101 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,  in vm_object_page_remove()  argument
2106 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_page_remove()
2107 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||  in vm_object_page_remove()
2109 	    ("vm_object_page_remove: illegal options for object %p", object));  in vm_object_page_remove()
2110 	if (object->resident_page_count == 0)  in vm_object_page_remove()
2112 	vm_object_pip_add(object, 1);  in vm_object_page_remove()
2114 	p = vm_page_find_least(object, start);  in vm_object_page_remove()
2148 				VM_OBJECT_WLOCK(object);  in vm_object_page_remove()
2158 			    object->ref_count != 0)  in vm_object_page_remove()
2172 			    object->ref_count != 0 &&  in vm_object_page_remove()
2181 		    object->ref_count != 0 && !vm_page_try_remove_all(p))  in vm_object_page_remove()
2185 	vm_object_pip_wakeup(object);  in vm_object_page_remove()
2187 	vm_pager_freespace(object, start, (end == 0 ? object->size : end) -  in vm_object_page_remove()
2208 vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_page_noreuse()  argument
2212 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_page_noreuse()
2213 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,  in vm_object_page_noreuse()
2214 	    ("vm_object_page_noreuse: illegal object %p", object));  in vm_object_page_noreuse()
2215 	if (object->resident_page_count == 0)  in vm_object_page_noreuse()
2217 	p = vm_page_find_least(object, start);  in vm_object_page_noreuse()
2240 vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)  in vm_object_populate()  argument
2246 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_object_populate()
2248 		rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);  in vm_object_populate()
2258 		m = vm_page_lookup(object, start);  in vm_object_populate()
2378 vm_object_set_writeable_dirty_(vm_object_t object)  in vm_object_set_writeable_dirty_()  argument
2380 	atomic_add_int(&object->generation, 1);  in vm_object_set_writeable_dirty_()
2384 vm_object_mightbedirty_(vm_object_t object)  in vm_object_mightbedirty_()  argument
2386 	return (object->generation != object->cleangeneration);  in vm_object_mightbedirty_()
2398 vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,  in vm_object_unwire()  argument
2411 	if ((object->flags & OBJ_FICTITIOUS) != 0)  in vm_object_unwire()
2417 	VM_OBJECT_RLOCK(object);  in vm_object_unwire()
2418 	m = vm_page_find_least(object, pindex);  in vm_object_unwire()
2426 			tobject = object;  in vm_object_unwire()
2449 			for (tobject = object; locked_depth >= 1;  in vm_object_unwire()
2452 				if (tm->object != tobject)  in vm_object_unwire()
2456 			tobject = tm->object;  in vm_object_unwire()
2468 	for (tobject = object; locked_depth >= 1; locked_depth--) {  in vm_object_unwire()
2481 vm_object_vnode(vm_object_t object)  in vm_object_vnode()  argument
2485 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_object_vnode()
2486 	vm_pager_getvp(object, &vp, NULL);  in vm_object_vnode()
2699 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)  in _vm_object_in_map()  argument
2710 			if (_vm_object_in_map(map, object, tmpe)) {  in _vm_object_in_map()
2715 		tmpm = entry->object.sub_map;  in _vm_object_in_map()
2717 			if (_vm_object_in_map(tmpm, object, tmpe)) {  in _vm_object_in_map()
2721 	} else if ((obj = entry->object.vm_object) != NULL) {  in _vm_object_in_map()
2723 			if (obj == object) {  in _vm_object_in_map()
2731 vm_object_in_map(vm_object_t object)  in vm_object_in_map()  argument
2739 		if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {  in vm_object_in_map()
2745 	if (_vm_object_in_map(kernel_map, object, 0))  in vm_object_in_map()
2752 	vm_object_t object;  in DB_SHOW_COMMAND()  local
2758 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND()
2759 		if ((object->flags & OBJ_ANON) != 0) {  in DB_SHOW_COMMAND()
2760 			if (object->ref_count == 0) {  in DB_SHOW_COMMAND()
2762 					(long)object->size);  in DB_SHOW_COMMAND()
2764 			if (!vm_object_in_map(object)) {  in DB_SHOW_COMMAND()
2768 				    object->ref_count, (u_long)object->size,   in DB_SHOW_COMMAND()
2769 				    (u_long)object->size,  in DB_SHOW_COMMAND()
2770 				    (void *)object->backing_object);  in DB_SHOW_COMMAND()
2781 DB_SHOW_COMMAND(object, vm_object_print_static)  in DB_SHOW_COMMAND()  argument
2784 	vm_object_t object = (vm_object_t)addr;  in DB_SHOW_COMMAND()  local
2794 	if (object == NULL)  in DB_SHOW_COMMAND()
2799 	    object, (int)object->type, (uintmax_t)object->size,  in DB_SHOW_COMMAND()
2800 	    object->resident_page_count, object->ref_count, object->flags,  in DB_SHOW_COMMAND()
2801 	    object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);  in DB_SHOW_COMMAND()
2803 	    atomic_load_int(&object->shadow_count),  in DB_SHOW_COMMAND()
2804 	    object->backing_object ? object->backing_object->ref_count : 0,  in DB_SHOW_COMMAND()
2805 	    object->backing_object, (uintmax_t)object->backing_object_offset);  in DB_SHOW_COMMAND()
2812 	TAILQ_FOREACH(p, &object->memq, listq) {  in DB_SHOW_COMMAND()
2850 	vm_object_t object;  in DB_SHOW_COMMAND()  local
2856 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in DB_SHOW_COMMAND()
2857 		db_printf("new object: %p\n", (void *)object);  in DB_SHOW_COMMAND()
2864 		TAILQ_FOREACH(m, &object->memq, listq) {  in DB_SHOW_COMMAND()