177 static int vm_page_insert_after(vm_page_t m, vm_object_t object,
179 static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
186 static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
504 	m->object = NULL;  in vm_page_init_page()
885 	obj = atomic_load_ptr(&m->object);  in vm_page_busy_acquire()
901 		KASSERT(m->object == obj || m->object == NULL,  in vm_page_busy_acquire()
1018 	obj = m->object;  in vm_page_busy_sleep()
1091 		    m->object != obj || m->pindex != pindex) {  in _vm_page_busy_sleep()
1120 	obj = m->object;  in vm_page_trysbusy()
1137 	obj = m->object;  in vm_page_trysbusy()
1161 	obj = m->object;  in vm_page_tryxbusy()
1382 	KASSERT(m->object != NULL, ("page %p has no object", m));  in vm_page_free_invalid()
1383 	VM_OBJECT_ASSERT_WLOCKED(m->object);  in vm_page_free_invalid()
1417 	VM_OBJECT_ASSERT_WLOCKED(m->object);  in vm_page_sleep_if_busy()
1426 	obj = m->object;  in vm_page_sleep_if_busy()
1449 	VM_OBJECT_ASSERT_WLOCKED(m->object);  in vm_page_sleep_if_xbusy()
1458 	obj = m->object;  in vm_page_sleep_if_xbusy()
1496 vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)  in vm_page_insert()  argument
1500 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_insert()
1501 	mpred = vm_radix_lookup_le(&object->rtree, pindex);  in vm_page_insert()
1502 	return (vm_page_insert_after(m, object, pindex, mpred));  in vm_page_insert()
1516 vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex,  in vm_page_insert_after()  argument
1521 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_insert_after()
1522 	KASSERT(m->object == NULL,  in vm_page_insert_after()
1525 		KASSERT(mpred->object == object,  in vm_page_insert_after()
1531 		msucc = TAILQ_FIRST(&object->memq);  in vm_page_insert_after()
1539 	m->object = object;  in vm_page_insert_after()
1546 	if (vm_radix_insert(&object->rtree, m)) {  in vm_page_insert_after()
1547 		m->object = NULL;  in vm_page_insert_after()
1552 	vm_page_insert_radixdone(m, object, mpred);  in vm_page_insert_after()
1568 vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred)  in vm_page_insert_radixdone()  argument
1571 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_insert_radixdone()
1572 	KASSERT(object != NULL && m->object == object,  in vm_page_insert_radixdone()
1577 		KASSERT(mpred->object == object,  in vm_page_insert_radixdone()
1584 		TAILQ_INSERT_AFTER(&object->memq, mpred, m, listq);  in vm_page_insert_radixdone()
1586 		TAILQ_INSERT_HEAD(&object->memq, m, listq);  in vm_page_insert_radixdone()
1591 	object->resident_page_count++;  in vm_page_insert_radixdone()
1596 	if (object->resident_page_count == 1 && object->type == OBJT_VNODE)  in vm_page_insert_radixdone()
1597 		vhold(object->handle);  in vm_page_insert_radixdone()
1604 		vm_object_set_writeable_dirty(object);  in vm_page_insert_radixdone()
1614 	vm_object_t object;  in vm_page_object_remove()  local
1618 	object = m->object;  in vm_page_object_remove()
1619 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_object_remove()
1627 	m->object = NULL;  in vm_page_object_remove()
1628 	mrem = vm_radix_remove(&object->rtree, m->pindex);  in vm_page_object_remove()
1634 	TAILQ_REMOVE(&object->memq, m, listq);  in vm_page_object_remove()
1639 	object->resident_page_count--;  in vm_page_object_remove()
1644 	if (object->resident_page_count == 0 && object->type == OBJT_VNODE)  in vm_page_object_remove()
1645 		vdrop(object->handle);  in vm_page_object_remove()
1694 vm_page_lookup(vm_object_t object, vm_pindex_t pindex)  in vm_page_lookup()  argument
1697 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_page_lookup()
1698 	return (vm_radix_lookup(&object->rtree, pindex));  in vm_page_lookup()
1710 vm_page_lookup_unlocked(vm_object_t object, vm_pindex_t pindex)  in vm_page_lookup_unlocked()  argument
1713 	return (vm_radix_lookup_unlocked(&object->rtree, pindex));  in vm_page_lookup_unlocked()
1723 vm_page_relookup(vm_object_t object, vm_pindex_t pindex)  in vm_page_relookup()  argument
1727 	m = vm_radix_lookup_unlocked(&object->rtree, pindex);  in vm_page_relookup()
1729 	    m->object == object && m->pindex == pindex,  in vm_page_relookup()
1775 vm_page_find_least(vm_object_t object, vm_pindex_t pindex)  in vm_page_find_least()  argument
1779 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_page_find_least()
1780 	if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex)  in vm_page_find_least()
1781 		m = vm_radix_lookup_ge(&object->rtree, pindex);  in vm_page_find_least()
1796 	VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_page_next()
1798 		MPASS(next->object == m->object);  in vm_page_next()
1816 	VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_page_prev()
1818 		MPASS(prev->object == m->object);  in vm_page_prev()
1837 vm_page_replace_hold(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,  in vm_page_replace_hold()  argument
1843 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_replace_hold()
1845 	KASSERT(mnew->object == NULL && (mnew->ref_count & VPRC_OBJREF) == 0,  in vm_page_replace_hold()
1854 	mnew->object = object;  in vm_page_replace_hold()
1857 	mret = vm_radix_replace(&object->rtree, mnew);  in vm_page_replace_hold()
1865 	TAILQ_INSERT_AFTER(&object->memq, mold, mnew, listq);  in vm_page_replace_hold()
1866 	TAILQ_REMOVE(&object->memq, mold, listq);  in vm_page_replace_hold()
1867 	mold->object = NULL;  in vm_page_replace_hold()
1875 		vm_object_set_writeable_dirty(object);  in vm_page_replace_hold()
1883 vm_page_replace(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex,  in vm_page_replace()  argument
1889 	if (vm_page_replace_hold(mnew, object, pindex, mold))  in vm_page_replace()
1945 	m->object = new_object;  in vm_page_rename()
1977 vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)  in vm_page_alloc()  argument
1980 	return (vm_page_alloc_after(object, pindex, req, object != NULL ?  in vm_page_alloc()
1981 	    vm_radix_lookup_le(&object->rtree, pindex) : NULL));  in vm_page_alloc()
1985 vm_page_alloc_domain(vm_object_t object, vm_pindex_t pindex, int domain,  in vm_page_alloc_domain()  argument
1989 	return (vm_page_alloc_domain_after(object, pindex, domain, req,  in vm_page_alloc_domain()
1990 	    object != NULL ? vm_radix_lookup_le(&object->rtree, pindex) :  in vm_page_alloc_domain()
2001 vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex,  in vm_page_alloc_after()  argument
2008 	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);  in vm_page_alloc_after()
2010 		m = vm_page_alloc_domain_after(object, pindex, domain, req,  in vm_page_alloc_after()
2014 	} while (vm_domainset_iter_page(&di, object, &domain) == 0);  in vm_page_alloc_after()
2073 vm_page_alloc_domain_after(vm_object_t object, vm_pindex_t pindex, int domain,  in vm_page_alloc_domain_after()  argument
2080 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&  in vm_page_alloc_domain_after()
2081 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&  in vm_page_alloc_domain_after()
2084 	    ("inconsistent object(%p)/req(%x)", object, req));  in vm_page_alloc_domain_after()
2085 	KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,  in vm_page_alloc_domain_after()
2090 	if (object != NULL)  in vm_page_alloc_domain_after()
2091 		VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_alloc_domain_after()
2095 	pool = object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT;  in vm_page_alloc_domain_after()
2101 	if (vm_object_reserv(object) &&  in vm_page_alloc_domain_after()
2102 	    (m = vm_reserv_alloc_page(object, pindex, domain, req, mpred)) !=  in vm_page_alloc_domain_after()
2134 		if (vm_domain_alloc_fail(vmd, object, req))  in vm_page_alloc_domain_after()
2155 	m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ?  in vm_page_alloc_domain_after()
2169 	if (object != NULL) {  in vm_page_alloc_domain_after()
2170 		if (vm_page_insert_after(m, object, pindex, mpred)) {  in vm_page_alloc_domain_after()
2175 			KASSERT(m->object == NULL, ("page %p has object", m));  in vm_page_alloc_domain_after()
2181 				VM_OBJECT_WUNLOCK(object);  in vm_page_alloc_domain_after()
2183 				VM_OBJECT_WLOCK(object);  in vm_page_alloc_domain_after()
2189 		if (object->memattr != VM_MEMATTR_DEFAULT &&  in vm_page_alloc_domain_after()
2190 		    (object->flags & OBJ_FICTITIOUS) == 0)  in vm_page_alloc_domain_after()
2191 			pmap_page_set_memattr(m, object->memattr);  in vm_page_alloc_domain_after()
2238 vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,  in vm_page_alloc_contig()  argument
2246 	vm_domainset_iter_page_init(&di, object, pindex, &domain, &req);  in vm_page_alloc_contig()
2248 		m = vm_page_alloc_contig_domain(object, pindex, domain, req,  in vm_page_alloc_contig()
2252 	} while (vm_domainset_iter_page(&di, object, &domain) == 0);  in vm_page_alloc_contig()
2258 vm_page_alloc_contig_domain(vm_object_t object, vm_pindex_t pindex, int domain,  in vm_page_alloc_contig_domain()  argument
2267 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&  in vm_page_alloc_contig_domain()
2268 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&  in vm_page_alloc_contig_domain()
2271 	    ("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object,  in vm_page_alloc_contig_domain()
2273 	KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,  in vm_page_alloc_contig_domain()
2275 	if (object != NULL) {  in vm_page_alloc_contig_domain()
2276 		VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_alloc_contig_domain()
2277 		KASSERT((object->flags & OBJ_FICTITIOUS) == 0,  in vm_page_alloc_contig_domain()
2279 		    object));  in vm_page_alloc_contig_domain()
2283 	if (object != NULL) {  in vm_page_alloc_contig_domain()
2284 		mpred = vm_radix_lookup_le(&object->rtree, pindex);  in vm_page_alloc_contig_domain()
2299 	if (vm_object_reserv(object) &&  in vm_page_alloc_contig_domain()
2300 	    (m_ret = vm_reserv_alloc_contig(object, pindex, domain, req,  in vm_page_alloc_contig_domain()
2324 		if (vm_domain_alloc_fail(vmd, object, req))  in vm_page_alloc_contig_domain()
2344 	oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ?  in vm_page_alloc_contig_domain()
2354 	if (object != NULL) {  in vm_page_alloc_contig_domain()
2355 		if (object->memattr != VM_MEMATTR_DEFAULT &&  in vm_page_alloc_contig_domain()
2357 			memattr = object->memattr;  in vm_page_alloc_contig_domain()
2367 		if (object != NULL) {  in vm_page_alloc_contig_domain()
2368 			if (vm_page_insert_after(m, object, pindex, mpred)) {  in vm_page_alloc_contig_domain()
2371 				KASSERT(m->object == NULL,  in vm_page_alloc_contig_domain()
2384 					VM_OBJECT_WUNLOCK(object);  in vm_page_alloc_contig_domain()
2386 					VM_OBJECT_WLOCK(object);  in vm_page_alloc_contig_domain()
2407 	KASSERT(m->object == NULL, ("page %p has object", m));  in vm_page_alloc_check()
2572 	vm_object_t object;  in vm_page_scan_contig()  local
2629 		else if ((object = atomic_load_ptr(&m->object)) != NULL) {  in vm_page_scan_contig()
2635 			VM_OBJECT_RLOCK(object);  in vm_page_scan_contig()
2636 			if (object != m->object) {  in vm_page_scan_contig()
2637 				VM_OBJECT_RUNLOCK(object);  in vm_page_scan_contig()
2641 			if (object->type != OBJT_DEFAULT &&  in vm_page_scan_contig()
2642 			    object->type != OBJT_SWAP &&  in vm_page_scan_contig()
2643 			    object->type != OBJT_VNODE) {  in vm_page_scan_contig()
2654 			} else if (object->memattr == VM_MEMATTR_DEFAULT &&  in vm_page_scan_contig()
2671 			VM_OBJECT_RUNLOCK(object);  in vm_page_scan_contig()
2750 	vm_object_t object;  in vm_page_reclaim_run()  local
2771 		else if ((object = atomic_load_ptr(&m->object)) != NULL) {  in vm_page_reclaim_run()
2776 			VM_OBJECT_WLOCK(object);  in vm_page_reclaim_run()
2778 			if (m->object != object ||  in vm_page_reclaim_run()
2779 			    (object->type != OBJT_DEFAULT &&  in vm_page_reclaim_run()
2780 			    object->type != OBJT_SWAP &&  in vm_page_reclaim_run()
2781 			    object->type != OBJT_VNODE))  in vm_page_reclaim_run()
2783 			else if (object->memattr != VM_MEMATTR_DEFAULT)  in vm_page_reclaim_run()
2846 					if (object->ref_count != 0 &&  in vm_page_reclaim_run()
2870 					if (vm_page_replace_hold(m_new, object,  in vm_page_reclaim_run()
2893 			VM_OBJECT_WUNLOCK(object);  in vm_page_reclaim_run()
3292 vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req)  in vm_domain_alloc_fail()  argument
3300 		if (object != NULL)   in vm_domain_alloc_fail()
3301 			VM_OBJECT_WUNLOCK(object);  in vm_domain_alloc_fail()
3303 		if (object != NULL)   in vm_domain_alloc_fail()
3304 			VM_OBJECT_WLOCK(object);  in vm_domain_alloc_fail()
3772 	if (m->object != NULL) {  in vm_page_free_prep()
3774 		    ((m->object->flags & OBJ_UNMANAGED) != 0),  in vm_page_free_prep()
3906 	if (m->object != NULL && !vm_page_busied(m) &&  in vm_page_wire()
3907 	    !vm_object_busied(m->object))  in vm_page_wire()
3908 		VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_page_wire()
4194 	vm_object_t object;  in vm_page_release()  local
4201 			object = atomic_load_ptr(&m->object);  in vm_page_release()
4202 			if (object == NULL)  in vm_page_release()
4205 			if (vm_page_busied(m) || !VM_OBJECT_TRYWLOCK(object))  in vm_page_release()
4207 			if (object == m->object) {  in vm_page_release()
4209 				VM_OBJECT_WUNLOCK(object);  in vm_page_release()
4212 			VM_OBJECT_WUNLOCK(object);  in vm_page_release()
4223 	VM_OBJECT_ASSERT_WLOCKED(m->object);  in vm_page_release_locked()
4229 		    (m->object->ref_count == 0 || !pmap_page_is_mapped(m)) &&  in vm_page_release_locked()
4252 	KASSERT(m->object != NULL && (m->oflags & VPO_UNMANAGED) == 0,  in vm_page_try_blocked_op()
4256 	VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_page_try_blocked_op()
4273 	KASSERT(!VM_OBJECT_WOWNED(m->object) ||  in vm_page_try_blocked_op()
4309 	VM_OBJECT_ASSERT_WLOCKED(m->object);  in vm_page_advise()
4375 vm_page_grab_sleep(vm_object_t object, vm_page_t m, vm_pindex_t pindex,  in vm_page_grab_sleep()  argument
4389 	if (_vm_page_busy_sleep(object, m, pindex, wmesg, allocflags, locked) &&  in vm_page_grab_sleep()
4391 		VM_OBJECT_WLOCK(object);  in vm_page_grab_sleep()
4445 vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)  in vm_page_grab()  argument
4449 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_grab()
4453 	if ((m = vm_page_lookup(object, pindex)) != NULL) {  in vm_page_grab()
4455 			if (vm_page_grab_sleep(object, m, pindex, "pgrbwt",  in vm_page_grab()
4464 	m = vm_page_alloc(object, pindex, vm_page_grab_pflags(allocflags));  in vm_page_grab()
4497 vm_page_acquire_unlocked(vm_object_t object, vm_pindex_t pindex,  in vm_page_acquire_unlocked()  argument
4514 		    atomic_load_ptr(&m->object) != object) {  in vm_page_acquire_unlocked()
4520 			m = vm_radix_lookup_unlocked(&object->rtree, pindex);  in vm_page_acquire_unlocked()
4525 			if (m->object == object && m->pindex == pindex)  in vm_page_acquire_unlocked()
4532 		if (!vm_page_grab_sleep(object, m, pindex, "pgnslp",  in vm_page_acquire_unlocked()
4548 vm_page_grab_unlocked(vm_object_t object, vm_pindex_t pindex, int allocflags)  in vm_page_grab_unlocked()  argument
4554 	if (!vm_page_acquire_unlocked(object, pindex, NULL, &m, allocflags))  in vm_page_grab_unlocked()
4568 	VM_OBJECT_WLOCK(object);  in vm_page_grab_unlocked()
4569 	m = vm_page_grab(object, pindex, allocflags);  in vm_page_grab_unlocked()
4570 	VM_OBJECT_WUNLOCK(object);  in vm_page_grab_unlocked()
4583 vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)  in vm_page_grab_valid()  argument
4595 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_grab_valid()
4601 	if ((m = vm_page_lookup(object, pindex)) != NULL) {  in vm_page_grab_valid()
4613 			(void)vm_page_grab_sleep(object, m, pindex, "pgrbwt",  in vm_page_grab_valid()
4627 	} else if ((m = vm_page_alloc(object, pindex, pflags)) == NULL) {  in vm_page_grab_valid()
4632 	if (vm_pager_has_page(object, pindex, NULL, &after)) {  in vm_page_grab_valid()
4642 				ma[i] = vm_page_alloc(object, m->pindex + i,  in vm_page_grab_valid()
4649 		vm_object_pip_add(object, after);  in vm_page_grab_valid()
4650 		VM_OBJECT_WUNLOCK(object);  in vm_page_grab_valid()
4651 		rv = vm_pager_get_pages(object, ma, after, NULL, NULL);  in vm_page_grab_valid()
4652 		VM_OBJECT_WLOCK(object);  in vm_page_grab_valid()
4653 		vm_object_pip_wakeupn(object, after);  in vm_page_grab_valid()
4688 vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object,  in vm_page_grab_valid_unlocked()  argument
4708 	if (!vm_page_acquire_unlocked(object, pindex, NULL, mp, flags))  in vm_page_grab_valid_unlocked()
4723 	VM_OBJECT_WLOCK(object);  in vm_page_grab_valid_unlocked()
4724 	error = vm_page_grab_valid(mp, object, pindex, allocflags);  in vm_page_grab_valid_unlocked()
4725 	VM_OBJECT_WUNLOCK(object);  in vm_page_grab_valid_unlocked()
4757 vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,  in vm_page_grab_pages()  argument
4764 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_grab_pages()
4774 	m = vm_radix_lookup_le(&object->rtree, pindex + i);  in vm_page_grab_pages()
4783 				if (vm_page_grab_sleep(object, m, pindex + i,  in vm_page_grab_pages()
4791 			m = vm_page_alloc_after(object, pindex + i,  in vm_page_grab_pages()
4818 vm_page_grab_pages_unlocked(vm_object_t object, vm_pindex_t pindex,  in vm_page_grab_pages_unlocked()  argument
4836 		if (!vm_page_acquire_unlocked(object, pindex, pred, &m, flags))  in vm_page_grab_pages_unlocked()
4852 	VM_OBJECT_WLOCK(object);  in vm_page_grab_pages_unlocked()
4853 	i += vm_page_grab_pages(object, pindex, allocflags, &ma[i], count);  in vm_page_grab_pages_unlocked()
4854 	VM_OBJECT_WUNLOCK(object);  in vm_page_grab_pages_unlocked()
5216 	vm_object_t object;  in vm_page_set_invalid()  local
5222 	object = m->object;  in vm_page_set_invalid()
5223 	VM_OBJECT_ASSERT_WLOCKED(object);  in vm_page_set_invalid()
5226 	if (object->type == OBJT_VNODE && base == 0 && IDX_TO_OFF(m->pindex) +  in vm_page_set_invalid()
5227 	    size >= object->un_pager.vnp.vnp_size)  in vm_page_set_invalid()
5231 	if (object->ref_count != 0 && vm_page_all_valid(m) && bits != 0)  in vm_page_set_invalid()
5258 	VM_OBJECT_ASSERT_LOCKED(m->object);  in vm_page_invalid()
5337 	vm_object_t object;  in vm_page_ps_test()  local
5340 	object = m->object;  in vm_page_ps_test()
5341 	if (skip_m != NULL && skip_m->object != object)  in vm_page_ps_test()
5343 	VM_OBJECT_ASSERT_LOCKED(object);  in vm_page_ps_test()
5353 		if (m[i].object != object)  in vm_page_ps_test()
5445 	if (m->object != NULL && !vm_page_busied(m))  in vm_page_object_busy_assert()
5446 		VM_OBJECT_ASSERT_BUSY(m->object);  in vm_page_object_busy_assert()
5464 		VM_OBJECT_ASSERT_BUSY(m->object);  in vm_page_assert_pga_writeable()
5527 	    m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr,  in DB_SHOW_COMMAND()